preprocess-no-lower


1
2
3
4
5
6
7
8

#!/bin/bash

pushd "$(dirname "$0")" > /dev/null
P="$(pwd -P)"
popd > /dev/null

LANG=$1
$P/no-non-printables | sed "s|[-,\.]\{4,\}|...|g" | $P/htmlentities 2>htmlentities.$LANG.err | $P/normalize-punctuation 2>normalize-punctuation.$LANG.err | $P/tokenizer-no-escape.perl -a -b -threads 1 -l $LANG 2>tokenizer.$LANG.err