blob: 716255d9b88a224b2c2f7262f695cc0ea7f60489 (
plain)
1
2
3
4
5
|
#!/bin/zsh
LANG=$1
~/scripts/htmlentities 2>htmlentities.$LANG.err | ~/scripts/normalize-punctuation 2>normalize-punctuation.$LANG.err | ~/moses/scripts/tokenizer/tokenizer.perl -a -b -threads 1 -l $LANG 2>tokenizer.$LANG.err | ~/moses/scripts/tokenizer/lowercase.perl 2>lowercase.$LANG.err
|