summaryrefslogtreecommitdiff
path: root/preprocess
diff options
context:
space:
mode:
Diffstat (limited to 'preprocess')
-rwxr-xr-xpreprocess5
1 files changed, 5 insertions, 0 deletions
diff --git a/preprocess b/preprocess
new file mode 100755
index 0000000..716255d
--- /dev/null
+++ b/preprocess
@@ -0,0 +1,5 @@
+#!/bin/zsh
+
+LANG=$1
+~/scripts/htmlentities 2>htmlentities.$LANG.err | ~/scripts/normalize-punctuation 2>normalize-punctuation.$LANG.err | ~/moses/scripts/tokenizer/tokenizer.perl -a -b -threads 1 -l $LANG 2>tokenizer.$LANG.err | ~/moses/scripts/tokenizer/lowercase.perl 2>lowercase.$LANG.err
+