From 224be546596aae5fe3bcd9c46d3eac3f84b386fd Mon Sep 17 00:00:00 2001
From: Patrick Simianer <p@simianer.de>
Date: Sun, 16 Mar 2014 19:17:24 +0100
Subject: no non printables in preproc

---
 preprocess | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/preprocess b/preprocess
index 4bf782a..f3c3d10 100755
--- a/preprocess
+++ b/preprocess
@@ -1,5 +1,5 @@
 #!/bin/bash
 
 LANG=$1
-/toolbox/scripts/htmlentities 2>htmlentities.$LANG.err | /toolbox/scripts/normalize_punctuation 2>normalize-punctuation.$LANG.err | /toolbox/moses/scripts/tokenizer/tokenizer.no-escape.perl -a -b -threads 1 -l $LANG 2>tokenizer.$LANG.err | /toolbox/moses/scripts/tokenizer/lowercase.perl 2>lowercase.$LANG.err
+/toolbox/scripts/no_non_printables | /toolbox/scripts/htmlentities 2>htmlentities.$LANG.err | /toolbox/scripts/normalize_punctuation 2>normalize-punctuation.$LANG.err | /toolbox/moses/scripts/tokenizer/tokenizer.no-escape.perl -a -b -threads 1 -l $LANG 2>tokenizer.$LANG.err | /toolbox/moses/scripts/tokenizer/lowercase.perl 2>lowercase.$LANG.err
 
-- 
cgit v1.2.3