From bef5c40937619e8360a87f988f204ba0ff2ad300 Mon Sep 17 00:00:00 2001
From: Michael Denkowski <mdenkows@cs.cmu.edu>
Date: Thu, 5 Sep 2013 14:15:43 -0700
Subject: Slower but correct (wrt buffered) unbuffered version.

---
 corpus/tokenize-anything.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'corpus/tokenize-anything.sh')

diff --git a/corpus/tokenize-anything.sh b/corpus/tokenize-anything.sh
index 52739e81..a20a022f 100755
--- a/corpus/tokenize-anything.sh
+++ b/corpus/tokenize-anything.sh
@@ -4,12 +4,12 @@ ROOTDIR=`dirname $0`
 SUPPORT=$ROOTDIR/support
 
 if [[ $# == 1 && $1 == '-u' ]] ; then
-    NORMCMD=cat
+    NORMARGS="--batchline"
 else
-    NORMCMD=$SUPPORT/utf8-normalize.sh
+    NORMARGS=""
 fi
 
-$NORMCMD |
+$SUPPORT/utf8-normalize.sh $NORMARGS |
   $SUPPORT/quote-norm.pl |
   $SUPPORT/tokenizer.pl |
   sed -u -e 's/ al - / al-/g' |
-- 
cgit v1.2.3