From 65e999da12ea6d88e7703a89932b30b9ccbfcdb6 Mon Sep 17 00:00:00 2001
From: Michael Denkowski <mdenkows@cs.cmu.edu>
Date: Thu, 5 Sep 2013 11:26:22 -0700
Subject: Unbuffered mode, flush after each line where possible, skip otherwise

---
 corpus/tokenize-anything.sh | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

(limited to 'corpus/tokenize-anything.sh')

diff --git a/corpus/tokenize-anything.sh b/corpus/tokenize-anything.sh
index 028992cf..52739e81 100755
--- a/corpus/tokenize-anything.sh
+++ b/corpus/tokenize-anything.sh
@@ -3,11 +3,17 @@
 ROOTDIR=`dirname $0`
 SUPPORT=$ROOTDIR/support
 
-$SUPPORT/utf8-normalize.sh |
+if [[ $# == 1 && $1 == '-u' ]] ; then
+    NORMCMD=cat
+else
+    NORMCMD=$SUPPORT/utf8-normalize.sh
+fi
+
+$NORMCMD |
   $SUPPORT/quote-norm.pl |
   $SUPPORT/tokenizer.pl |
-  sed -e 's/ al - / al-/g' |
+  sed -u -e 's/ al - / al-/g' |
   $SUPPORT/fix-contract.pl |
-  sed -e 's/^ //' | sed -e 's/ $//' |
-  perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/;  print;}'
+  sed -u -e 's/^ //' | sed -u -e 's/ $//' |
+  perl -e '$|++; while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/;  print;}'
 
-- 
cgit v1.2.3