From f39c95a0d0aff298d957aee018f4d6fbe87562a3 Mon Sep 17 00:00:00 2001
From: mjdenkowski <michael.j.denkowski@gmail.com>
Date: Thu, 12 Dec 2013 15:12:25 -0500
Subject: Restore unbuffered functionality as option

---
 corpus/tokenize-anything.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'corpus')

diff --git a/corpus/tokenize-anything.sh b/corpus/tokenize-anything.sh
index c7adfa61..5b7933d8 100755
--- a/corpus/tokenize-anything.sh
+++ b/corpus/tokenize-anything.sh
@@ -5,15 +5,17 @@ SUPPORT=$ROOTDIR/support
 
 if [[ $# == 1 && $1 == '-u' ]] ; then
     NORMARGS="--batchline"
+    SEDFLAGS="-u"
 else
     NORMARGS=""
+    SEDFLAGS=""
 fi
 
 $SUPPORT/utf8-normalize.sh $NORMARGS |
   $SUPPORT/quote-norm.pl |
   $SUPPORT/tokenizer.pl |
-  sed -e 's/ al - / al-/g' |
+  sed $SEDFLAGS -e 's/ al - / al-/g' |
   $SUPPORT/fix-contract.pl |
-  sed -e 's/^ //' | sed -e 's/ $//' |
+  sed $SEDFLAGS -e 's/^ //' | sed $SEDFLAGS -e 's/ $//' |
   perl -e '$|++; while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/;  print;}'
 
-- 
cgit v1.2.3