summaryrefslogtreecommitdiff
path: root/corpus
diff options
context:
space:
mode:
authormjdenkowski <michael.j.denkowski@gmail.com>2013-12-12 15:12:25 -0500
committermjdenkowski <michael.j.denkowski@gmail.com>2013-12-12 15:12:25 -0500
commitf39c95a0d0aff298d957aee018f4d6fbe87562a3 (patch)
tree02ce48129cefd6563e8b0881d03cde44fb9f0566 /corpus
parent45d094ed4520b87372ad6a4afb06e23d752129d7 (diff)
Restore unbuffered functionality as option
Diffstat (limited to 'corpus')
-rwxr-xr-xcorpus/tokenize-anything.sh6
1 files changed, 4 insertions, 2 deletions
diff --git a/corpus/tokenize-anything.sh b/corpus/tokenize-anything.sh
index c7adfa61..5b7933d8 100755
--- a/corpus/tokenize-anything.sh
+++ b/corpus/tokenize-anything.sh
@@ -5,15 +5,17 @@ SUPPORT=$ROOTDIR/support
if [[ $# == 1 && $1 == '-u' ]] ; then
NORMARGS="--batchline"
+ SEDFLAGS="-u"
else
NORMARGS=""
+ SEDFLAGS=""
fi
$SUPPORT/utf8-normalize.sh $NORMARGS |
$SUPPORT/quote-norm.pl |
$SUPPORT/tokenizer.pl |
- sed -e 's/ al - / al-/g' |
+ sed $SEDFLAGS -e 's/ al - / al-/g' |
$SUPPORT/fix-contract.pl |
- sed -e 's/^ //' | sed -e 's/ $//' |
+ sed $SEDFLAGS -e 's/^ //' | sed $SEDFLAGS -e 's/ $//' |
perl -e '$|++; while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/; print;}'