diff options
| author | Michael Denkowski <mdenkows@cs.cmu.edu> | 2013-09-05 11:26:22 -0700 |
|---|---|---|
| committer | Michael Denkowski <mdenkows@cs.cmu.edu> | 2013-09-05 11:26:22 -0700 |
| commit | 65e999da12ea6d88e7703a89932b30b9ccbfcdb6 (patch) | |
| tree | ae438759a50e6d045a87f6b23808f5e5dd81b309 /corpus/tokenize-anything.sh | |
| parent | cf7f814acfebf03b577e105a9295ef67cfdb74cd (diff) | |
Unbuffered mode, flush after each line where possible, skip otherwise
Diffstat (limited to 'corpus/tokenize-anything.sh')
| -rwxr-xr-x | corpus/tokenize-anything.sh | 14 |
1 files changed, 10 insertions, 4 deletions
diff --git a/corpus/tokenize-anything.sh b/corpus/tokenize-anything.sh index 028992cf..52739e81 100755 --- a/corpus/tokenize-anything.sh +++ b/corpus/tokenize-anything.sh @@ -3,11 +3,17 @@ ROOTDIR=`dirname $0` SUPPORT=$ROOTDIR/support -$SUPPORT/utf8-normalize.sh | +if [[ $# == 1 && $1 == '-u' ]] ; then + NORMCMD=cat +else + NORMCMD=$SUPPORT/utf8-normalize.sh +fi + +$NORMCMD | $SUPPORT/quote-norm.pl | $SUPPORT/tokenizer.pl | - sed -e 's/ al - / al-/g' | + sed -u -e 's/ al - / al-/g' | $SUPPORT/fix-contract.pl | - sed -e 's/^ //' | sed -e 's/ $//' | - perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/; print;}' + sed -u -e 's/^ //' | sed -u -e 's/ $//' | + perl -e '$|++; while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/; print;}' |