diff options
| -rwxr-xr-x | corpus/support/fix-contract.pl | 2 | ||||
| -rwxr-xr-x | corpus/support/quote-norm.pl | 1 | ||||
| -rwxr-xr-x | corpus/support/tokenizer.pl | 1 | ||||
| -rwxr-xr-x | corpus/support/utf8-normalize.sh | 3 | ||||
| -rwxr-xr-x | corpus/tokenize-anything.sh | 14 |
5 files changed, 16 insertions, 5 deletions
diff --git a/corpus/support/fix-contract.pl b/corpus/support/fix-contract.pl index f1e191ab..49e88981 100755 --- a/corpus/support/fix-contract.pl +++ b/corpus/support/fix-contract.pl @@ -1,4 +1,6 @@ #!/usr/bin/perl -w +$|++; + use strict; while(<>) { #s/ (pre|anti|re|pro|inter|intra|multi|e|x|neo) - / $1- /ig; diff --git a/corpus/support/quote-norm.pl b/corpus/support/quote-norm.pl index b104e73c..57f4ad77 100755 --- a/corpus/support/quote-norm.pl +++ b/corpus/support/quote-norm.pl @@ -1,4 +1,5 @@ #!/usr/bin/perl -w +$|++; use strict; use utf8; binmode(STDIN,"utf8"); diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl index acc537fb..e0df16a7 100755 --- a/corpus/support/tokenizer.pl +++ b/corpus/support/tokenizer.pl @@ -1,4 +1,5 @@ #!/usr/bin/env perl +$|++; my $script_dir; BEGIN {$^W = 1; use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, $script_dir; } diff --git a/corpus/support/utf8-normalize.sh b/corpus/support/utf8-normalize.sh index 2f347854..c85ae9f7 100755 --- a/corpus/support/utf8-normalize.sh +++ b/corpus/support/utf8-normalize.sh @@ -25,7 +25,8 @@ else fi fi -perl -e 'while(<>){s/\r\n*/\n/g; print;}' | $CMD | /usr/bin/perl -w -e ' +perl -e '$|++; while(<>){s/\r\n*/\n/g; print;}' | $CMD | /usr/bin/perl -w -e ' + $|++; while (<>) { chomp; s/[\x00-\x1F]+/ /g; diff --git a/corpus/tokenize-anything.sh b/corpus/tokenize-anything.sh index 028992cf..52739e81 100755 --- a/corpus/tokenize-anything.sh +++ b/corpus/tokenize-anything.sh @@ -3,11 +3,17 @@ ROOTDIR=`dirname $0` SUPPORT=$ROOTDIR/support -$SUPPORT/utf8-normalize.sh | +if [[ $# == 1 && $1 == '-u' ]] ; then + NORMCMD=cat +else + NORMCMD=$SUPPORT/utf8-normalize.sh +fi + +$NORMCMD | $SUPPORT/quote-norm.pl | $SUPPORT/tokenizer.pl | - sed -e 's/ al - / al-/g' | + sed -u -e 's/ al - / al-/g' | $SUPPORT/fix-contract.pl | - sed -e 's/^ //' | sed -e 's/ $//' | - perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/; print;}' + sed -u -e 's/^ //' | sed -u -e 's/ $//' | + perl -e '$|++; while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/; print;}' |