summaryrefslogtreecommitdiff
path: root/corpus
diff options
context:
space:
mode:
Diffstat (limited to 'corpus')
-rwxr-xr-xcorpus/support/fix-contract.pl2
-rwxr-xr-xcorpus/support/quote-norm.pl1
-rwxr-xr-xcorpus/support/tokenizer.pl1
-rwxr-xr-xcorpus/support/utf8-normalize.sh3
-rwxr-xr-xcorpus/tokenize-anything.sh14
5 files changed, 16 insertions, 5 deletions
diff --git a/corpus/support/fix-contract.pl b/corpus/support/fix-contract.pl
index f1e191ab..49e88981 100755
--- a/corpus/support/fix-contract.pl
+++ b/corpus/support/fix-contract.pl
@@ -1,4 +1,6 @@
#!/usr/bin/perl -w
+$|++;
+
use strict;
while(<>) {
#s/ (pre|anti|re|pro|inter|intra|multi|e|x|neo) - / $1- /ig;
diff --git a/corpus/support/quote-norm.pl b/corpus/support/quote-norm.pl
index b104e73c..57f4ad77 100755
--- a/corpus/support/quote-norm.pl
+++ b/corpus/support/quote-norm.pl
@@ -1,4 +1,5 @@
#!/usr/bin/perl -w
+$|++;
use strict;
use utf8;
binmode(STDIN,"utf8");
diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl
index acc537fb..e0df16a7 100755
--- a/corpus/support/tokenizer.pl
+++ b/corpus/support/tokenizer.pl
@@ -1,4 +1,5 @@
#!/usr/bin/env perl
+$|++;
my $script_dir;
BEGIN {$^W = 1; use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, $script_dir; }
diff --git a/corpus/support/utf8-normalize.sh b/corpus/support/utf8-normalize.sh
index 2f347854..c85ae9f7 100755
--- a/corpus/support/utf8-normalize.sh
+++ b/corpus/support/utf8-normalize.sh
@@ -25,7 +25,8 @@ else
fi
fi
-perl -e 'while(<>){s/\r\n*/\n/g; print;}' | $CMD | /usr/bin/perl -w -e '
+perl -e '$|++; while(<>){s/\r\n*/\n/g; print;}' | $CMD | /usr/bin/perl -w -e '
+ $|++;
while (<>) {
chomp;
s/[\x00-\x1F]+/ /g;
diff --git a/corpus/tokenize-anything.sh b/corpus/tokenize-anything.sh
index 028992cf..52739e81 100755
--- a/corpus/tokenize-anything.sh
+++ b/corpus/tokenize-anything.sh
@@ -3,11 +3,17 @@
ROOTDIR=`dirname $0`
SUPPORT=$ROOTDIR/support
-$SUPPORT/utf8-normalize.sh |
+if [[ $# == 1 && $1 == '-u' ]] ; then
+ NORMCMD=cat
+else
+ NORMCMD=$SUPPORT/utf8-normalize.sh
+fi
+
+$NORMCMD |
$SUPPORT/quote-norm.pl |
$SUPPORT/tokenizer.pl |
- sed -e 's/ al - / al-/g' |
+ sed -u -e 's/ al - / al-/g' |
$SUPPORT/fix-contract.pl |
- sed -e 's/^ //' | sed -e 's/ $//' |
- perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/; print;}'
+ sed -u -e 's/^ //' | sed -u -e 's/ $//' |
+ perl -e '$|++; while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/; print;}'