summaryrefslogtreecommitdiff
path: root/corpus/tokenize-anything.sh
diff options
context:
space:
mode:
Diffstat (limited to 'corpus/tokenize-anything.sh')
-rwxr-xr-xcorpus/tokenize-anything.sh13
1 files changed, 13 insertions, 0 deletions
diff --git a/corpus/tokenize-anything.sh b/corpus/tokenize-anything.sh
new file mode 100755
index 00000000..028992cf
--- /dev/null
+++ b/corpus/tokenize-anything.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+ROOTDIR=`dirname $0`
+SUPPORT=$ROOTDIR/support
+
+$SUPPORT/utf8-normalize.sh |
+ $SUPPORT/quote-norm.pl |
+ $SUPPORT/tokenizer.pl |
+ sed -e 's/ al - / al-/g' |
+ $SUPPORT/fix-contract.pl |
+ sed -e 's/^ //' | sed -e 's/ $//' |
+ perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/; print;}'
+