From f8d9ff4aaeb1d1f773bacfe9ee75d1d1778ec26b Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@allegro.clab.cs.cmu.edu>
Date: Wed, 14 Nov 2012 20:33:51 -0500
Subject: major mert clean up, stuff for simple system demo

---
 corpus/tokenize-anything.sh | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100755 corpus/tokenize-anything.sh

(limited to 'corpus/tokenize-anything.sh')

diff --git a/corpus/tokenize-anything.sh b/corpus/tokenize-anything.sh
new file mode 100755
index 00000000..1a24193d
--- /dev/null
+++ b/corpus/tokenize-anything.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+ROOTDIR=`dirname $0`
+SUPPORT=$ROOTDIR/support
+
+$SUPPORT/utf8-normalize.sh |
+  $SUPPORT/quote-norm.pl |
+  $SUPPORT/tokenizer.pl |
+  sed -e 's/ al - / al-/g' |
+  $SUPPORT/fix-contract.pl |
+  sed -e 's/^ //' | sed -e 's/ $//' |
+  perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./;print;}'
+
-- 
cgit v1.2.3