From 7928695272b000de7142b91e05959a8fab6b1d2a Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@allegro.clab.cs.cmu.edu>
Date: Wed, 14 Nov 2012 20:33:51 -0500
Subject: major mert clean up, stuff for simple system demo

---
 corpus/tokenize-anything.sh | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100755 corpus/tokenize-anything.sh

(limited to 'corpus/tokenize-anything.sh')

diff --git a/corpus/tokenize-anything.sh b/corpus/tokenize-anything.sh
new file mode 100755
index 00000000..1a24193d
--- /dev/null
+++ b/corpus/tokenize-anything.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+ROOTDIR=`dirname $0`
+SUPPORT=$ROOTDIR/support
+
+$SUPPORT/utf8-normalize.sh |
+  $SUPPORT/quote-norm.pl |
+  $SUPPORT/tokenizer.pl |
+  sed -e 's/ al - / al-/g' |
+  $SUPPORT/fix-contract.pl |
+  sed -e 's/^ //' | sed -e 's/ $//' |
+  perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./;print;}'
+
-- 
cgit v1.2.3


From 99538847039c06bdcc288e2c5dfcdb507ff879ca Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@Chriss-MacBook-Air.local>
Date: Wed, 27 Feb 2013 20:14:18 -0500
Subject: quick fix

---
 corpus/tokenize-anything.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'corpus/tokenize-anything.sh')

diff --git a/corpus/tokenize-anything.sh b/corpus/tokenize-anything.sh
index 1a24193d..028992cf 100755
--- a/corpus/tokenize-anything.sh
+++ b/corpus/tokenize-anything.sh
@@ -9,5 +9,5 @@ $SUPPORT/utf8-normalize.sh |
   sed -e 's/ al - / al-/g' |
   $SUPPORT/fix-contract.pl |
   sed -e 's/^ //' | sed -e 's/ $//' |
-  perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./;print;}'
+  perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/;  print;}'
 
-- 
cgit v1.2.3