summaryrefslogtreecommitdiff
path: root/corpus
diff options
context:
space:
mode:
authorChris Dyer <cdyer@Chriss-MacBook-Air.local>2013-02-27 20:14:18 -0500
committerChris Dyer <cdyer@Chriss-MacBook-Air.local>2013-02-27 20:14:18 -0500
commit99538847039c06bdcc288e2c5dfcdb507ff879ca (patch)
treecea445ddee00df985b0331cf8e0e5230de06426f /corpus
parenteba1853a8f537bec4fa98309cc24021c684907cd (diff)
quick fix
Diffstat (limited to 'corpus')
-rwxr-xr-xcorpus/tokenize-anything.sh2
1 files changed, 1 insertions, 1 deletions
diff --git a/corpus/tokenize-anything.sh b/corpus/tokenize-anything.sh
index 1a24193d..028992cf 100755
--- a/corpus/tokenize-anything.sh
+++ b/corpus/tokenize-anything.sh
@@ -9,5 +9,5 @@ $SUPPORT/utf8-normalize.sh |
sed -e 's/ al - / al-/g' |
$SUPPORT/fix-contract.pl |
sed -e 's/^ //' | sed -e 's/ $//' |
- perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./;print;}'
+ perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/; print;}'