From 754e9fd68b617b9dfee89461d51a5dcb98b18fbe Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@allegro.clab.cs.cmu.edu>
Date: Sat, 23 Feb 2013 04:23:48 -0500
Subject: one missing quote type

---
 corpus/support/quote-norm.pl | 1 +
 1 file changed, 1 insertion(+)

(limited to 'corpus')

diff --git a/corpus/support/quote-norm.pl b/corpus/support/quote-norm.pl
index e4e5055e..d2980092 100755
--- a/corpus/support/quote-norm.pl
+++ b/corpus/support/quote-norm.pl
@@ -20,6 +20,7 @@ while(<STDIN>) {
   s/&\s*#45\s*;/--/g;
   s/�c/--/g;
   s/ ,,/ "/g;
+  s/„/"/g;
   s/``/"/g;
   s/''/"/g;
   s/[「」]/"/g;
-- 
cgit v1.2.3


From 7973046992b710a170a84888820ce76a9e4f2346 Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@Chriss-MacBook-Air.local>
Date: Wed, 27 Feb 2013 20:14:18 -0500
Subject: quick fix

---
 corpus/tokenize-anything.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'corpus')

diff --git a/corpus/tokenize-anything.sh b/corpus/tokenize-anything.sh
index 1a24193d..028992cf 100755
--- a/corpus/tokenize-anything.sh
+++ b/corpus/tokenize-anything.sh
@@ -9,5 +9,5 @@ $SUPPORT/utf8-normalize.sh |
   sed -e 's/ al - / al-/g' |
   $SUPPORT/fix-contract.pl |
   sed -e 's/^ //' | sed -e 's/ $//' |
-  perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./;print;}'
+  perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/;  print;}'
 
-- 
cgit v1.2.3


From e419e52189ce8472bc5f5517796d00f0887affa7 Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@allegro.clab.cs.cmu.edu>
Date: Fri, 8 Mar 2013 22:44:49 -0500
Subject: few preproc fixes

---
 corpus/paste-files.pl        | 1 +
 corpus/support/quote-norm.pl | 2 ++
 corpus/support/token_list    | 2 ++
 3 files changed, 5 insertions(+)

(limited to 'corpus')

diff --git a/corpus/paste-files.pl b/corpus/paste-files.pl
index 4cb424ad..ef2cd937 100755
--- a/corpus/paste-files.pl
+++ b/corpus/paste-files.pl
@@ -34,6 +34,7 @@ while(1) {
       $done = 1;
       last;
     }
+    $r =~ s/\r//g;
     chomp $r;
     if ($r =~ /\|\|\|/) {
       $r = '';
diff --git a/corpus/support/quote-norm.pl b/corpus/support/quote-norm.pl
index d2980092..b104e73c 100755
--- a/corpus/support/quote-norm.pl
+++ b/corpus/support/quote-norm.pl
@@ -11,6 +11,8 @@ while(<STDIN>) {
   s/&\s*squot\s*;/'/gi;
   s/&\s*quot\s*;/"/gi;
   s/&\s*amp\s*;/&/gi;
+  s/&\s*nbsp\s*;/&/gi;
+  s/&\s*#\s*160\s*;/ /gi;
   s/ (\d\d): (\d\d)/ $1:$2/g;
   s/[\x{20a0}]\x{20ac}]/ EUR /g;
   s/[\x{00A3}]/ GBP /g;
diff --git a/corpus/support/token_list b/corpus/support/token_list
index d470cb22..366cd7ff 100644
--- a/corpus/support/token_list
+++ b/corpus/support/token_list
@@ -37,6 +37,8 @@ tel.
 10.
 
 ##################### abbreviation: words that contain period.
+EE.UU.
+ee.uu.
 U.A.E
 Ala.
 Ph.D.
-- 
cgit v1.2.3