summaryrefslogtreecommitdiff
path: root/corpus/support
diff options
context:
space:
mode:
authorChris Dyer <cdyer@allegro.clab.cs.cmu.edu>2013-03-08 22:44:49 -0500
committerChris Dyer <cdyer@allegro.clab.cs.cmu.edu>2013-03-08 22:44:49 -0500
commite419e52189ce8472bc5f5517796d00f0887affa7 (patch)
tree807736bc422f12a6890fa6f953114b2576f5f3dc /corpus/support
parent754e9fd68b617b9dfee89461d51a5dcb98b18fbe (diff)
few preproc fixes
Diffstat (limited to 'corpus/support')
-rwxr-xr-xcorpus/support/quote-norm.pl2
-rw-r--r--corpus/support/token_list2
2 files changed, 4 insertions, 0 deletions
diff --git a/corpus/support/quote-norm.pl b/corpus/support/quote-norm.pl
index d2980092..b104e73c 100755
--- a/corpus/support/quote-norm.pl
+++ b/corpus/support/quote-norm.pl
@@ -11,6 +11,8 @@ while(<STDIN>) {
s/&\s*squot\s*;/'/gi;
s/&\s*quot\s*;/"/gi;
s/&\s*amp\s*;/&/gi;
+ s/&\s*nbsp\s*;/&/gi;
+ s/&\s*#\s*160\s*;/ /gi;
s/ (\d\d): (\d\d)/ $1:$2/g;
s/[\x{20a0}]\x{20ac}]/ EUR /g;
s/[\x{00A3}]/ GBP /g;
diff --git a/corpus/support/token_list b/corpus/support/token_list
index d470cb22..366cd7ff 100644
--- a/corpus/support/token_list
+++ b/corpus/support/token_list
@@ -37,6 +37,8 @@ tel.
10.
##################### abbreviation: words that contain period.
+EE.UU.
+ee.uu.
U.A.E
Ala.
Ph.D.