summaryrefslogtreecommitdiff
path: root/corpus
diff options
context:
space:
mode:
authormjdenkowski <michael.j.denkowski@gmail.com>2014-03-11 15:47:04 -0400
committermjdenkowski <michael.j.denkowski@gmail.com>2014-03-11 15:47:04 -0400
commitefbc43b40c8c3204245814b65a7be280498281bd (patch)
treef2b58b5f4208bf535522a788cfd241b59f7026ac /corpus
parent1197fb64e67b95ed497df4ebca5dd69e3e2db1b5 (diff)
parent284383880f043edb2d67afbe2f64237c466245c1 (diff)
Merge branch 'master' of github.com:redpony/cdec
Diffstat (limited to 'corpus')
-rwxr-xr-xcorpus/support/quote-norm.pl1
-rw-r--r--corpus/support/token_list6
2 files changed, 5 insertions, 2 deletions
diff --git a/corpus/support/quote-norm.pl b/corpus/support/quote-norm.pl
index 33604027..0366fad5 100755
--- a/corpus/support/quote-norm.pl
+++ b/corpus/support/quote-norm.pl
@@ -39,6 +39,7 @@ while(<STDIN>) {
s/&\#([0-9]+);/pack("U", $1)/ge;
# Regularlize spaces:
+ s/\x{ad}//g; # soft hyphen
s/\x{a0}/ /g; # non-breaking space
s/\x{2009}/ /g; # thin space
s/\x{2028}/ /g; # "line separator"
diff --git a/corpus/support/token_list b/corpus/support/token_list
index 228663f6..d38638cf 100644
--- a/corpus/support/token_list
+++ b/corpus/support/token_list
@@ -156,8 +156,9 @@ Mass.
Md.
Mfg.
Mgr.
-Mexican-U.S.
-Mich.
+Mio.
+Mrd.
+Bio.
Minn.
Mo.
Mon.
@@ -187,6 +188,7 @@ Rd.
Rev.
R.J.
C.L
+Rs.
Rte.
Sat.
W.T