summaryrefslogtreecommitdiff
path: root/corpus/support
diff options
context:
space:
mode:
Diffstat (limited to 'corpus/support')
-rwxr-xr-xcorpus/support/quote-norm.pl3
-rw-r--r--corpus/support/token_list2
2 files changed, 5 insertions, 0 deletions
diff --git a/corpus/support/quote-norm.pl b/corpus/support/quote-norm.pl
index e4e5055e..b104e73c 100755
--- a/corpus/support/quote-norm.pl
+++ b/corpus/support/quote-norm.pl
@@ -11,6 +11,8 @@ while(<STDIN>) {
s/&\s*squot\s*;/'/gi;
s/&\s*quot\s*;/"/gi;
s/&\s*amp\s*;/&/gi;
+ s/&\s*nbsp\s*;/&/gi;
+ s/&\s*#\s*160\s*;/ /gi;
s/ (\d\d): (\d\d)/ $1:$2/g;
s/[\x{20a0}]\x{20ac}]/ EUR /g;
s/[\x{00A3}]/ GBP /g;
@@ -20,6 +22,7 @@ while(<STDIN>) {
s/&\s*#45\s*;/--/g;
s/�c/--/g;
s/ ,,/ "/g;
+ s/„/"/g;
s/``/"/g;
s/''/"/g;
s/[「」]/"/g;
diff --git a/corpus/support/token_list b/corpus/support/token_list
index d470cb22..366cd7ff 100644
--- a/corpus/support/token_list
+++ b/corpus/support/token_list
@@ -37,6 +37,8 @@ tel.
10.
##################### abbreviation: words that contain period.
+EE.UU.
+ee.uu.
U.A.E
Ala.
Ph.D.