summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xcorpus/paste-files.pl1
-rwxr-xr-xcorpus/support/quote-norm.pl2
-rw-r--r--corpus/support/token_list2
3 files changed, 5 insertions, 0 deletions
diff --git a/corpus/paste-files.pl b/corpus/paste-files.pl
index 4cb424ad..ef2cd937 100755
--- a/corpus/paste-files.pl
+++ b/corpus/paste-files.pl
@@ -34,6 +34,7 @@ while(1) {
$done = 1;
last;
}
+ $r =~ s/\r//g;
chomp $r;
if ($r =~ /\|\|\|/) {
$r = '';
diff --git a/corpus/support/quote-norm.pl b/corpus/support/quote-norm.pl
index d2980092..b104e73c 100755
--- a/corpus/support/quote-norm.pl
+++ b/corpus/support/quote-norm.pl
@@ -11,6 +11,8 @@ while(<STDIN>) {
s/&\s*squot\s*;/'/gi;
s/&\s*quot\s*;/"/gi;
s/&\s*amp\s*;/&/gi;
+ s/&\s*nbsp\s*;/&/gi;
+ s/&\s*#\s*160\s*;/ /gi;
s/ (\d\d): (\d\d)/ $1:$2/g;
s/[\x{20a0}]\x{20ac}]/ EUR /g;
s/[\x{00A3}]/ GBP /g;
diff --git a/corpus/support/token_list b/corpus/support/token_list
index d470cb22..366cd7ff 100644
--- a/corpus/support/token_list
+++ b/corpus/support/token_list
@@ -37,6 +37,8 @@ tel.
10.
##################### abbreviation: words that contain period.
+EE.UU.
+ee.uu.
U.A.E
Ala.
Ph.D.