Merge remote-tracking branch 'upstream/master'

4 files changed, 7 insertions, 1 deletions

diff --git a/corpus/paste-files.pl b/corpus/paste-files.pl
index 4cb424ad..ef2cd937 100755
--- a/corpus/paste-files.pl
+++ b/corpus/paste-files.pl
@@ -34,6 +34,7 @@ while(1) {
       $done = 1;
       last;
     }
+    $r =~ s/\r//g;
     chomp $r;
     if ($r =~ /\|\|\|/) {
       $r = '';
diff --git a/corpus/support/quote-norm.pl b/corpus/support/quote-norm.pl
index e4e5055e..b104e73c 100755
--- a/corpus/support/quote-norm.pl
+++ b/corpus/support/quote-norm.pl
@@ -11,6 +11,8 @@ while(<STDIN>) {
   s/&\s*squot\s*;/'/gi;
   s/&\s*quot\s*;/"/gi;
   s/&\s*amp\s*;/&/gi;
+  s/&\s*nbsp\s*;/&/gi;
+  s/&\s*#\s*160\s*;/ /gi;
   s/ (\d\d): (\d\d)/ $1:$2/g;
   s/[\x{20a0}]\x{20ac}]/ EUR /g;
   s/[\x{00A3}]/ GBP /g;
@@ -20,6 +22,7 @@ while(<STDIN>) {
   s/&\s*#45\s*;/--/g;
   s/�c/--/g;
   s/ ,,/ "/g;
+  s/„/"/g;
   s/``/"/g;
   s/''/"/g;
   s/[「」]/"/g;
diff --git a/corpus/support/token_list b/corpus/support/token_list
index d470cb22..366cd7ff 100644
--- a/corpus/support/token_list
+++ b/corpus/support/token_list
@@ -37,6 +37,8 @@ tel.
 10.
 
 ##################### abbreviation: words that contain period.
+EE.UU.
+ee.uu.
 U.A.E
 Ala.
 Ph.D.
diff --git a/corpus/tokenize-anything.sh b/corpus/tokenize-anything.sh
index 1a24193d..028992cf 100755
--- a/corpus/tokenize-anything.sh
+++ b/corpus/tokenize-anything.sh
@@ -9,5 +9,5 @@ $SUPPORT/utf8-normalize.sh |
   sed -e 's/ al - / al-/g' |
   $SUPPORT/fix-contract.pl |
   sed -e 's/^ //' | sed -e 's/ $//' |
-  perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./;print;}'
+  perl -e 'while(<>){s/(\d+)(\.+)$/$1 ./; s/(\d+)(\.+) \|\|\|/$1 . |||/;  print;}'