diff options
| author | Chris Dyer <cdyer@allegro.clab.cs.cmu.edu> | 2013-03-08 22:44:49 -0500 |
|---|---|---|
| committer | Chris Dyer <cdyer@allegro.clab.cs.cmu.edu> | 2013-03-08 22:44:49 -0500 |
| commit | 3a162d28033d1b9d5241e31f32978dba4eba6296 (patch) | |
| tree | ada0ec536f5a1835cc1507784ca3c8798636d273 | |
| parent | 9f0109076f1c95170cbe46e1708597bc6e0f9fd4 (diff) | |
few preproc fixes
| -rwxr-xr-x | corpus/paste-files.pl | 1 | ||||
| -rwxr-xr-x | corpus/support/quote-norm.pl | 2 | ||||
| -rw-r--r-- | corpus/support/token_list | 2 |
3 files changed, 5 insertions, 0 deletions
diff --git a/corpus/paste-files.pl b/corpus/paste-files.pl index 4cb424ad..ef2cd937 100755 --- a/corpus/paste-files.pl +++ b/corpus/paste-files.pl @@ -34,6 +34,7 @@ while(1) { $done = 1; last; } + $r =~ s/\r//g; chomp $r; if ($r =~ /\|\|\|/) { $r = ''; diff --git a/corpus/support/quote-norm.pl b/corpus/support/quote-norm.pl index d2980092..b104e73c 100755 --- a/corpus/support/quote-norm.pl +++ b/corpus/support/quote-norm.pl @@ -11,6 +11,8 @@ while(<STDIN>) { s/&\s*squot\s*;/'/gi; s/&\s*quot\s*;/"/gi; s/&\s*amp\s*;/&/gi; + s/&\s*nbsp\s*;/&/gi; + s/&\s*#\s*160\s*;/ /gi; s/ (\d\d): (\d\d)/ $1:$2/g; s/[\x{20a0}]\x{20ac}]/ EUR /g; s/[\x{00A3}]/ GBP /g; diff --git a/corpus/support/token_list b/corpus/support/token_list index d470cb22..366cd7ff 100644 --- a/corpus/support/token_list +++ b/corpus/support/token_list @@ -37,6 +37,8 @@ tel. 10. ##################### abbreviation: words that contain period. +EE.UU. +ee.uu. U.A.E Ala. Ph.D. |