diff options
| author | Patrick Simianer <p@simianer.de> | 2013-03-15 11:39:18 +0100 |
|---|---|---|
| committer | Patrick Simianer <p@simianer.de> | 2013-03-15 11:39:18 +0100 |
| commit | 3d8aa307d233f58cfe9ddcc616a8297551a738e3 (patch) | |
| tree | b9556fe62c7ac0d7b48d92481acdd06a546b1dc2 /corpus/support | |
| parent | cf67d34738e1487f75739dc1e027b1864a06513b (diff) | |
| parent | 9f1d72fa4dc231eb8cdb737becfc10452b5daef4 (diff) | |
Merge remote-tracking branch 'upstream/master'
Diffstat (limited to 'corpus/support')
| -rwxr-xr-x | corpus/support/quote-norm.pl | 3 | ||||
| -rw-r--r-- | corpus/support/token_list | 2 |
2 files changed, 5 insertions, 0 deletions
diff --git a/corpus/support/quote-norm.pl b/corpus/support/quote-norm.pl index e4e5055e..b104e73c 100755 --- a/corpus/support/quote-norm.pl +++ b/corpus/support/quote-norm.pl @@ -11,6 +11,8 @@ while(<STDIN>) { s/&\s*squot\s*;/'/gi; s/&\s*quot\s*;/"/gi; s/&\s*amp\s*;/&/gi; + s/&\s*nbsp\s*;/&/gi; + s/&\s*#\s*160\s*;/ /gi; s/ (\d\d): (\d\d)/ $1:$2/g; s/[\x{20a0}]\x{20ac}]/ EUR /g; s/[\x{00A3}]/ GBP /g; @@ -20,6 +22,7 @@ while(<STDIN>) { s/&\s*#45\s*;/--/g; s/�c/--/g; s/ ,,/ "/g; + s/„/"/g; s/``/"/g; s/''/"/g; s/[「」]/"/g; diff --git a/corpus/support/token_list b/corpus/support/token_list index d470cb22..366cd7ff 100644 --- a/corpus/support/token_list +++ b/corpus/support/token_list @@ -37,6 +37,8 @@ tel. 10. ##################### abbreviation: words that contain period. +EE.UU. +ee.uu. U.A.E Ala. Ph.D. |