diff options
| author | Patrick Simianer <p@simianer.de> | 2014-06-12 13:56:42 +0200 |
|---|---|---|
| committer | Patrick Simianer <p@simianer.de> | 2014-06-12 13:56:42 +0200 |
| commit | 244971287003d079e46193b8a209c28955f90134 (patch) | |
| tree | 8beaae6b12b913acb213fc7f2415fd63886192f9 /corpus/support/tokenizer.pl | |
| parent | 5250fd67a4b8f242068cff87f0a6a4211f8b0fcf (diff) | |
| parent | b66e838ed52decc0be1eb5817b2a77c3840db2c5 (diff) | |
Merge remote-tracking branch 'upstream/master'
Diffstat (limited to 'corpus/support/tokenizer.pl')
| -rwxr-xr-x | corpus/support/tokenizer.pl | 4 |
1 files changed, 4 insertions, 0 deletions
diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl index 7771201f..f57bc87a 100755 --- a/corpus/support/tokenizer.pl +++ b/corpus/support/tokenizer.pl @@ -240,6 +240,10 @@ sub proc_token { return $token; } + if($token =~ /^\d+(.\d+)+(亿|百万|万|千)?$/){ + return $token; + } + ## 1,234,345.34 if($token =~ /^\d+(\.\d{3})*,\d+$/){ ## number |