diff options
| author | Patrick Simianer <p@simianer.de> | 2014-10-13 19:03:48 +0100 |
|---|---|---|
| committer | Patrick Simianer <p@simianer.de> | 2014-10-13 19:03:48 +0100 |
| commit | cb9fb7088dde35881516c088db402abe747d49fa (patch) | |
| tree | a91e4935a7941f1b261f76d88ab41fa3078a1891 /corpus/support | |
| parent | 0a00e57e921c8eca8e02364db7d2e6607bfdcebc (diff) | |
| parent | b1ed81ef3216b212295afa76c5d20a56fb647204 (diff) | |
Merge remote-tracking branch 'upstream/master'
Diffstat (limited to 'corpus/support')
| -rwxr-xr-x | corpus/support/tokenizer.pl | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl index f57bc87a..aa285be4 100755 --- a/corpus/support/tokenizer.pl +++ b/corpus/support/tokenizer.pl @@ -386,7 +386,7 @@ sub deep_proc_token { } ##### step 1: separate by punct T2 on the boundary - my $t2 = '\`|\!|\@|\+|\=|\[|\]|\<|\>|\||\(|\)|\{|\}|\?|\"|;'; + my $t2 = '\`|\!|\@|\+|\=|\[|\]|\<|\>|\||\(|\)|\{|\}|\?|\"|;|●|○'; if($line =~ s/^(($t2)+)/$1 /){ return proc_line($line); } |