diff options
Diffstat (limited to 'corpus/support/tokenizer.pl')
| -rwxr-xr-x | corpus/support/tokenizer.pl | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl index f57bc87a..aa285be4 100755 --- a/corpus/support/tokenizer.pl +++ b/corpus/support/tokenizer.pl @@ -386,7 +386,7 @@ sub deep_proc_token { } ##### step 1: separate by punct T2 on the boundary - my $t2 = '\`|\!|\@|\+|\=|\[|\]|\<|\>|\||\(|\)|\{|\}|\?|\"|;'; + my $t2 = '\`|\!|\@|\+|\=|\[|\]|\<|\>|\||\(|\)|\{|\}|\?|\"|;|●|○'; if($line =~ s/^(($t2)+)/$1 /){ return proc_line($line); } |