deal with curly quotes

1 files changed, 3 insertions, 0 deletions

diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl
index 718d78cc..6cc9f4e1 100755
--- a/corpus/support/tokenizer.pl
+++ b/corpus/support/tokenizer.pl
@@ -388,15 +388,18 @@ sub deep_proc_token {
     ##### step 1: separate by punct T2 on the boundary
     my $t2 = '\`|\!|\@|\+|\=|\[|\]|\<|\>|\||\(|\)|\{|\}|\?|\"|;|●|○';
     if($line =~ s/^(($t2)+)/$1 /){
+        $line =~ s/"/“/;
 	return proc_line($line);
     }
 	
     if($line =~ s/(($t2)+)$/ $1/){
+        $line =~ s/"/”/;
 	return proc_line($line);
     }	
 
     ## step 2: separate by punct T2 in any position
     if($line =~ s/(($t2)+)/ $1 /g){
+        $line =~ s/"/”/g;  # probably before punctuation char
 	return proc_line($line);
     }