tokenizer support for utf8 patterns

1 files changed, 2 insertions, 0 deletions

diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl
index b5190858..0350a894 100755
--- a/corpus/support/tokenizer.pl
+++ b/corpus/support/tokenizer.pl
@@ -73,6 +73,7 @@ my $dict_file = "$workdir/token_list";
 my $word_patt_file = "$workdir/token_patterns";
 
 open(my $dict_fp, "$dict_file") or die;
+binmode($dict_fp, ":utf8");
 
 # read in the list of words that should not be segmented, 
 ##  e.g.,"I.B.M.", co-operation.
@@ -89,6 +90,7 @@ while(<$dict_fp>){
 }
 
 open(my $patt_fp, "$word_patt_file") or die;
+binmode($patt_fp, ":utf8");
 my @word_patts = ();
 my $word_patt_num = 0;
 while(<$patt_fp>){