summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChris Dyer <cdyer@allegro.clab.cs.cmu.edu>2013-01-21 17:28:19 -0500
committerChris Dyer <cdyer@allegro.clab.cs.cmu.edu>2013-01-21 17:28:19 -0500
commitee681f3d0337dc3d93c25de373a50fbda69252fe (patch)
tree66a89cdc31969aa36440bb7061fbe39f2ffdd4f6
parente1f71a6ce868d116f04082b697a8d61afcd625f1 (diff)
tokenizer support for utf8 patterns
-rw-r--r--corpus/support/token_patterns1
-rwxr-xr-xcorpus/support/tokenizer.pl2
2 files changed, 3 insertions, 0 deletions
diff --git a/corpus/support/token_patterns b/corpus/support/token_patterns
index 8e69432b..b25ac6de 100644
--- a/corpus/support/token_patterns
+++ b/corpus/support/token_patterns
@@ -1,3 +1,4 @@
/^(al|el|ul|e)\-[a-z]+$/
+/^((а|А)(ль|ш)|уль)-\p{Cyrillic}+$/
/^(\d|\d\d|\d\d\d)\.$/
diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl
index b5190858..0350a894 100755
--- a/corpus/support/tokenizer.pl
+++ b/corpus/support/tokenizer.pl
@@ -73,6 +73,7 @@ my $dict_file = "$workdir/token_list";
my $word_patt_file = "$workdir/token_patterns";
open(my $dict_fp, "$dict_file") or die;
+binmode($dict_fp, ":utf8");
# read in the list of words that should not be segmented,
## e.g.,"I.B.M.", co-operation.
@@ -89,6 +90,7 @@ while(<$dict_fp>){
}
open(my $patt_fp, "$word_patt_file") or die;
+binmode($patt_fp, ":utf8");
my @word_patts = ();
my $word_patt_num = 0;
while(<$patt_fp>){