summaryrefslogtreecommitdiff
path: root/corpus
diff options
context:
space:
mode:
authorChris Dyer <cdyer@allegro.clab.cs.cmu.edu>2013-01-21 17:28:19 -0500
committerChris Dyer <cdyer@allegro.clab.cs.cmu.edu>2013-01-21 17:28:19 -0500
commitfba0f3db9aa896807e3932f9d3323767cedd9338 (patch)
tree8ca7a0d7bf3402c2c4041db3e75041afde788946 /corpus
parent55a1914e8998b2dc613d0f1e452a714b51169953 (diff)
tokenizer support for utf8 patterns
Diffstat (limited to 'corpus')
-rw-r--r--corpus/support/token_patterns1
-rwxr-xr-xcorpus/support/tokenizer.pl2
2 files changed, 3 insertions, 0 deletions
diff --git a/corpus/support/token_patterns b/corpus/support/token_patterns
index 8e69432b..b25ac6de 100644
--- a/corpus/support/token_patterns
+++ b/corpus/support/token_patterns
@@ -1,3 +1,4 @@
/^(al|el|ul|e)\-[a-z]+$/
+/^((а|А)(ль|ш)|уль)-\p{Cyrillic}+$/
/^(\d|\d\d|\d\d\d)\.$/
diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl
index b5190858..0350a894 100755
--- a/corpus/support/tokenizer.pl
+++ b/corpus/support/tokenizer.pl
@@ -73,6 +73,7 @@ my $dict_file = "$workdir/token_list";
my $word_patt_file = "$workdir/token_patterns";
open(my $dict_fp, "$dict_file") or die;
+binmode($dict_fp, ":utf8");
# read in the list of words that should not be segmented,
## e.g.,"I.B.M.", co-operation.
@@ -89,6 +90,7 @@ while(<$dict_fp>){
}
open(my $patt_fp, "$word_patt_file") or die;
+binmode($patt_fp, ":utf8");
my @word_patts = ();
my $word_patt_num = 0;
while(<$patt_fp>){