From fba0f3db9aa896807e3932f9d3323767cedd9338 Mon Sep 17 00:00:00 2001
From: Chris Dyer <cdyer@allegro.clab.cs.cmu.edu>
Date: Mon, 21 Jan 2013 17:28:19 -0500
Subject: tokenizer support for utf8 patterns

---
 corpus/support/token_patterns | 1 +
 1 file changed, 1 insertion(+)

(limited to 'corpus/support/token_patterns')

diff --git a/corpus/support/token_patterns b/corpus/support/token_patterns
index 8e69432b..b25ac6de 100644
--- a/corpus/support/token_patterns
+++ b/corpus/support/token_patterns
@@ -1,3 +1,4 @@
 /^(al|el|ul|e)\-[a-z]+$/
+/^((а|А)(ль|ш)|уль)-\p{Cyrillic}+$/
 /^(\d|\d\d|\d\d\d)\.$/
 
-- 
cgit v1.2.3