summaryrefslogtreecommitdiff
path: root/corpus/support
diff options
context:
space:
mode:
authorChris Dyer <redpony@gmail.com>2014-12-29 02:01:38 -0500
committerChris Dyer <redpony@gmail.com>2014-12-29 02:01:38 -0500
commit4c4ab517897a217734e7a7e5942735712b561981 (patch)
tree8e11ab45ae3b374b7474a2eae17e15d71933d5ff /corpus/support
parent5bc712f5bfa3ec7924a52ded4a7d6b3bab9539f0 (diff)
parent8646b68e5b124f612fd65b51ea40624f65a2f3d6 (diff)
foo
Diffstat (limited to 'corpus/support')
-rw-r--r--corpus/support/token_list49
-rw-r--r--corpus/support/token_patterns1
2 files changed, 48 insertions, 2 deletions
diff --git a/corpus/support/token_list b/corpus/support/token_list
index c857b363..00daa82b 100644
--- a/corpus/support/token_list
+++ b/corpus/support/token_list
@@ -1,9 +1,54 @@
##################### hyphenated words added by Fei since 3/7/05
##X-ray
-# Finnish abbreviation patterns
+# Finnish
eaa.
-v.
+ap.
+arv.
+ay.
+eKr.
+em.
+engl.
+esim.
+fil.
+lis.
+fil.
+maist.
+fil.toht.
+harv.
+ilt.
+jatk.
+jKr.
+jms.
+jne.
+joht.
+klo
+ko.
+ks.
+leht.
+lv.
+lyh.
+mm.
+mon.
+nim.
+nro.
+ns.
+nti.
+os.
+oy.
+pj.
+pnä.
+puh.
+pvm.
+rva.
+tms.
+ts.
+vars.
+vrt.
+ym.
+yms.
+yo.
+>>>>>>> 8646b68e5b124f612fd65b51ea40624f65a2f3d6
# hindi abbreviation patterns
जन.
diff --git a/corpus/support/token_patterns b/corpus/support/token_patterns
index de64fb2a..e51e5e72 100644
--- a/corpus/support/token_patterns
+++ b/corpus/support/token_patterns
@@ -1,5 +1,6 @@
/^(al|el|ul|e)\-[a-z]+$/
/^((а|А)(ль|ш)|уль)-\p{Cyrillic}+$/
+/^(юл)-\p{Cyrillic}+$/
/^\p{Cyrillic}\.\p{Cyrillic}\.$/
/^(\d|\d\d|\d\d\d)\.$/