summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--corpus/support/token_list48
-rw-r--r--corpus/support/token_patterns1
2 files changed, 49 insertions, 0 deletions
diff --git a/corpus/support/token_list b/corpus/support/token_list
index d38638cf..69dbfdc2 100644
--- a/corpus/support/token_list
+++ b/corpus/support/token_list
@@ -1,6 +1,54 @@
##################### hyphenated words added by Fei since 3/7/05
##X-ray
+# Finnish
+eaa.
+ap.
+arv.
+ay.
+eKr.
+em.
+engl.
+esim.
+fil.
+lis.
+fil.
+maist.
+fil.toht.
+harv.
+ilt.
+jatk.
+jKr.
+jms.
+jne.
+joht.
+klo
+ko.
+ks.
+leht.
+lv.
+lyh.
+mm.
+mon.
+nim.
+nro.
+ns.
+nti.
+os.
+oy.
+pj.
+pnä.
+puh.
+pvm.
+rva.
+tms.
+ts.
+vars.
+vrt.
+ym.
+yms.
+yo.
+
# hindi abbreviation patterns
जन.
फर.
diff --git a/corpus/support/token_patterns b/corpus/support/token_patterns
index de64fb2a..e51e5e72 100644
--- a/corpus/support/token_patterns
+++ b/corpus/support/token_patterns
@@ -1,5 +1,6 @@
/^(al|el|ul|e)\-[a-z]+$/
/^((а|А)(ль|ш)|уль)-\p{Cyrillic}+$/
+/^(юл)-\p{Cyrillic}+$/
/^\p{Cyrillic}\.\p{Cyrillic}\.$/
/^(\d|\d\d|\d\d\d)\.$/