From 8646b68e5b124f612fd65b51ea40624f65a2f3d6 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 29 Dec 2014 01:59:41 -0500 Subject: finnish abbrevs --- corpus/support/token_list | 48 +++++++++++++++++++++++++++++++++++++++++++ corpus/support/token_patterns | 1 + 2 files changed, 49 insertions(+) (limited to 'corpus/support') diff --git a/corpus/support/token_list b/corpus/support/token_list index d38638cf..69dbfdc2 100644 --- a/corpus/support/token_list +++ b/corpus/support/token_list @@ -1,6 +1,54 @@ ##################### hyphenated words added by Fei since 3/7/05 ##X-ray +# Finnish +eaa. +ap. +arv. +ay. +eKr. +em. +engl. +esim. +fil. +lis. +fil. +maist. +fil.toht. +harv. +ilt. +jatk. +jKr. +jms. +jne. +joht. +klo +ko. +ks. +leht. +lv. +lyh. +mm. +mon. +nim. +nro. +ns. +nti. +os. +oy. +pj. +pnä. +puh. +pvm. +rva. +tms. +ts. +vars. +vrt. +ym. +yms. +yo. + # hindi abbreviation patterns जन. फर. diff --git a/corpus/support/token_patterns b/corpus/support/token_patterns index de64fb2a..e51e5e72 100644 --- a/corpus/support/token_patterns +++ b/corpus/support/token_patterns @@ -1,5 +1,6 @@ /^(al|el|ul|e)\-[a-z]+$/ /^((а|А)(ль|ш)|уль)-\p{Cyrillic}+$/ +/^(юл)-\p{Cyrillic}+$/ /^\p{Cyrillic}\.\p{Cyrillic}\.$/ /^(\d|\d\d|\d\d\d)\.$/ -- cgit v1.2.3 From 5bc712f5bfa3ec7924a52ded4a7d6b3bab9539f0 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 29 Dec 2014 02:00:56 -0500 Subject: foo --- corpus/support/token_list | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'corpus/support') diff --git a/corpus/support/token_list b/corpus/support/token_list index d38638cf..c857b363 100644 --- a/corpus/support/token_list +++ b/corpus/support/token_list @@ -1,6 +1,10 @@ ##################### hyphenated words added by Fei since 3/7/05 ##X-ray +# Finnish abbreviation patterns +eaa. +v. + # hindi abbreviation patterns जन. फर. -- cgit v1.2.3 From a66b19342f105546adfb9cd9189f862e8a9c0b15 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 29 Dec 2014 02:02:27 -0500 Subject: finnish case markings --- corpus/support/token_patterns | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'corpus/support') diff --git a/corpus/support/token_patterns b/corpus/support/token_patterns index e51e5e72..12558cdd 100644 --- a/corpus/support/token_patterns +++ b/corpus/support/token_patterns @@ -1,6 +1,7 @@ /^(al|el|ul|e)\-[a-z]+$/ +/\.(fi|fr|es|co\.uk|de)$/ +/:[a-zä]+$/ /^((а|А)(ль|ш)|уль)-\p{Cyrillic}+$/ -/^(юл)-\p{Cyrillic}+$/ /^\p{Cyrillic}\.\p{Cyrillic}\.$/ /^(\d|\d\d|\d\d\d)\.$/ -- cgit v1.2.3 From 992556cc0931b255b9e299c0f489c3b449b22ab4 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Mon, 29 Dec 2014 19:20:49 -0500 Subject: deal with eur symbol --- corpus/support/tokenizer.pl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'corpus/support') diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl index aa285be4..718d78cc 100755 --- a/corpus/support/tokenizer.pl +++ b/corpus/support/tokenizer.pl @@ -415,7 +415,7 @@ sub deep_proc_token { } ## remove the ending periods that follow number etc. - if($line =~ /^(.*(\d|\~|\^|\&|\:|\,|\#|\*|\%|\-|\_|\/|\\|\$|\'))(\.+)$/){ + if($line =~ /^(.*(\d|\~|\^|\&|\:|\,|\#|\*|\%|€|\-|\_|\/|\\|\$|\'))(\.+)$/){ ## 12~13. => 12~13 . my $t1 = $1; my $t3 = $3; @@ -600,12 +600,12 @@ sub deep_proc_token { ## deal with "%" - if(($line =~ /\%/) && ($Split_On_PercentSign > 0)){ + if(($line =~ /\%|€/) && ($Split_On_PercentSign > 0)){ my $suc = 0; if($Split_On_PercentSign >= 2){ - $suc += ($line =~ s/(\D)(\%+)/$1 $2/g); + $suc += ($line =~ s/(\D)(\%+|€+)/$1 $2/g); }else{ - $suc += ($line =~ s/(\%+)/ $1 /g); + $suc += ($line =~ s/(\%+|€+)/ $1 /g); } if($suc){ -- cgit v1.2.3