summaryrefslogtreecommitdiff
path: root/corpus/support/tokenizer.pl
diff options
context:
space:
mode:
Diffstat (limited to 'corpus/support/tokenizer.pl')
-rwxr-xr-xcorpus/support/tokenizer.pl8
1 files changed, 4 insertions, 4 deletions
diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl
index aa285be4..718d78cc 100755
--- a/corpus/support/tokenizer.pl
+++ b/corpus/support/tokenizer.pl
@@ -415,7 +415,7 @@ sub deep_proc_token {
}
## remove the ending periods that follow number etc.
- if($line =~ /^(.*(\d|\~|\^|\&|\:|\,|\#|\*|\%|\-|\_|\/|\\|\$|\'))(\.+)$/){
+ if($line =~ /^(.*(\d|\~|\^|\&|\:|\,|\#|\*|\%|€|\-|\_|\/|\\|\$|\'))(\.+)$/){
## 12~13. => 12~13 .
my $t1 = $1;
my $t3 = $3;
@@ -600,12 +600,12 @@ sub deep_proc_token {
## deal with "%"
- if(($line =~ /\%/) && ($Split_On_PercentSign > 0)){
+ if(($line =~ /\%|€/) && ($Split_On_PercentSign > 0)){
my $suc = 0;
if($Split_On_PercentSign >= 2){
- $suc += ($line =~ s/(\D)(\%+)/$1 $2/g);
+ $suc += ($line =~ s/(\D)(\%+|€+)/$1 $2/g);
}else{
- $suc += ($line =~ s/(\%+)/ $1 /g);
+ $suc += ($line =~ s/(\%+|€+)/ $1 /g);
}
if($suc){