summaryrefslogtreecommitdiff
path: root/corpus/support
diff options
context:
space:
mode:
Diffstat (limited to 'corpus/support')
-rwxr-xr-xcorpus/support/tokenizer.pl8
1 files changed, 4 insertions, 4 deletions
diff --git a/corpus/support/tokenizer.pl b/corpus/support/tokenizer.pl
index 0350a894..acc537fb 100755
--- a/corpus/support/tokenizer.pl
+++ b/corpus/support/tokenizer.pl
@@ -226,7 +226,7 @@ sub proc_token {
}
## step 1: check the most common case
- if($token =~ /^[a-z0-9\p{Cyrillic}\p{Greek}\p{Hebrew}\p{Han}\p{Arabic}]+$/i){
+ if($token =~ /^[a-z0-9\p{Cyrillic}\p{Greek}\p{Hebrew}\p{Han}\p{Arabic}\p{Devanagari}]+$/i){
### most common cases
return $token;
}
@@ -246,7 +246,7 @@ sub proc_token {
## number
return $token;
}
- if($token =~ /^(@|#)[A-Za-z0-9_\p{Cyrillic}\p{Greek}\p{Hebrew}\p{Han}\p{Arabic}]+.*$/){
+ if($token =~ /^(@|#)[A-Za-z0-9_\p{Cyrillic}\p{Greek}\p{Hebrew}\p{Han}\p{Arabic}\p{Devanagari}]+.*$/){
## twitter hashtag or address
return proc_rightpunc($token);
}
@@ -277,7 +277,7 @@ sub proc_token {
}
#my $t1 = '[\x{0600}-\x{06ff}a-z\d\_\.\-]';
- my $t1 = '[a-z\d\_\-\.\p{Cyrillic}\p{Greek}\p{Hebrew}\p{Han}\p{Arabic}]';
+ my $t1 = '[a-z\d\_\-\.\p{Cyrillic}\p{Greek}\p{Hebrew}\p{Han}\p{Arabic}\p{Devanagari}]';
if($token =~ /^\/(($t1)+\/)+($t1)+\/?$/i){
### /nls/p/....
return $token;
@@ -361,7 +361,7 @@ sub deep_proc_token {
}
##### step 0: if it mades up of all puncts, remove one punct at a time.
- if($line !~ /[\p{Cyrillic}\p{Greek}\p{Hebrew}\p{Han}\p{Arabic}a-zA-Z\d]/){
+ if($line !~ /[\p{Cyrillic}\p{Greek}\p{Hebrew}\p{Han}\p{Arabic}\p{Devanagari}a-zA-Z\d]/){
if($line =~ /^(\!+|\@+|\++|\=+|\*+|\<+|\>+|\|+|\?+|\.+|\-+|\_+|\&+)$/){
## ++ @@@@ !!! ....
return $line;