blob: be3e97c0c94c458253995c1fa5b1c5be6be9b395 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
|
#!/usr/bin/perl -w
use strict;
use Getopt::Long "GetOptions";
my $LANGUAGE = shift @ARGV;
$LANGUAGE = 'target' unless ($LANGUAGE);
my $lno = 0;
while(my $line = <>) {
$lno++;
chomp $line;
my @fields = split / \|\|\| /, $line;
if ($LANGUAGE eq "source" or $LANGUAGE eq "both") {
my @cwords = split /\s+/, $fields[0];
foreach my $token (@cwords) {
my @parts = split /_(?!.*_)/, $token;
if (scalar @parts == 2) {
$token = $parts[0]
} else {
print STDERR "WARNING: invalid tagged token $token\n";
}
}
$fields[0] = join ' ', @cwords;
}
if ($LANGUAGE eq "target" or $LANGUAGE eq "both") {
my @cwords = split /\s+/, $fields[1];
foreach my $token (@cwords) {
my @parts = split /_(?!.*_)/, $token;
if (scalar @parts == 2) {
$token = $parts[1]
} else {
print STDERR "WARNING: invalid tagged token $token\n";
}
}
$fields[0] = join ' ', @cwords;
}
print join ' ||| ', @fields;
print "\n";
}
|
