blob: c644ec6d8660343a95c13ef365bfbfa451f39cc2 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
|
#!/usr/bin/perl -w
use strict;
use utf8;
die "Usage: $0 f.voc corpus.f-e grammar.f-e.gz\n" unless scalar @ARGV == 3;
my $MAX_INMEM = 3000;
open FV,"<$ARGV[0]" or die "Can't read $ARGV[0]: $!";
open C,"<$ARGV[1]" or die "Can't read $ARGV[1]: $!";
open G,"gunzip -c $ARGV[2]|" or die "Can't read $ARGV[2]: $!";
binmode FV, ":utf8";
binmode C, ":utf8";
binmode G, ":utf8";
my $vc = 0;
my %most_freq;
$most_freq{"<eps>"} = 1;
while(my $f = <FV>) {
chomp $f;
$most_freq{$f}=1;
$vc++;
last if $vc == $MAX_INMEM;
}
close FV;
print STDERR "Loaded $vc vocabulary items for permanent translation cache\n";
my %grammar;
my $memrc = 0;
my $loadrc = 0;
while(<G>) {
chomp;
my ($f, $e, $feats) = split / \|\|\| /;
if ($most_freq{$f}) {
#print "$_\n";
$memrc++;
} else {
$loadrc++;
my $r = $grammar{$f};
if (!defined $r) {
$r = [];
$grammar{$f} = $r;
}
push @$r, "$e ||| $feats";
}
}
close G;
print STDERR " mem rc: $memrc\n";
print STDERR " load rc: $loadrc\n";
while(<C>) {
my ($f,$e) = split / \|\|\| /;
my @fwords = split /\s+/, $f;
my $tot = 0;
for my $f (@fwords) {
my $r = $grammar{$f};
die "No translations for: $f" unless $r;
my $num = scalar @$r;
$tot += $num;
}
print "RULES: $tot\n";
}
|
