1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
|
#!/usr/bin/perl -w
use strict;
use Getopt::Long;
use Cwd;
my $CWD = getcwd;
my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR; }
my $EXTOOLS = "$SCRIPT_DIR/../../extools";
die "Can't find extools: $EXTOOLS" unless -e $EXTOOLS && -d $EXTOOLS;
my $VEST = "$SCRIPT_DIR/../../vest";
die "Can't find vest: $VEST" unless -e $VEST && -d $VEST;
my $DISTVEST = "$VEST/dist-vest.pl";
my $FILTSCORE = "$EXTOOLS/filter_score_grammar";
assert_exec($FILTSCORE, $DISTVEST);
my %init_weights = qw(
EGivenF -0.3
FGivenE -0.3
LexE2F -0.3
LexF2E -0.3
WordPenalty -1.5
LanguageModel 1.2
Glue -1.0
GlueTop 0.00001
PassThrough -10.0
X_EGivenF -0.3
X_FGivenE -0.3
);
my $config = "$SCRIPT_DIR/config.eval";
open CONF, "<$config" or die "Can't read $config: $!";
my %paths;
my %corpora;
my %lms;
my %devs;
my %devrefs;
my %tests;
my %testevals;
print STDERR "LANGUAGE PAIRS:";
while(<CONF>) {
chomp;
next if /^#/;
next if /^\s*$/;
s/^\s+//;
s/\s+$//;
my ($name, $path, $corpus, $lm, $dev, $devref, @xtests) = split /\s+/;
$paths{$name} = $path;
$corpora{$name} = $corpus;
$lms{$name} = $lm;
$devs{$name} = $dev;
$devrefs{$name} = $devref;
$tests{$name} = $xtests[0];
$testevals{$name} = $xtests[1];
print STDERR " $name";
}
print STDERR "\n";
my %langpairs = map { $_ => 1 } qw( btec zhen fbis aren uren nlfr );
my $outdir = "$CWD/exp";
my $help;
my $dataDir = '/export/ws10smt/data';
if (GetOptions(
"data=s" => \$dataDir,
) == 0 || @ARGV!=2 || $help) {
print_help();
exit;
}
my $lp = $ARGV[0];
my $grammar = $ARGV[1];
print STDERR " CORPUS REPO: $dataDir\n";
print STDERR " LANGUAGE PAIR: $lp\n";
die "I don't know about that language pair\n" unless $paths{$lp};
my $corpdir = "$dataDir";
if ($paths{$lp} =~ /^\//) { $corpdir = $paths{$lp}; } else { $corpdir .= '/' . $paths{$lp}; }
die "I can't find the corpora directory: $corpdir" unless -d $corpdir;
print STDERR " GRAMMAR: $grammar\n";
my $LANG_MODEL = mydircat($corpdir, $lms{$lp});
print STDERR " LM: $LANG_MODEL\n";
my $CORPUS = mydircat($corpdir, $corpora{$lp});
die "Can't find corpus: $CORPUS" unless -f $CORPUS;
my $dev = mydircat($corpdir, $devs{$lp});
my $drefs = $devrefs{$lp};
die "Can't find dev: $dev\n" unless -f $dev;
die "Dev refs not set" unless $drefs;
$drefs = mydircat($corpdir, $drefs);
my $test = mydircat($corpdir, $tests{$lp});
my $teval = mydircat($corpdir, $testevals{$lp});
die "Can't find test: $test\n" unless -f $test;
assert_exec($teval);
# MAKE DEV
print STDERR "\nFILTERING FOR dev...\n";
print STDERR "DEV: $dev (REFS=$drefs)\n";
`mkdir -p $outdir`;
my $devgrammar = filter($grammar, $dev, 'dev', $outdir);
my $devini = mydircat($outdir, "cdec-dev.ini");
write_cdec_ini($devini, $devgrammar);
# MAKE TEST
print STDERR "\nFILTERING FOR test...\n";
print STDERR "TEST: $test (EVAL=$teval)\n";
`mkdir -p $outdir`;
my $testgrammar = filter($grammar, $test, 'test', $outdir);
my $testini = mydircat($outdir, "cdec-test.ini");
write_cdec_ini($testini, $testgrammar);
sub filter {
my ($grammar, $set, $name, $outdir) = @_;
my $outgrammar = mydircat($outdir, "$name.scfg.gz");
if (-f $outgrammar) { print STDERR "$outgrammar exists - REUSING!\n"; } else {
my $cmd = "gunzip -c $grammar | $FILTSCORE -c $CORPUS -t $dev | gzip > $outgrammar";
safesystem($cmd) or die "Can't filter and score grammar!";
}
return $outgrammar;
}
sub mydircat {
my ($base, $suffix) = @_;
if ($suffix =~ /^\//) { return $suffix; }
my $res = $base . '/' . $suffix;
$res =~ s/\/\//\//g;
return $res;
}
sub write_cdec_ini {
my ($filename, $grammar_path) = (@_);
open CDECINI, ">$filename" or die "Can't write $filename: $!";
print CDECINI <<EOT;
formalism=scfg
cubepruning_pop_limit=100
add_pass_through_rules=true
scfg_extra_glue_grammar=/export/ws10smt/data/glue/glue.scfg.gz
grammar=$grammar_path
feature_function=WordPenalty
feature_function=LanguageModel -o 3 $LANG_MODEL
EOT
close CDECINI;
};
sub print_help {
print STDERR<<EOT;
Usage: $0 [OPTIONS] language-pair grammar.bidir.gz
Given an induced grammar for an entire corpus (i.e., generated by
local-gi-pipeline.pl), filter and featurize it for a dev and test set,
run MERT, report scores.
EOT
}
sub safesystem {
print STDERR "Executing: @_\n";
system(@_);
if ($? == -1) {
print STDERR "ERROR: Failed to execute: @_\n $!\n";
exit(1);
}
elsif ($? & 127) {
printf STDERR "ERROR: Execution of: @_\n died with signal %d, %s coredump\n",
($? & 127), ($? & 128) ? 'with' : 'without';
exit(1);
}
else {
my $exitcode = $? >> 8;
print STDERR "Exit code: $exitcode\n" if $exitcode;
return ! $exitcode;
}
}
sub assert_exec {
my @files = @_;
for my $file (@files) {
die "Can't find $file - did you run make?\n" unless -e $file;
die "Can't execute $file" unless -e $file;
}
};
|
