1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
|
import subprocess
class GeoWorld:
def __init__(self, config):
self.config = config
def run(self):
self.write_queries()
infile = open('%s/eval.pl' % self.config.experiment_dir)
log = open('%s/prolog.log' % self.config.experiment_dir, 'w')
outfile = open('%s/eval.out' % self.config.experiment_dir, 'w')
p = subprocess.Popen([self.config.prolog,
'-l', self.config.wasp_eval],
stdin=infile,
stdout=outfile,
stderr=log)
p.wait()
infile.close()
log.close()
outfile.close()
self.extract_results()
def write_queries(self):
hyp_file = open('%s/hyp.fun' % self.config.experiment_dir)
ref_file = open('%s/test.fun' % self.config.experiment_dir)
query_file = open('%s/eval.pl' % self.config.experiment_dir, 'w')
examples = []
hyp_list = []
last_idx = 0
for hyp_line in hyp_file.readlines():
idx, hyp, scoreparts, score = hyp_line.split('|||')
idx = int(idx)
hyp = hyp.strip()
if idx != last_idx:
examples.append(hyp_list)
for i in range(last_idx, idx-1):
examples.append([])
hyp_list = []
last_idx = idx
hyp_list.append((hyp,float(score)))
examples.append(hyp_list)
i = 0
for ref, hyp_list in zip(ref_file.readlines(), examples):
ref = ref.strip()
for hyp, score in hyp_list:
print >>query_file, \
'catch(call_with_time_limit(1,eval([%d,%f,%s,%s])),E,writeln(\'error\')).\n' \
% (i, score, ref, hyp)
i += 1
hyp_file.close()
ref_file.close()
query_file.close()
def extract_results(self):
eval_file = open('%s/eval.out' % self.config.experiment_dir)
result_file = open('%s/eval.scored' % self.config.experiment_dir, 'w')
examples = []
hyp_list = []
last_idx = 0
for line in eval_file.readlines():
if line == 'error\n':
continue
idx, score, result = line.split()
idx = int(idx)
score = float(score)
if idx > last_idx:
examples.append(hyp_list)
last_idx += 1
while idx > last_idx:
examples.append([])
last_idx += 1
hyp_list = []
hyp_list.append((result,score))
examples.append(hyp_list)
last_idx += 1
if self.config.corpus == 'geo' and self.config.run in ('debug', 'dev'):
top = 60
elif self.config.corpus == 'geo' and self.config.run == 'test':
top = 280
else:
assert False
while top > last_idx:
examples.append([])
last_idx += 1
for hyp_list in examples:
if len(hyp_list) == 0:
print >>result_file, 'empty'
continue
choice, score = hyp_list[0]
if choice == 'y':
print >>result_file, 'yes', score
else:
print >>result_file, 'no', score
eval_file.close()
result_file.close()
|
