1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
|
#include <iostream>
#include <vector>
#include <boost/program_options.hpp>
#include <boost/program_options/variables_map.hpp>
#include "stringlib.h"
#include "filelib.h"
#include "tdict.h"
#include "ns.h"
#include "ns_docscorer.h"
using namespace std;
namespace po = boost::program_options;
void InitCommandLine(int argc, char** argv, po::variables_map* conf) {
po::options_description opts("Configuration options");
opts.add_options()
("reference,r",po::value<vector<string> >(), "[1 or more required] Reference translation(s) in tokenized text files")
("evaluation_metric,m",po::value<string>()->default_value("IBM_BLEU"), "Evaluation metric (ibm_bleu, koehn_bleu, nist_bleu, ter, meteor, etc.)")
("in_file,i", po::value<string>()->default_value("-"), "Input file")
("help,h", "Help");
po::options_description dcmdline_options;
dcmdline_options.add(opts);
po::store(parse_command_line(argc, argv, dcmdline_options), *conf);
bool flag = false;
if (!conf->count("reference")) {
cerr << "Please specify one or more references using -r <REF1.TXT> -r <REF2.TXT> ...\n";
flag = true;
}
if (flag || conf->count("help")) {
cerr << dcmdline_options << endl;
exit(1);
}
}
int main(int argc, char** argv) {
po::variables_map conf;
InitCommandLine(argc, argv, &conf);
string loss_function = UppercaseString(conf["evaluation_metric"].as<string>());
if (loss_function == "COMBI") {
cerr << "WARNING: 'combi' metric is no longer supported, switching to 'COMB:TER=-0.5;IBM_BLEU=0.5'\n";
loss_function = "COMB:TER=-0.5;IBM_BLEU=0.5";
} else if (loss_function == "BLEU") {
cerr << "WARNING: 'BLEU' is ambiguous, assuming 'IBM_BLEU'\n";
loss_function = "IBM_BLEU";
}
EvaluationMetric* metric = EvaluationMetric::Instance(loss_function);
DocumentScorer ds(metric, conf["reference"].as<vector<string> >());
cerr << "Loaded " << ds.size() << " references for scoring with " << loss_function << endl;
ReadFile rf(conf["in_file"].as<string>());
SufficientStats acc;
istream& in = *rf.stream();
int lc = 0;
string line;
while(getline(in, line)) {
vector<WordID> sent;
TD::ConvertSentence(line, &sent);
SufficientStats t;
ds[lc]->Evaluate(sent, &t);
acc += t;
++lc;
}
assert(lc > 0);
if (lc > ds.size()) {
cerr << "Too many (" << lc << ") translations in input, expected " << ds.size() << endl;
return 1;
}
if (lc != ds.size())
cerr << "Fewer sentences in hyp (" << lc << ") than refs ("
<< ds.size() << "): scoring partial set!\n";
float score = metric->ComputeScore(acc);
const string details = metric->DetailedScore(acc);
cerr << details << endl;
cout << score << endl;
return 0;
}
|
