1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
#include "ns_ext.h"
#include <cstdio> // popen
#include <cstdlib>
#include <cstring>
#include <unistd.h>
#include <sstream>
#include <iostream>
#include <cassert>
#include "stringlib.h"
#include "tdict.h"
using namespace std;
struct NScoreServer {
NScoreServer(const std::string& cmd);
~NScoreServer();
float ComputeScore(const std::vector<float>& fields);
void Evaluate(const std::vector<std::vector<WordID> >& refs, const std::vector<WordID>& hyp, std::vector<float>* fields);
private:
void RequestResponse(const std::string& request, std::string* response);
int p2c[2];
int c2p[2];
};
NScoreServer::NScoreServer(const string& cmd) {
cerr << "Invoking " << cmd << " ..." << endl;
if (pipe(p2c) < 0) { perror("pipe"); exit(1); }
if (pipe(c2p) < 0) { perror("pipe"); exit(1); }
pid_t cpid = fork();
if (cpid < 0) { perror("fork"); exit(1); }
if (cpid == 0) { // child
close(p2c[1]);
close(c2p[0]);
dup2(p2c[0], 0);
close(p2c[0]);
dup2(c2p[1], 1);
close(c2p[1]);
cerr << "Exec'ing from child " << cmd << endl;
vector<string> vargs;
SplitOnWhitespace(cmd, &vargs);
const char** cargv = static_cast<const char**>(malloc(sizeof(const char*) * vargs.size()));
for (unsigned i = 1; i < vargs.size(); ++i) cargv[i-1] = vargs[i].c_str();
cargv[vargs.size() - 1] = NULL;
execvp(vargs[0].c_str(), (char* const*)cargv);
} else { // parent
close(c2p[1]);
close(p2c[0]);
}
string dummy;
RequestResponse("SCORE ||| Reference initialization string . ||| Testing initialization string .", &dummy);
assert(dummy.size() > 0);
cerr << "Connection established.\n";
}
NScoreServer::~NScoreServer() {
// TODO close stuff, join stuff
}
float NScoreServer::ComputeScore(const vector<float>& fields) {
ostringstream os;
os << "EVAL |||";
for (unsigned i = 0; i < fields.size(); ++i)
os << ' ' << fields[i];
string sres;
RequestResponse(os.str(), &sres);
return strtod(sres.c_str(), NULL);
}
void NScoreServer::Evaluate(const vector<vector<WordID> >& refs, const vector<WordID>& hyp, vector<float>* fields) {
ostringstream os;
os << "SCORE";
for (unsigned i = 0; i < refs.size(); ++i) {
os << " |||";
for (unsigned j = 0; j < refs[i].size(); ++j) {
os << ' ' << TD::Convert(refs[i][j]);
}
}
os << " |||";
for (unsigned i = 0; i < hyp.size(); ++i) {
os << ' ' << TD::Convert(hyp[i]);
}
string sres;
RequestResponse(os.str(), &sres);
istringstream is(sres);
float val;
fields->clear();
while(is >> val)
fields->push_back(val);
}
#define MAX_BUF 16000
void NScoreServer::RequestResponse(const string& request, string* response) {
// cerr << "@SERVER: " << request << endl;
string x = request + "\n";
write(p2c[1], x.c_str(), x.size());
char buf[MAX_BUF];
size_t n = read(c2p[0], buf, MAX_BUF);
while (n < MAX_BUF && buf[n-1] != '\n')
n += read(c2p[0], &buf[n], MAX_BUF - n);
buf[n-1] = 0;
if (n < 2) {
cerr << "Malformed response: " << buf << endl;
}
*response = Trim(buf, " \t\n");
// cerr << "@RESPONSE: '" << *response << "'\n";
}
void ExternalMetric::ComputeSufficientStatistics(const std::vector<WordID>& hyp,
const std::vector<std::vector<WordID> >& refs,
SufficientStats* out) const {
eval_server->Evaluate(refs, hyp, &out->fields);
}
float ExternalMetric::ComputeScore(const SufficientStats& stats) const {
eval_server->ComputeScore(stats.fields);
}
ExternalMetric::ExternalMetric(const string& metric_name, const std::string& command) :
EvaluationMetric(metric_name),
eval_server(new NScoreServer(command)) {}
ExternalMetric::~ExternalMetric() {
delete eval_server;
}
|
