1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
|
#include "ns_ext.h"
#include <cstdio> // popen
#include <cstdlib>
#include <cstring>
#include <unistd.h>
#include <signal.h>
#include <sys/wait.h>
#include <sstream>
#include <iostream>
#include <cassert>
#include "stringlib.h"
#include "tdict.h"
using namespace std;
void metric_child_signal_handler(int signo) {
int status = 0;
cerr << "Received SIGCHLD(" << signo << ") ... aborting.\n";
// reap zombies
while (waitpid(-1, &status, WNOHANG) > 0) {}
abort();
}
struct NScoreServer {
NScoreServer(const std::string& cmd);
~NScoreServer();
float ComputeScore(const std::vector<float>& fields);
void Evaluate(const std::vector<std::vector<WordID> >& refs, const std::vector<WordID>& hyp, std::vector<float>* fields);
private:
void RequestResponse(const std::string& request, std::string* response);
int p2c[2];
int c2p[2];
};
NScoreServer::NScoreServer(const string& cmd) {
static bool need_init = true;
if (need_init) {
need_init = false;
signal(SIGCHLD, metric_child_signal_handler);
}
cerr << "Invoking " << cmd << " ..." << endl;
if (pipe(p2c) < 0) { perror("pipe"); exit(1); }
if (pipe(c2p) < 0) { perror("pipe"); exit(1); }
pid_t cpid = fork();
if (cpid < 0) { perror("fork"); exit(1); }
if (cpid == 0) { // child
close(p2c[1]);
close(c2p[0]);
dup2(p2c[0], 0);
close(p2c[0]);
dup2(c2p[1], 1);
close(c2p[1]);
cerr << "Exec'ing from child " << cmd << endl;
vector<string> vargs;
SplitOnWhitespace(cmd, &vargs);
const char** cargv = static_cast<const char**>(malloc(sizeof(const char*) * vargs.size()));
for (unsigned i = 1; i < vargs.size(); ++i) cargv[i-1] = vargs[i].c_str();
cargv[vargs.size() - 1] = NULL;
execvp(vargs[0].c_str(), (char* const*)cargv);
} else { // parent
close(c2p[1]);
close(p2c[0]);
}
string dummy;
RequestResponse("SCORE ||| Reference initialization string . ||| Testing initialization string .", &dummy);
assert(dummy.size() > 0);
cerr << "Connection established.\n";
}
NScoreServer::~NScoreServer() {
// TODO close stuff, join stuff
}
float NScoreServer::ComputeScore(const vector<float>& fields) {
ostringstream os;
os << "EVAL |||";
for (unsigned i = 0; i < fields.size(); ++i)
os << ' ' << fields[i];
string sres;
RequestResponse(os.str(), &sres);
return strtod(sres.c_str(), NULL);
}
void NScoreServer::Evaluate(const vector<vector<WordID> >& refs, const vector<WordID>& hyp, vector<float>* fields) {
ostringstream os;
os << "SCORE";
for (unsigned i = 0; i < refs.size(); ++i) {
os << " |||";
for (unsigned j = 0; j < refs[i].size(); ++j) {
os << ' ' << TD::Convert(refs[i][j]);
}
}
os << " |||";
for (unsigned i = 0; i < hyp.size(); ++i) {
os << ' ' << TD::Convert(hyp[i]);
}
string sres;
RequestResponse(os.str(), &sres);
istringstream is(sres);
float val;
fields->clear();
while(is >> val)
fields->push_back(val);
}
#define MAX_BUF 16000
void NScoreServer::RequestResponse(const string& request, string* response) {
// cerr << "@SERVER: " << request << endl;
string x = request + "\n";
write(p2c[1], x.c_str(), x.size());
char buf[MAX_BUF];
size_t n = read(c2p[0], buf, MAX_BUF);
while (n < MAX_BUF && buf[n-1] != '\n')
n += read(c2p[0], &buf[n], MAX_BUF - n);
buf[n-1] = 0;
if (n < 2) {
cerr << "Malformed response: " << buf << endl;
}
*response = Trim(buf, " \t\n");
// cerr << "@RESPONSE: '" << *response << "'\n";
}
void ExternalMetric::ComputeSufficientStatistics(const std::vector<WordID>& hyp,
const std::vector<std::vector<WordID> >& refs,
SufficientStats* out) const {
eval_server->Evaluate(refs, hyp, &out->fields);
}
float ExternalMetric::ComputeScore(const SufficientStats& stats) const {
return eval_server->ComputeScore(stats.fields);
}
ExternalMetric::ExternalMetric(const string& metric_name, const std::string& command) :
EvaluationMetric(metric_name),
eval_server(new NScoreServer(command)) {}
ExternalMetric::~ExternalMetric() {
delete eval_server;
}
|
