From 0d2b0f252370260042a456f215b2b79c456796df Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Wed, 8 Jun 2011 00:39:09 -0400 Subject: external MT evaluator client code. most logic in place, needs to be integrated. actually, the whole evaluation architecture needs to be trashed and rewritten from scratch. what a disaster it is --- mteval/Makefile.am | 2 +- mteval/external_scorer.cc | 150 ++++++++++++++++++++++++++++++++++++++++++++++ mteval/external_scorer.h | 35 +++++++++++ 3 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 mteval/external_scorer.cc create mode 100644 mteval/external_scorer.h diff --git a/mteval/Makefile.am b/mteval/Makefile.am index f9277779..95845090 100644 --- a/mteval/Makefile.am +++ b/mteval/Makefile.am @@ -10,7 +10,7 @@ endif noinst_LIBRARIES = libmteval.a -libmteval_a_SOURCES = ter.cc comb_scorer.cc aer_scorer.cc scorer.cc +libmteval_a_SOURCES = ter.cc comb_scorer.cc aer_scorer.cc scorer.cc external_scorer.cc fast_score_SOURCES = fast_score.cc fast_score_LDADD = libmteval.a $(top_srcdir)/utils/libutils.a -lz diff --git a/mteval/external_scorer.cc b/mteval/external_scorer.cc new file mode 100644 index 00000000..4327ce9b --- /dev/null +++ b/mteval/external_scorer.cc @@ -0,0 +1,150 @@ +#include "external_scorer.h" + +#include // popen +#include +#include +#include +#include + +#include "tdict.h" + +using namespace std; + +ScoreServer::ScoreServer(const string& cmd) : pipe_() { + cerr << "Invoking " << cmd << " ..." << endl; + pipe_ = popen(cmd.c_str(), "r+"); + assert(pipe_); + string dummy; + RequestResponse("EVAL ||| Reference initialization string . ||| Testing initialization string .\n", &dummy); + assert(dummy.size() > 0); + cerr << "Connection established.\n"; +} + +ScoreServer::~ScoreServer() { + pclose(pipe_); +} + +double ScoreServer::ComputeScore(const vector& fields) { + ostringstream os; + os << "EVAL"; + for (unsigned i = 0; i < fields.size(); ++i) + os << ' ' << fields[i]; + os << endl; + string sres; + RequestResponse(os.str(), &sres); + return strtod(sres.c_str(), NULL); +} + +void ScoreServer::Evaluate(const vector >& refs, const vector& hyp, vector* fields) { + ostringstream os; + os << "SCORE"; + for (unsigned i = 0; i < refs.size(); ++i) { + os << " |||"; + for (unsigned j = 0; j < refs[i].size(); ++j) { + os << ' ' << TD::Convert(refs[i][j]); + } + } + os << " |||"; + for (unsigned i = 0; i < hyp.size(); ++i) { + os << ' ' << TD::Convert(hyp[i]); + } + os << endl; + string sres; + RequestResponse(os.str(), &sres); + istringstream is(sres); + double val; + fields->clear(); + while(is >> val) { + fields->push_back(val); + } +} + +#define MAX_BUF 16000 + +void ScoreServer::RequestResponse(const string& request, string* response) { + fprintf(pipe_, "%s", request.c_str()); + fflush(pipe_); + char buf[MAX_BUF]; + size_t cr = fread(buf, 1, MAX_BUF, pipe_); + if (cr == 0) { + cerr << "Read error. Request: " << request << endl; + abort(); + } + while (buf[cr-1] != '\n') { + size_t n = fread(&buf[cr], 1, MAX_BUF-cr, pipe_); + assert(n > 0); + cr += n; + assert(cr < MAX_BUF); + } + buf[cr - 1] = 0; + *response = buf; +} + +struct ExternalScore : public ScoreBase { + ExternalScore() : score_server() {} + explicit ExternalScore(const ScoreServer* s) : score_server(s), fields() {} + ExternalScore(const ScoreServer* s, const vector& f) : score_server(s), fields(f) {} + float ComputePartialScore() const { return 0.0;} + float ComputeScore() const { + // TODO make EVAL call + assert(!"not implemented"); + } + void ScoreDetails(string* details) const { + ostringstream os; + os << "EXT=" << ComputeScore() << " <"; + for (unsigned i = 0; i < fields.size(); ++i) + os << (i ? " " : "") << fields[i]; + os << '>'; + *details = os.str(); + } + void PlusPartialEquals(const Score&, int, int, int){ + assert(!"not implemented"); // no idea + } + void PlusEquals(const Score& delta, const float scale) { + assert(!"not implemented"); // don't even know what this is + } + void PlusEquals(const Score& delta) { + if (static_cast(delta).score_server) score_server = static_cast(delta).score_server; + if (fields.size() != static_cast(delta).fields.size()) + fields.resize(max(fields.size(), static_cast(delta).fields.size())); + for (unsigned i = 0; i < static_cast(delta).fields.size(); ++i) + fields[i] += static_cast(delta).fields[i]; + } + ScoreP GetZero() const { + return ScoreP(new ExternalScore(score_server)); + } + ScoreP GetOne() const { + return ScoreP(new ExternalScore(score_server)); + } + void Subtract(const Score& rhs, Score* res) const { + static_cast(res)->score_server = score_server; + vector& rf = static_cast(res)->fields; + rf.resize(max(fields.size(), static_cast(rhs).fields.size())); + for (unsigned i = 0; i < rf.size(); ++i) { + rf[i] = (i < fields.size() ? fields[i] : 0.0f) - + (i < static_cast(rhs).fields.size() ? static_cast(rhs).fields[i] : 0.0f); + } + } + void Encode(string* out) const { + ostringstream os; + } + bool IsAdditiveIdentity() const { + for (int i = 0; i < fields.size(); ++i) + if (fields[i]) return false; + return true; + } + + const ScoreServer* score_server; + vector fields; +}; + +ScoreP ExternalSentenceScorer::ScoreCandidate(const Sentence& hyp) const { + ExternalScore* res = new ExternalScore(eval_server); + eval_server->Evaluate(refs, hyp, &res->fields); + return ScoreP(res); +} + +ScoreP ExternalSentenceScorer::ScoreCCandidate(const Sentence& hyp) const { + assert(!"not implemented"); +} + diff --git a/mteval/external_scorer.h b/mteval/external_scorer.h new file mode 100644 index 00000000..a2c91960 --- /dev/null +++ b/mteval/external_scorer.h @@ -0,0 +1,35 @@ +#ifndef _EXTERNAL_SCORER_H_ +#define _EXTERNAL_SCORER_H_ + +#include +#include + +#include "scorer.h" + +class ScoreServer { + public: + explicit ScoreServer(const std::string& cmd); + virtual ~ScoreServer(); + + double ComputeScore(const std::vector& fields); + void Evaluate(const std::vector >& refs, const std::vector& hyp, std::vector* fields); + + private: + void RequestResponse(const std::string& request, std::string* response); + FILE* pipe_; +}; + +class ExternalSentenceScorer : public SentenceScorer { + public: + virtual ScoreP ScoreCandidate(const Sentence& hyp) const = 0; + virtual ScoreP ScoreCCandidate(const Sentence& hyp) const =0; + protected: + ScoreServer* eval_server; +}; + +class METEORServer : public ScoreServer { + public: + METEORServer() : ScoreServer("java -Xmx1024m -jar meteor-1.3.jar - - -mira -lower") {} +}; + +#endif -- cgit v1.2.3