From 2eb3bb96c6f780c477585b33273fc0c0d56c80e4 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Tue, 20 Dec 2011 15:51:11 -0500 Subject: new scorer interface is implemented, but not used --- mteval/Makefile.am | 2 +- mteval/ns.cc | 67 ++++++++++++++++++++------ mteval/ns.h | 23 +++++---- mteval/ns_comb.cc | 87 +++++++++++++++++++++++++++++++++ mteval/ns_comb.h | 19 ++++++++ mteval/ns_ext.cc | 130 ++++++++++++++++++++++++++++++++++++++++++++++++++ mteval/ns_ext.h | 21 ++++++++ mteval/ns_ter.cc | 126 ++++++++++-------------------------------------- mteval/ns_ter.h | 1 + mteval/scorer_test.cc | 12 +++-- 10 files changed, 355 insertions(+), 133 deletions(-) create mode 100644 mteval/ns_comb.cc create mode 100644 mteval/ns_comb.h create mode 100644 mteval/ns_ext.cc create mode 100644 mteval/ns_ext.h (limited to 'mteval') diff --git a/mteval/Makefile.am b/mteval/Makefile.am index 95845090..6679d949 100644 --- a/mteval/Makefile.am +++ b/mteval/Makefile.am @@ -10,7 +10,7 @@ endif noinst_LIBRARIES = libmteval.a -libmteval_a_SOURCES = ter.cc comb_scorer.cc aer_scorer.cc scorer.cc external_scorer.cc +libmteval_a_SOURCES = ter.cc comb_scorer.cc aer_scorer.cc scorer.cc external_scorer.cc ns.cc ns_ter.cc ns_ext.cc ns_comb.cc fast_score_SOURCES = fast_score.cc fast_score_LDADD = libmteval.a $(top_srcdir)/utils/libutils.a -lz diff --git a/mteval/ns.cc b/mteval/ns.cc index 1045a51f..6139757d 100644 --- a/mteval/ns.cc +++ b/mteval/ns.cc @@ -1,5 +1,7 @@ #include "ns.h" #include "ns_ter.h" +#include "ns_ext.h" +#include "ns_comb.h" #include #include @@ -7,6 +9,9 @@ #include #include +#include "tdict.h" +#include "stringlib.h" + using namespace std; using boost::shared_ptr; @@ -19,6 +24,7 @@ struct DefaultSegmentEvaluator : public SegmentEvaluator { DefaultSegmentEvaluator(const vector >& refs, const EvaluationMetric* em) : refs_(refs), em_(em) {} void Evaluate(const vector& hyp, SufficientStats* out) const { em_->ComputeSufficientStatistics(hyp, refs_, out); + out->id_ = em_->MetricId(); } const vector > refs_; const EvaluationMetric* em_; @@ -28,6 +34,11 @@ shared_ptr EvaluationMetric::CreateSegmentEvaluator(const vect return shared_ptr(new DefaultSegmentEvaluator(refs, this)); } +#define MAX_SS_VECTOR_SIZE 50 +unsigned EvaluationMetric::SufficientStatisticsVectorSize() const { + return MAX_SS_VECTOR_SIZE; +} + void EvaluationMetric::ComputeSufficientStatistics(const vector&, const vector >&, SufficientStats*) const { @@ -35,6 +46,12 @@ void EvaluationMetric::ComputeSufficientStatistics(const vector&, abort(); } +string EvaluationMetric::DetailedScore(const SufficientStats& stats) const { + ostringstream os; + os << MetricId() << "=" << ComputeScore(stats); + return os.str(); +} + enum BleuType { IBM, Koehn, NIST }; template struct BleuSegmentEvaluator : public SegmentEvaluator { @@ -57,7 +74,7 @@ struct BleuSegmentEvaluator : public SegmentEvaluator { void Evaluate(const vector& hyp, SufficientStats* out) const { out->fields.resize(N + N + 2); - out->evaluation_metric = evaluation_metric; + out->id_ = evaluation_metric->MetricId(); for (unsigned i = 0; i < N+N+2; ++i) out->fields[i] = 0; ComputeNgramStats(hyp, &out->fields[0], &out->fields[N], true); @@ -157,7 +174,12 @@ struct BleuSegmentEvaluator : public SegmentEvaluator { template struct BleuMetric : public EvaluationMetric { BleuMetric() : EvaluationMetric("IBM_BLEU") {} - float ComputeScore(const SufficientStats& stats) const { + unsigned SufficientStatisticsVectorSize() const { return N*2 + 2; } + shared_ptr CreateSegmentEvaluator(const vector >& refs) const { + return shared_ptr(new BleuSegmentEvaluator(refs, this)); + } + float ComputeBreakdown(const SufficientStats& stats, float* bp, vector* out) const { + if (out) { out->clear(); } float log_bleu = 0; int count = 0; for (int i = 0; i < N; ++i) { @@ -166,7 +188,7 @@ struct BleuMetric : public EvaluationMetric { // smooth bleu if (!cor_count) { cor_count = 0.01; } float lprec = log(cor_count) - log(stats.fields[i+N]); // log(hyp_ngram_counts[i]); - // if (precs) precs->push_back(exp(lprec)); + if (out) out->push_back(exp(lprec)); log_bleu += lprec; ++count; } @@ -178,32 +200,51 @@ struct BleuMetric : public EvaluationMetric { if (hyp_len < ref_len) lbp = (hyp_len - ref_len) / hyp_len; log_bleu += lbp; - //if (bp) *bp = exp(lbp); + if (bp) *bp = exp(lbp); return exp(log_bleu); } - shared_ptr CreateSegmentEvaluator(const vector >& refs) const { - return shared_ptr(new BleuSegmentEvaluator(refs, this)); + string DetailedScore(const SufficientStats& stats) const { + char buf[2000]; + vector precs(N); + float bp; + float bleu = ComputeBreakdown(stats, &bp, &precs); + sprintf(buf, "BLEU = %.2f, %.1f|%.1f|%.1f|%.1f (brev=%.3f)", + bleu*100.0, + precs[0]*100.0, + precs[1]*100.0, + precs[2]*100.0, + precs[3]*100.0, + bp); + return buf; + } + float ComputeScore(const SufficientStats& stats) const { + return ComputeBreakdown(stats, NULL, NULL); } }; -EvaluationMetric* EvaluationMetric::Instance(const string& metric_id) { +EvaluationMetric* EvaluationMetric::Instance(const string& imetric_id) { static bool is_first = true; if (is_first) { instances_["NULL"] = NULL; is_first = false; } + const string metric_id = UppercaseString(imetric_id); map::iterator it = instances_.find(metric_id); if (it == instances_.end()) { EvaluationMetric* m = NULL; - if (metric_id == "IBM_BLEU") { + if (metric_id == "IBM_BLEU") { m = new BleuMetric<4, IBM>; } else if (metric_id == "NIST_BLEU") { m = new BleuMetric<4, NIST>; - } else if (metric_id == "Koehn_BLEU") { + } else if (metric_id == "KOEHN_BLEU") { m = new BleuMetric<4, Koehn>; } else if (metric_id == "TER") { m = new TERMetric; + } else if (metric_id == "METEOR") { + m = new ExternalMetric("METEOR", "java -Xmx1536m -jar /Users/cdyer/software/meteor/meteor-1.3.jar - - -mira -lower -t tune -l en"); + } else if (metric_id.find("COMB:") == 0) { + m = new CombinationMetric(metric_id); } else { cerr << "Implement please: " << metric_id << endl; abort(); @@ -220,9 +261,7 @@ EvaluationMetric* EvaluationMetric::Instance(const string& metric_id) { SufficientStats::SufficientStats(const string& encoded) { istringstream is(encoded); - string type; - is >> type; - evaluation_metric = EvaluationMetric::Instance(type); + is >> id_; float val; while(is >> val) fields.push_back(val); @@ -230,8 +269,8 @@ SufficientStats::SufficientStats(const string& encoded) { void SufficientStats::Encode(string* out) const { ostringstream os; - if (evaluation_metric) - os << evaluation_metric->MetricId(); + if (id_.size() > 0) + os << id_; else os << "NULL"; for (unsigned i = 0; i < fields.size(); ++i) diff --git a/mteval/ns.h b/mteval/ns.h index f19b7509..622265db 100644 --- a/mteval/ns.h +++ b/mteval/ns.h @@ -7,18 +7,15 @@ #include #include "wordid.h" -class EvaluationMetric; - class SufficientStats { public: - SufficientStats() : evaluation_metric() {} + SufficientStats() : id_() {} explicit SufficientStats(const std::string& encoded); - explicit SufficientStats(const EvaluationMetric* s) : evaluation_metric(s) {} - SufficientStats(const EvaluationMetric* s, const std::vector& f) : - evaluation_metric(s), fields(f) {} + SufficientStats(const std::string& mid, const std::vector& f) : + id_(mid), fields(f) {} SufficientStats& operator+=(const SufficientStats& delta) { - if (delta.evaluation_metric) evaluation_metric = delta.evaluation_metric; + if (id_.empty() && delta.id_.size()) id_ = delta.id_; if (fields.size() != delta.fields.size()) fields.resize(std::max(fields.size(), delta.fields.size())); for (unsigned i = 0; i < delta.fields.size(); ++i) @@ -26,7 +23,7 @@ class SufficientStats { return *this; } SufficientStats& operator-=(const SufficientStats& delta) { - if (delta.evaluation_metric) evaluation_metric = delta.evaluation_metric; + if (id_.empty() && delta.id_.size()) id_ = delta.id_; if (fields.size() != delta.fields.size()) fields.resize(std::max(fields.size(), delta.fields.size())); for (unsigned i = 0; i < delta.fields.size(); ++i) @@ -53,7 +50,7 @@ class SufficientStats { } void Encode(std::string* out) const; - const EvaluationMetric* evaluation_metric; + std::string id_; std::vector fields; }; @@ -73,13 +70,13 @@ struct SegmentEvaluator { }; // Instructions for implementing a new metric -// Override MetricId() and give the metric a unique string name (no spaces) // To Instance(), add something that creates the metric +// Implement ComputeScore(const SufficientStats& stats) const; // Implement ONE of the following: // 1) void ComputeSufficientStatistics(const std::vector >& refs, SufficientStats* out) const; // 2) a new SegmentEvaluator class AND CreateSegmentEvaluator(const std::vector >& refs) const; -// The later (#2) is only used when it is necessary to precompute per-segment data from a set of refs -// Implement ComputeScore(const SufficientStats& stats) const; +// [The later (#2) is only used when it is necessary to precompute per-segment data from a set of refs] +// OPTIONAL: Override SufficientStatisticsVectorSize() if it is easy to do so class EvaluationMetric { public: static EvaluationMetric* Instance(const std::string& metric_id = "IBM_BLEU"); @@ -91,7 +88,9 @@ class EvaluationMetric { public: const std::string& MetricId() const { return name_; } + virtual unsigned SufficientStatisticsVectorSize() const; virtual float ComputeScore(const SufficientStats& stats) const = 0; + virtual std::string DetailedScore(const SufficientStats& stats) const; virtual boost::shared_ptr CreateSegmentEvaluator(const std::vector >& refs) const; virtual void ComputeSufficientStatistics(const std::vector& hyp, const std::vector >& refs, diff --git a/mteval/ns_comb.cc b/mteval/ns_comb.cc new file mode 100644 index 00000000..41c634cd --- /dev/null +++ b/mteval/ns_comb.cc @@ -0,0 +1,87 @@ +#include "ns_comb.h" + +#include + +#include "stringlib.h" + +using namespace std; + +// e.g. COMB:IBM_BLEU=0.5;TER=0.5 +CombinationMetric::CombinationMetric(const std::string& cmd) : + EvaluationMetric(cmd), + total_size() { + if (cmd.find("COMB:") != 0 || cmd.size() < 9) { + cerr << "Error in combination metric specifier: " << cmd << endl; + exit(1); + } + string mix = cmd.substr(5); + vector comps; + Tokenize(cmd.substr(5), ';', &comps); + if(comps.size() < 2) { + cerr << "Error in combination metric specifier: " << cmd << endl; + exit(1); + } + vector cwpairs; + for (unsigned i = 0; i < comps.size(); ++i) { + Tokenize(comps[i], '=', &cwpairs); + if (cwpairs.size() != 2) { cerr << "Error in combination metric specifier: " << cmd << endl; exit(1); } + metrics.push_back(EvaluationMetric::Instance(cwpairs[0])); + coeffs.push_back(atof(cwpairs[1].c_str())); + offsets.push_back(total_size); + total_size += metrics.back()->SufficientStatisticsVectorSize(); + cerr << (i > 0 ? " + " : "( ") << coeffs.back() << " * " << cwpairs[0]; + } + cerr << " )\n"; +} + +struct CombinationSegmentEvaluator : public SegmentEvaluator { + CombinationSegmentEvaluator(const string& id, + const vector >& refs, + const vector& metrics, + const vector& offsets, + const unsigned ts) : id_(id), offsets_(offsets), total_size_(ts), component_evaluators_(metrics.size()) { + for (unsigned i = 0; i < metrics.size(); ++i) + component_evaluators_[i] = metrics[i]->CreateSegmentEvaluator(refs); + } + virtual void Evaluate(const std::vector& hyp, SufficientStats* out) const { + out->id_ = id_; + out->fields.resize(total_size_); + for (unsigned i = 0; i < component_evaluators_.size(); ++i) { + SufficientStats t; + component_evaluators_[i]->Evaluate(hyp, &t); + for (unsigned j = 0; j < t.fields.size(); ++j) { + unsigned op = j + offsets_[i]; + assert(op < out->fields.size()); + out->fields[op] = t[j]; + } + } + } + const string& id_; + const vector& offsets_; + const unsigned total_size_; + vector > component_evaluators_; +}; + +boost::shared_ptr CombinationMetric::CreateSegmentEvaluator(const std::vector >& refs) const { + boost::shared_ptr res; + res.reset(new CombinationSegmentEvaluator(MetricId(), refs, metrics, offsets, total_size)); + return res; +} + +float CombinationMetric::ComputeScore(const SufficientStats& stats) const { + float tot = 0; + for (unsigned i = 0; i < metrics.size(); ++i) { + SufficientStats t; + unsigned next = total_size; + if (i + 1 < offsets.size()) next = offsets[i+1]; + for (unsigned j = offsets[i]; j < next; ++j) + t.fields.push_back(stats[j]); + tot += metrics[i]->ComputeScore(t) * coeffs[i]; + } + return tot; +} + +unsigned CombinationMetric::SufficientStatisticsVectorSize() const { + return total_size; +} + diff --git a/mteval/ns_comb.h b/mteval/ns_comb.h new file mode 100644 index 00000000..140e7e6a --- /dev/null +++ b/mteval/ns_comb.h @@ -0,0 +1,19 @@ +#ifndef _NS_COMB_H_ +#define _NS_COMB_H_ + +#include "ns.h" + +class CombinationMetric : public EvaluationMetric { + public: + CombinationMetric(const std::string& cmd); + virtual boost::shared_ptr CreateSegmentEvaluator(const std::vector >& refs) const; + virtual float ComputeScore(const SufficientStats& stats) const; + virtual unsigned SufficientStatisticsVectorSize() const; + private: + std::vector metrics; + std::vector coeffs; + std::vector offsets; + unsigned total_size; +}; + +#endif diff --git a/mteval/ns_ext.cc b/mteval/ns_ext.cc new file mode 100644 index 00000000..956708af --- /dev/null +++ b/mteval/ns_ext.cc @@ -0,0 +1,130 @@ +#include "ns_ext.h" + +#include // popen +#include +#include +#include +#include +#include +#include + +#include "stringlib.h" +#include "tdict.h" + +using namespace std; + +struct NScoreServer { + NScoreServer(const std::string& cmd); + ~NScoreServer(); + + float ComputeScore(const std::vector& fields); + void Evaluate(const std::vector >& refs, const std::vector& hyp, std::vector* fields); + + private: + void RequestResponse(const std::string& request, std::string* response); + int p2c[2]; + int c2p[2]; +}; + +NScoreServer::NScoreServer(const string& cmd) { + cerr << "Invoking " << cmd << " ..." << endl; + if (pipe(p2c) < 0) { perror("pipe"); exit(1); } + if (pipe(c2p) < 0) { perror("pipe"); exit(1); } + pid_t cpid = fork(); + if (cpid < 0) { perror("fork"); exit(1); } + if (cpid == 0) { // child + close(p2c[1]); + close(c2p[0]); + dup2(p2c[0], 0); + close(p2c[0]); + dup2(c2p[1], 1); + close(c2p[1]); + cerr << "Exec'ing from child " << cmd << endl; + vector vargs; + SplitOnWhitespace(cmd, &vargs); + const char** cargv = static_cast(malloc(sizeof(const char*) * vargs.size())); + for (unsigned i = 1; i < vargs.size(); ++i) cargv[i-1] = vargs[i].c_str(); + cargv[vargs.size() - 1] = NULL; + execvp(vargs[0].c_str(), (char* const*)cargv); + } else { // parent + close(c2p[1]); + close(p2c[0]); + } + string dummy; + RequestResponse("SCORE ||| Reference initialization string . ||| Testing initialization string .", &dummy); + assert(dummy.size() > 0); + cerr << "Connection established.\n"; +} + +NScoreServer::~NScoreServer() { + // TODO close stuff, join stuff +} + +float NScoreServer::ComputeScore(const vector& fields) { + ostringstream os; + os << "EVAL |||"; + for (unsigned i = 0; i < fields.size(); ++i) + os << ' ' << fields[i]; + string sres; + RequestResponse(os.str(), &sres); + return strtod(sres.c_str(), NULL); +} + +void NScoreServer::Evaluate(const vector >& refs, const vector& hyp, vector* fields) { + ostringstream os; + os << "SCORE"; + for (unsigned i = 0; i < refs.size(); ++i) { + os << " |||"; + for (unsigned j = 0; j < refs[i].size(); ++j) { + os << ' ' << TD::Convert(refs[i][j]); + } + } + os << " |||"; + for (unsigned i = 0; i < hyp.size(); ++i) { + os << ' ' << TD::Convert(hyp[i]); + } + string sres; + RequestResponse(os.str(), &sres); + istringstream is(sres); + float val; + fields->clear(); + while(is >> val) + fields->push_back(val); +} + +#define MAX_BUF 16000 + +void NScoreServer::RequestResponse(const string& request, string* response) { +// cerr << "@SERVER: " << request << endl; + string x = request + "\n"; + write(p2c[1], x.c_str(), x.size()); + char buf[MAX_BUF]; + size_t n = read(c2p[0], buf, MAX_BUF); + while (n < MAX_BUF && buf[n-1] != '\n') + n += read(c2p[0], &buf[n], MAX_BUF - n); + + buf[n-1] = 0; + if (n < 2) { + cerr << "Malformed response: " << buf << endl; + } + *response = Trim(buf, " \t\n"); +// cerr << "@RESPONSE: '" << *response << "'\n"; +} + +void ExternalMetric::ComputeSufficientStatistics(const std::vector& hyp, + const std::vector >& refs, + SufficientStats* out) const { + eval_server->Evaluate(refs, hyp, &out->fields); +} + +float ExternalMetric::ComputeScore(const SufficientStats& stats) const { + eval_server->ComputeScore(stats.fields); +} + +ExternalMetric::ExternalMetric(const string& metric_name, const std::string& command) : + EvaluationMetric(metric_name), + eval_server(new NScoreServer(command)) {} + +ExternalMetric::~ExternalMetric() { + delete eval_server; +} diff --git a/mteval/ns_ext.h b/mteval/ns_ext.h new file mode 100644 index 00000000..78badb2e --- /dev/null +++ b/mteval/ns_ext.h @@ -0,0 +1,21 @@ +#ifndef _NS_EXTERNAL_SCORER_H_ +#define _NS_EXTERNAL_SCORER_H_ + +#include "ns.h" + +struct NScoreServer; +class ExternalMetric : public EvaluationMetric { + public: + ExternalMetric(const std::string& metricid, const std::string& command); + ~ExternalMetric(); + + virtual void ComputeSufficientStatistics(const std::vector& hyp, + const std::vector >& refs, + SufficientStats* out) const; + virtual float ComputeScore(const SufficientStats& stats) const; + + protected: + NScoreServer* eval_server; +}; + +#endif diff --git a/mteval/ns_ter.cc b/mteval/ns_ter.cc index 14dc6e49..8c969e58 100644 --- a/mteval/ns_ter.cc +++ b/mteval/ns_ter.cc @@ -1,15 +1,11 @@ #include "ns_ter.h" -#include #include #include #include -#include #include #include -#include #include -#include #include "tdict.h" static const bool ter_use_average_ref_len = true; @@ -25,7 +21,7 @@ static const unsigned kDUMMY_LAST_ENTRY = 5; using namespace std; using namespace std::tr1; -#if 0 +namespace NewScorer { struct COSTS { static const float substitution; @@ -82,7 +78,7 @@ class TERScorerImpl { enum TransType { MATCH, SUBSTITUTION, INSERTION, DELETION }; explicit TERScorerImpl(const vector& ref) : ref_(ref) { - for (int i = 0; i < ref.size(); ++i) + for (unsigned i = 0; i < ref.size(); ++i) rwexists_.insert(ref[i]); } @@ -95,7 +91,7 @@ class TERScorerImpl { } private: - vector ref_; + const vector& ref_; set rwexists_; typedef unordered_map, set, boost::hash > > NgramToIntsMap; @@ -421,68 +417,7 @@ class TERScorerImpl { } }; -class TERScore : public ScoreBase { - friend class TERScorer; - - public: - - TERScore() : stats(0,kDUMMY_LAST_ENTRY) {} - float ComputePartialScore() const { return 0.0;} - float ComputeScore() const { - float edits = static_cast(stats[kINSERTIONS] + stats[kDELETIONS] + stats[kSUBSTITUTIONS] + stats[kSHIFTS]); - return edits / static_cast(stats[kREF_WORDCOUNT]); - } - void ScoreDetails(string* details) const; - void PlusPartialEquals(const Score& rhs, int oracle_e_cover, int oracle_f_cover, int src_len){} - void PlusEquals(const Score& delta, const float scale) { - if (scale==1) - stats += static_cast(delta).stats; - if (scale==-1) - stats -= static_cast(delta).stats; - throw std::runtime_error("TERScore::PlusEquals with scale != +-1"); - } - void PlusEquals(const Score& delta) { - stats += static_cast(delta).stats; - } - - ScoreP GetZero() const { - return ScoreP(new TERScore); - } - ScoreP GetOne() const { - return ScoreP(new TERScore); - } - void Subtract(const Score& rhs, Score* res) const { - static_cast(res)->stats = stats - static_cast(rhs).stats; - } - void Encode(std::string* out) const { - ostringstream os; - os << stats[kINSERTIONS] << ' ' - << stats[kDELETIONS] << ' ' - << stats[kSUBSTITUTIONS] << ' ' - << stats[kSHIFTS] << ' ' - << stats[kREF_WORDCOUNT]; - *out = os.str(); - } - bool IsAdditiveIdentity() const { - for (int i = 0; i < kDUMMY_LAST_ENTRY; ++i) - if (stats[i] != 0) return false; - return true; - } - private: - valarray stats; -}; - -ScoreP TERScorer::ScoreFromString(const std::string& data) { - istringstream is(data); - TERScore* r = new TERScore; - is >> r->stats[TERScore::kINSERTIONS] - >> r->stats[TERScore::kDELETIONS] - >> r->stats[TERScore::kSUBSTITUTIONS] - >> r->stats[TERScore::kSHIFTS] - >> r->stats[TERScore::kREF_WORDCOUNT]; - return ScoreP(r); -} - +#if 0 void TERScore::ScoreDetails(std::string* details) const { char buf[200]; sprintf(buf, "TER = %.2f, %3d|%3d|%3d|%3d (len=%d)", @@ -494,54 +429,43 @@ void TERScore::ScoreDetails(std::string* details) const { stats[kREF_WORDCOUNT]); *details = buf; } +#endif -TERScorer::~TERScorer() { - for (vector::iterator i = impl_.begin(); i != impl_.end(); ++i) - delete *i; -} +} // namespace NewScorer -TERScorer::TERScorer(const vector >& refs) : impl_(refs.size()) { +void TERMetric::ComputeSufficientStatistics(const vector& hyp, + const vector >& refs, + SufficientStats* out) const { + out->fields.resize(kDUMMY_LAST_ENTRY); + float best_score = numeric_limits::max(); + unsigned avg_len = 0; for (int i = 0; i < refs.size(); ++i) - impl_[i] = new TERScorerImpl(refs[i]); -} + avg_len += refs[i].size(); + avg_len /= refs.size(); -ScoreP TERScorer::ScoreCCandidate(const vector& hyp) const { - return ScoreP(); -} - -ScoreP TERScorer::ScoreCandidate(const std::vector& hyp) const { - float best_score = numeric_limits::max(); - TERScore* res = new TERScore; - int avg_len = 0; - for (int i = 0; i < impl_.size(); ++i) - avg_len += impl_[i]->GetRefLength(); - avg_len /= impl_.size(); - for (int i = 0; i < impl_.size(); ++i) { + for (int i = 0; i < refs.size(); ++i) { int subs, ins, dels, shifts; - float score = impl_[i]->Calculate(hyp, &subs, &ins, &dels, &shifts); + NewScorer::TERScorerImpl ter(refs[i]); + float score = ter.Calculate(hyp, &subs, &ins, &dels, &shifts); // cerr << "Component TER cost: " << score << endl; if (score < best_score) { - res->stats[TERScore::kINSERTIONS] = ins; - res->stats[TERScore::kDELETIONS] = dels; - res->stats[TERScore::kSUBSTITUTIONS] = subs; - res->stats[TERScore::kSHIFTS] = shifts; + out->fields[kINSERTIONS] = ins; + out->fields[kDELETIONS] = dels; + out->fields[kSUBSTITUTIONS] = subs; + out->fields[kSHIFTS] = shifts; if (ter_use_average_ref_len) { - res->stats[TERScore::kREF_WORDCOUNT] = avg_len; + out->fields[kREF_WORDCOUNT] = avg_len; } else { - res->stats[TERScore::kREF_WORDCOUNT] = impl_[i]->GetRefLength(); + out->fields[kREF_WORDCOUNT] = refs[i].size(); } best_score = score; } } - return ScoreP(res); } -#endif -void TERMetric::ComputeSufficientStatistics(const vector& hyp, - const vector >& refs, - SufficientStats* out) const { - out->fields.resize(kDUMMY_LAST_ENTRY); +unsigned TERMetric::SufficientStatisticsVectorSize() const { + return kDUMMY_LAST_ENTRY; } float TERMetric::ComputeScore(const SufficientStats& stats) const { diff --git a/mteval/ns_ter.h b/mteval/ns_ter.h index bb90f95e..6c020cfa 100644 --- a/mteval/ns_ter.h +++ b/mteval/ns_ter.h @@ -9,6 +9,7 @@ class TERMetric : public EvaluationMetric { TERMetric() : EvaluationMetric("TER") {} public: + virtual unsigned SufficientStatisticsVectorSize() const; virtual void ComputeSufficientStatistics(const std::vector& hyp, const std::vector >& refs, SufficientStats* out) const; diff --git a/mteval/scorer_test.cc b/mteval/scorer_test.cc index 09da250c..73159557 100644 --- a/mteval/scorer_test.cc +++ b/mteval/scorer_test.cc @@ -205,20 +205,22 @@ TEST_F(ScorerTest, Kernel) { } TEST_F(ScorerTest, NewScoreAPI) { - EvaluationMetric* metric = EvaluationMetric::Instance("IBM_BLEU"); + //EvaluationMetric* metric = EvaluationMetric::Instance("IBM_BLEU"); + //EvaluationMetric* metric = EvaluationMetric::Instance("METEOR"); + EvaluationMetric* metric = EvaluationMetric::Instance("COMB:IBM_BLEU=0.5;TER=-0.5"); boost::shared_ptr e1 = metric->CreateSegmentEvaluator(refs0); boost::shared_ptr e2 = metric->CreateSegmentEvaluator(refs1); SufficientStats stats1; - e1->Evaluate(hyp2, &stats1); + e1->Evaluate(hyp1, &stats1); SufficientStats stats2; - e2->Evaluate(hyp1, &stats2); + e2->Evaluate(hyp2, &stats2); stats1 += stats2; string ss; stats1.Encode(&ss); cerr << "SS: " << ss << endl; cerr << metric->ComputeScore(stats1) << endl; - SufficientStats statse("IBM_BLEU 53 32 18 11 65 63 61 59 65 72"); - cerr << metric->ComputeScore(statse) << endl; + //SufficientStats statse("IBM_BLEU 53 32 18 11 65 63 61 59 65 72"); + //cerr << metric->ComputeScore(statse) << endl; } int main(int argc, char **argv) { -- cgit v1.2.3