diff options
Diffstat (limited to 'mteval')
| -rw-r--r-- | mteval/comb_scorer.cc | 93 | ||||
| -rw-r--r-- | mteval/comb_scorer.h | 11 | ||||
| -rw-r--r-- | mteval/ns.cc | 26 | ||||
| -rw-r--r-- | mteval/ns.h | 3 | ||||
| -rw-r--r-- | mteval/ns_wer.cc | 4 | ||||
| -rw-r--r-- | mteval/scorer.cc | 58 | ||||
| -rw-r--r-- | mteval/scorer.h | 6 | ||||
| -rw-r--r-- | mteval/wer.cc | 16 |
8 files changed, 196 insertions, 21 deletions
diff --git a/mteval/comb_scorer.cc b/mteval/comb_scorer.cc index 9fc37868..63f327ca 100644 --- a/mteval/comb_scorer.cc +++ b/mteval/comb_scorer.cc @@ -95,3 +95,96 @@ ScoreP BLEUTERCombinationScorer::ScoreFromString(const std::string& in) { r->ter = SentenceScorer::CreateScoreFromString(TER, in.substr(1 + bss)); return ScoreP(r); } + + +class BLEUCBLEUCombinationScore : public ScoreBase<BLEUCBLEUCombinationScore> { + friend class BLEUCBLEUCombinationScorer; + public: + ~BLEUCBLEUCombinationScore(); + float ComputePartialScore() const { return 0.0;} + float ComputeScore() const { + return (bleu->ComputeScore() + cbleu->ComputeScore()) / 2.0f; + } + void ScoreDetails(string* details) const { + char buf[160]; + sprintf(buf, "Combi = %.2f, BLEU = %.2f, CBLEU = %.2f", + ComputeScore()*100.0f, bleu->ComputeScore()*100.0f, cbleu->ComputeScore()*100.0f); + *details = buf; + } + void PlusPartialEquals(const Score& rhs, int oracle_e_cover, int oracle_f_cover, int src_len){} + + void PlusEquals(const Score& delta, const float scale) { + bleu->PlusEquals(*static_cast<const BLEUCBLEUCombinationScore&>(delta).bleu, scale); + cbleu->PlusEquals(*static_cast<const BLEUCBLEUCombinationScore&>(delta).cbleu, scale); + } + void PlusEquals(const Score& delta) { + bleu->PlusEquals(*static_cast<const BLEUCBLEUCombinationScore&>(delta).bleu); + cbleu->PlusEquals(*static_cast<const BLEUCBLEUCombinationScore&>(delta).cbleu); + } + + + + ScoreP GetOne() const { + BLEUCBLEUCombinationScore* res = new BLEUCBLEUCombinationScore; + res->bleu = bleu->GetOne(); + res->cbleu = cbleu->GetOne(); + return ScoreP(res); + } + ScoreP GetZero() const { + BLEUCBLEUCombinationScore* res = new BLEUCBLEUCombinationScore; + res->bleu = bleu->GetZero(); + res->cbleu = cbleu->GetZero(); + return ScoreP(res); + } + void Subtract(const Score& rhs, Score* res) const { + bleu->Subtract(*static_cast<const BLEUCBLEUCombinationScore&>(rhs).bleu, + static_cast<BLEUCBLEUCombinationScore*>(res)->bleu.get()); + cbleu->Subtract(*static_cast<const BLEUCBLEUCombinationScore&>(rhs).cbleu, + static_cast<BLEUCBLEUCombinationScore*>(res)->cbleu.get()); + } + void Encode(std::string* out) const { + string bs, ts; + bleu->Encode(&bs); + cbleu->Encode(&ts); + out->clear(); + (*out) += static_cast<char>(bs.size()); + (*out) += bs; + (*out) += ts; + } + bool IsAdditiveIdentity() const { + return bleu->IsAdditiveIdentity() && cbleu->IsAdditiveIdentity(); + } + private: + ScoreP bleu; + ScoreP cbleu; +}; + +BLEUCBLEUCombinationScore::~BLEUCBLEUCombinationScore() { +} + +BLEUCBLEUCombinationScorer::BLEUCBLEUCombinationScorer(const vector<vector<WordID> >& refs) { + bleu_ = SentenceScorer::CreateSentenceScorer(IBM_BLEU, refs); + cbleu_ = SentenceScorer::CreateSentenceScorer(CBLEU, refs); +} + +BLEUCBLEUCombinationScorer::~BLEUCBLEUCombinationScorer() { +} + +ScoreP BLEUCBLEUCombinationScorer::ScoreCCandidate(const vector<WordID>& hyp) const { + return ScoreP(); +} + +ScoreP BLEUCBLEUCombinationScorer::ScoreCandidate(const std::vector<WordID>& hyp) const { + BLEUCBLEUCombinationScore* res = new BLEUCBLEUCombinationScore; + res->bleu = bleu_->ScoreCandidate(hyp); + res->cbleu = cbleu_->ScoreCandidate(hyp); + return ScoreP(res); +} + +ScoreP BLEUCBLEUCombinationScorer::ScoreFromString(const std::string& in) { + int bss = in[0]; + BLEUCBLEUCombinationScore* r = new BLEUCBLEUCombinationScore; + r->bleu = SentenceScorer::CreateScoreFromString(IBM_BLEU, in.substr(1, bss)); + r->cbleu = SentenceScorer::CreateScoreFromString(CBLEU, in.substr(1 + bss)); + return ScoreP(r); +} diff --git a/mteval/comb_scorer.h b/mteval/comb_scorer.h index d17d089d..1e2f0c25 100644 --- a/mteval/comb_scorer.h +++ b/mteval/comb_scorer.h @@ -14,4 +14,15 @@ class BLEUTERCombinationScorer : public SentenceScorer { ScorerP bleu_,ter_; }; +class BLEUCBLEUCombinationScorer : public SentenceScorer { + public: + BLEUCBLEUCombinationScorer(const std::vector<std::vector<WordID> >& refs); + ~BLEUCBLEUCombinationScorer(); + ScoreP ScoreCandidate(const std::vector<WordID>& hyp) const; + ScoreP ScoreCCandidate(const std::vector<WordID>& hyp) const; + static ScoreP ScoreFromString(const std::string& in); + private: + ScorerP bleu_, cbleu_; +}; + #endif diff --git a/mteval/ns.cc b/mteval/ns.cc index 2c8bd806..1d37c436 100644 --- a/mteval/ns.cc +++ b/mteval/ns.cc @@ -65,38 +65,41 @@ string EvaluationMetric::DetailedScore(const SufficientStats& stats) const { } enum BleuType { IBM, Koehn, NIST, QCRI }; -template <unsigned int N = 4u, BleuType BrevityType = IBM> +template <unsigned int N = 4u, BleuType BrevityType = IBM, bool CharBased = false> struct BleuSegmentEvaluator : public SegmentEvaluator { BleuSegmentEvaluator(const vector<vector<WordID> >& refs, const EvaluationMetric* em) : evaluation_metric(em) { - assert(refs.size() > 0); + const vector<vector<WordID> >& local_refs = (CharBased ? Characterize(refs) : refs); + + assert(local_refs.size() > 0); float tot = 0; int smallest = 9999999; - for (vector<vector<WordID> >::const_iterator ci = refs.begin(); - ci != refs.end(); ++ci) { + for (vector<vector<WordID> >::const_iterator ci = local_refs.begin(); + ci != local_refs.end(); ++ci) { lengths_.push_back(ci->size()); tot += lengths_.back(); if (lengths_.back() < smallest) smallest = lengths_.back(); CountRef(*ci); } if (BrevityType == Koehn) - lengths_[0] = tot / refs.size(); + lengths_[0] = tot / local_refs.size(); if (BrevityType == NIST) lengths_[0] = smallest; } void Evaluate(const vector<WordID>& hyp, SufficientStats* out) const { + const vector<WordID>& local_hyp = (CharBased ? Characterize(hyp) : hyp); out->fields.resize(N + N + 2); out->id_ = evaluation_metric->MetricId(); for (unsigned i = 0; i < N+N+2; ++i) out->fields[i] = 0; - ComputeNgramStats(hyp, &out->fields[0], &out->fields[N], true); + ComputeNgramStats(local_hyp, &out->fields[0], &out->fields[N], true); float& hyp_len = out->fields[2*N]; float& ref_len = out->fields[2*N + 1]; - hyp_len = hyp.size(); + hyp_len = local_hyp.size(); ref_len = lengths_[0]; if (lengths_.size() > 1 && (BrevityType == IBM || BrevityType == QCRI)) { float bestd = 2000000; - float hl = hyp.size(); + float hl = local_hyp.size(); float bl = -1; for (vector<float>::const_iterator ci = lengths_.begin(); ci != lengths_.end(); ++ci) { if (fabs(*ci - hl) < bestd) { @@ -187,12 +190,12 @@ struct BleuSegmentEvaluator : public SegmentEvaluator { mutable NGramCountMap ngrams_; }; -template <unsigned int N = 4u, BleuType BrevityType = IBM> +template <unsigned int N = 4u, BleuType BrevityType = IBM, bool CharBased = false> struct BleuMetric : public EvaluationMetric { BleuMetric() : EvaluationMetric(BrevityType == IBM ? "IBM_BLEU" : (BrevityType == Koehn ? "KOEHN_BLEU" : (BrevityType == NIST ? "NIST_BLEU" : "QCRI_BLEU"))) {} unsigned SufficientStatisticsVectorSize() const { return N*2 + 2; } boost::shared_ptr<SegmentEvaluator> CreateSegmentEvaluator(const vector<vector<WordID> >& refs) const { - return boost::shared_ptr<SegmentEvaluator>(new BleuSegmentEvaluator<N,BrevityType>(refs, this)); + return boost::shared_ptr<SegmentEvaluator>(new BleuSegmentEvaluator<N,BrevityType, CharBased>(refs, this)); } float ComputeBreakdown(const SufficientStats& stats, float* bp, vector<float>* out) const { if (out) { out->clear(); } @@ -290,6 +293,8 @@ EvaluationMetric* EvaluationMetric::Instance(const string& imetric_id) { m = new CERMetric; } else if (metric_id == "WER") { m = new WERMetric; + } else if (metric_id == "CBLEU") { + return new BleuMetric<5, IBM, true>; } else { cerr << "Implement please: " << metric_id << endl; abort(); @@ -322,4 +327,3 @@ void SufficientStats::Encode(string* out) const { os << ' ' << fields[i]; *out = os.str(); } - diff --git a/mteval/ns.h b/mteval/ns.h index f6329b65..16edfdf0 100644 --- a/mteval/ns.h +++ b/mteval/ns.h @@ -8,6 +8,9 @@ #include "wordid.h" #include <iostream> +std::vector<WordID> Characterize(const std::vector<WordID>& reference); +std::vector<std::vector<WordID> > Characterize(const std::vector<std::vector<WordID> >& references); + class SufficientStats { public: SufficientStats() : id_() {} diff --git a/mteval/ns_wer.cc b/mteval/ns_wer.cc index f9b2bbbb..057ad49e 100644 --- a/mteval/ns_wer.cc +++ b/mteval/ns_wer.cc @@ -18,10 +18,10 @@ void WERMetric::ComputeSufficientStatistics(const std::vector<WordID>& hyp, const std::vector<std::vector<WordID> >& refs, SufficientStats* out) const { out->fields.resize(kNUMFIELDS); - float best_score = hyp.size(); + float best_score = 0; for (size_t i = 0; i < refs.size(); ++i) { float score = cdec::LevenshteinDistance(hyp, refs[i]); - if (score < best_score) { + if (score < best_score || i == 0) { out->fields[kEDITDISTANCE] = score; out->fields[kCHARCOUNT] = refs[i].size(); best_score = score; diff --git a/mteval/scorer.cc b/mteval/scorer.cc index 4c05dbd8..71e05e9c 100644 --- a/mteval/scorer.cc +++ b/mteval/scorer.cc @@ -49,12 +49,17 @@ ScoreType ScoreTypeFromString(const string& st) { return METEOR; if (sl == "wer") return WER; + if (sl == "cbleu") + return CBLEU; + if (sl == "bleu_cbleu") + return BLEU_plus_CBLEU_over_2; cerr << "Don't understand score type '" << st << "', defaulting to ibm_bleu.\n"; + assert (false); return IBM_BLEU; } static char const* score_names[]={ - "IBM_BLEU", "NIST_BLEU", "Koehn_BLEU", "TER", "BLEU_minus_TER_over_2", "SER", "AER", "IBM_BLEU_3", "METEOR", "WER" + "IBM_BLEU", "NIST_BLEU", "Koehn_BLEU", "TER", "BLEU_minus_TER_over_2", "SER", "AER", "IBM_BLEU_3", "METEOR", "WER", "CBLEU", "BLEU_plus_CBLEU_over_2" }; std::string StringFromScoreType(ScoreType st) { @@ -291,6 +296,21 @@ ScoreP BLEUScorerBase::ScoreFromString(const string& in) { return ScoreP(r); } +class CBLEUScorer : public BLEUScorerBase { + public: + CBLEUScorer(const vector<vector<WordID> >& references, + int n=5) : BLEUScorerBase(Characterize(references), n), lengths_(references.size()) { + for (unsigned i=0; i < references.size(); ++i) + lengths_[i] = Characterize(references[i]).size(); + } + + float ComputeRefLength(const vector<WordID>& hyp) const { + return 1000; + } + private: + vector<int> lengths_; +}; + class IBM_BLEUScorer : public BLEUScorerBase { public: IBM_BLEUScorer(const vector<vector<WordID> >& references, @@ -362,8 +382,10 @@ ScorerP SentenceScorer::CreateSentenceScorer(const ScoreType type, case TER: r = new TERScorer(refs);break; case SER: r = new SERScorer(refs);break; case BLEU_minus_TER_over_2: r = new BLEUTERCombinationScorer(refs);break; + case BLEU_plus_CBLEU_over_2: r = new BLEUCBLEUCombinationScorer(refs); break; case METEOR: r = new ExternalSentenceScorer(ScoreServerManager::Instance("meteor"), refs); break; case WER: r = new WERScorer(refs);break; + case CBLEU: r = new CBLEUScorer(refs, 5); break; default: assert(!"Not implemented!"); } @@ -410,6 +432,10 @@ ScoreP SentenceScorer::CreateScoreFromString(const ScoreType type, const string& return ExternalSentenceScorer::ScoreFromString(ScoreServerManager::Instance("meteor"), in); case WER: return WERScorer::ScoreFromString(in); + case CBLEU: + return CBLEUScorer::ScoreFromString(in); + case BLEU_plus_CBLEU_over_2: + return BLEUCBLEUCombinationScorer::ScoreFromString(in); default: assert(!"Not implemented!"); } @@ -685,3 +711,33 @@ void DocStreamScorer::update(const std::string& ref) { TD::ConvertSentence(ref, &refs[0]); scorer = ScorerP(SentenceScorer::CreateSentenceScorer(type, refs, src_line)); } + +vector<WordID> Characterize(const vector<WordID>& reference) { + vector<WordID> r; + string space = " "; + for (WordID word_id: reference) { + string word = TD::Convert(word_id); + unsigned i = 0; + while (i < word.length()) { + unsigned char_length = UTF8Len(word[i]); + string c = word.substr(i, char_length); + i += char_length; + r.push_back(TD::Convert(c)); + } + r.push_back(TD::Convert(space)); + } + + // Remove the last space + if (r.size() > 0) { + r.pop_back(); + } + return r; +} + +vector<vector<WordID>> Characterize(const vector<vector<WordID> >& references) { + vector<vector<WordID> > r; + for (const vector<WordID>& reference : references) { + r.push_back(Characterize(reference)); + } + return r; +} diff --git a/mteval/scorer.h b/mteval/scorer.h index a411f14b..e7de0118 100644 --- a/mteval/scorer.h +++ b/mteval/scorer.h @@ -17,10 +17,14 @@ class ErrorSurface; class Hypergraph; // needed for alignment //TODO: BLEU N (N separate arg, not part of enum)? -enum ScoreType { IBM_BLEU, NIST_BLEU, Koehn_BLEU, TER, BLEU_minus_TER_over_2, SER, AER, IBM_BLEU_3, METEOR, WER }; +enum ScoreType { IBM_BLEU, NIST_BLEU, Koehn_BLEU, TER, BLEU_minus_TER_over_2, SER, AER, IBM_BLEU_3, METEOR, WER, CBLEU, BLEU_plus_CBLEU_over_2 }; ScoreType ScoreTypeFromString(const std::string& st); std::string StringFromScoreType(ScoreType st); +std::vector<WordID> Characterize(const std::vector<WordID>& reference); +std::vector<std::vector<WordID> > Characterize(const std::vector<std::vector<WordID> >& references); + + class Score : public boost::intrusive_refcount<Score> { public: virtual ~Score(); diff --git a/mteval/wer.cc b/mteval/wer.cc index c806b3be..b8cfd3d8 100644 --- a/mteval/wer.cc +++ b/mteval/wer.cc @@ -31,16 +31,17 @@ class WERScore : public ScoreBase<WERScore> { WERScore() : stats(0,kDUMMY_LAST_ENTRY) {} float ComputePartialScore() const { return 0.0;} float ComputeScore() const { + if (static_cast<float>(stats[kCHARCOUNT]) < 0.5) + return 0; return static_cast<float>(stats[kEDITDISTANCE]) / static_cast<float>(stats[kCHARCOUNT]); } void ScoreDetails(string* details) const; void PlusPartialEquals(const Score& rhs, int oracle_e_cover, int oracle_f_cover, int src_len){} void PlusEquals(const Score& delta, const float scale) { - if (scale==1) - stats += static_cast<const WERScore&>(delta).stats; - if (scale==-1) - stats -= static_cast<const WERScore&>(delta).stats; - throw std::runtime_error("WERScore::PlusEquals with scale != +-1"); + const WERScore& delta_stats = static_cast<const WERScore&>(delta); + for (unsigned i = 0; i < kDUMMY_LAST_ENTRY; ++i) { + stats[i] += scale * static_cast<float>(delta_stats.stats[i]); + } } void PlusEquals(const Score& delta) { stats += static_cast<const WERScore&>(delta).stats; @@ -88,7 +89,7 @@ void WERScore::ScoreDetails(std::string* details) const { } WERScorer::~WERScorer() {} -WERScorer::WERScorer(const vector<vector<WordID> >& refs) {} +WERScorer::WERScorer(const vector<vector<WordID> >& refs) {this->refs = refs;} ScoreP WERScorer::ScoreCCandidate(const vector<WordID>& hyp) const { return ScoreP(); @@ -97,6 +98,9 @@ ScoreP WERScorer::ScoreCCandidate(const vector<WordID>& hyp) const { float WERScorer::Calculate(const std::vector<WordID>& hyp, const Sentence& ref, int& edits, int& char_count) const { edits = cdec::LevenshteinDistance(hyp, ref); char_count = ref.size(); + if (char_count == 0) { + return 0; + } return static_cast<float>(edits) / static_cast<float>(char_count); } |