summaryrefslogtreecommitdiff
path: root/mteval
diff options
context:
space:
mode:
Diffstat (limited to 'mteval')
-rw-r--r--mteval/comb_scorer.cc93
-rw-r--r--mteval/comb_scorer.h11
-rw-r--r--mteval/ns.cc26
-rw-r--r--mteval/ns.h3
-rw-r--r--mteval/scorer.cc58
-rw-r--r--mteval/scorer.h6
-rw-r--r--mteval/wer.cc16
7 files changed, 194 insertions, 19 deletions
diff --git a/mteval/comb_scorer.cc b/mteval/comb_scorer.cc
index 9fc37868..63f327ca 100644
--- a/mteval/comb_scorer.cc
+++ b/mteval/comb_scorer.cc
@@ -95,3 +95,96 @@ ScoreP BLEUTERCombinationScorer::ScoreFromString(const std::string& in) {
r->ter = SentenceScorer::CreateScoreFromString(TER, in.substr(1 + bss));
return ScoreP(r);
}
+
+
+class BLEUCBLEUCombinationScore : public ScoreBase<BLEUCBLEUCombinationScore> {
+ friend class BLEUCBLEUCombinationScorer;
+ public:
+ ~BLEUCBLEUCombinationScore();
+ float ComputePartialScore() const { return 0.0;}
+ float ComputeScore() const {
+ return (bleu->ComputeScore() + cbleu->ComputeScore()) / 2.0f;
+ }
+ void ScoreDetails(string* details) const {
+ char buf[160];
+ sprintf(buf, "Combi = %.2f, BLEU = %.2f, CBLEU = %.2f",
+ ComputeScore()*100.0f, bleu->ComputeScore()*100.0f, cbleu->ComputeScore()*100.0f);
+ *details = buf;
+ }
+ void PlusPartialEquals(const Score& rhs, int oracle_e_cover, int oracle_f_cover, int src_len){}
+
+ void PlusEquals(const Score& delta, const float scale) {
+ bleu->PlusEquals(*static_cast<const BLEUCBLEUCombinationScore&>(delta).bleu, scale);
+ cbleu->PlusEquals(*static_cast<const BLEUCBLEUCombinationScore&>(delta).cbleu, scale);
+ }
+ void PlusEquals(const Score& delta) {
+ bleu->PlusEquals(*static_cast<const BLEUCBLEUCombinationScore&>(delta).bleu);
+ cbleu->PlusEquals(*static_cast<const BLEUCBLEUCombinationScore&>(delta).cbleu);
+ }
+
+
+
+ ScoreP GetOne() const {
+ BLEUCBLEUCombinationScore* res = new BLEUCBLEUCombinationScore;
+ res->bleu = bleu->GetOne();
+ res->cbleu = cbleu->GetOne();
+ return ScoreP(res);
+ }
+ ScoreP GetZero() const {
+ BLEUCBLEUCombinationScore* res = new BLEUCBLEUCombinationScore;
+ res->bleu = bleu->GetZero();
+ res->cbleu = cbleu->GetZero();
+ return ScoreP(res);
+ }
+ void Subtract(const Score& rhs, Score* res) const {
+ bleu->Subtract(*static_cast<const BLEUCBLEUCombinationScore&>(rhs).bleu,
+ static_cast<BLEUCBLEUCombinationScore*>(res)->bleu.get());
+ cbleu->Subtract(*static_cast<const BLEUCBLEUCombinationScore&>(rhs).cbleu,
+ static_cast<BLEUCBLEUCombinationScore*>(res)->cbleu.get());
+ }
+ void Encode(std::string* out) const {
+ string bs, ts;
+ bleu->Encode(&bs);
+ cbleu->Encode(&ts);
+ out->clear();
+ (*out) += static_cast<char>(bs.size());
+ (*out) += bs;
+ (*out) += ts;
+ }
+ bool IsAdditiveIdentity() const {
+ return bleu->IsAdditiveIdentity() && cbleu->IsAdditiveIdentity();
+ }
+ private:
+ ScoreP bleu;
+ ScoreP cbleu;
+};
+
+BLEUCBLEUCombinationScore::~BLEUCBLEUCombinationScore() {
+}
+
+BLEUCBLEUCombinationScorer::BLEUCBLEUCombinationScorer(const vector<vector<WordID> >& refs) {
+ bleu_ = SentenceScorer::CreateSentenceScorer(IBM_BLEU, refs);
+ cbleu_ = SentenceScorer::CreateSentenceScorer(CBLEU, refs);
+}
+
+BLEUCBLEUCombinationScorer::~BLEUCBLEUCombinationScorer() {
+}
+
+ScoreP BLEUCBLEUCombinationScorer::ScoreCCandidate(const vector<WordID>& hyp) const {
+ return ScoreP();
+}
+
+ScoreP BLEUCBLEUCombinationScorer::ScoreCandidate(const std::vector<WordID>& hyp) const {
+ BLEUCBLEUCombinationScore* res = new BLEUCBLEUCombinationScore;
+ res->bleu = bleu_->ScoreCandidate(hyp);
+ res->cbleu = cbleu_->ScoreCandidate(hyp);
+ return ScoreP(res);
+}
+
+ScoreP BLEUCBLEUCombinationScorer::ScoreFromString(const std::string& in) {
+ int bss = in[0];
+ BLEUCBLEUCombinationScore* r = new BLEUCBLEUCombinationScore;
+ r->bleu = SentenceScorer::CreateScoreFromString(IBM_BLEU, in.substr(1, bss));
+ r->cbleu = SentenceScorer::CreateScoreFromString(CBLEU, in.substr(1 + bss));
+ return ScoreP(r);
+}
diff --git a/mteval/comb_scorer.h b/mteval/comb_scorer.h
index d17d089d..1e2f0c25 100644
--- a/mteval/comb_scorer.h
+++ b/mteval/comb_scorer.h
@@ -14,4 +14,15 @@ class BLEUTERCombinationScorer : public SentenceScorer {
ScorerP bleu_,ter_;
};
+class BLEUCBLEUCombinationScorer : public SentenceScorer {
+ public:
+ BLEUCBLEUCombinationScorer(const std::vector<std::vector<WordID> >& refs);
+ ~BLEUCBLEUCombinationScorer();
+ ScoreP ScoreCandidate(const std::vector<WordID>& hyp) const;
+ ScoreP ScoreCCandidate(const std::vector<WordID>& hyp) const;
+ static ScoreP ScoreFromString(const std::string& in);
+ private:
+ ScorerP bleu_, cbleu_;
+};
+
#endif
diff --git a/mteval/ns.cc b/mteval/ns.cc
index 2c8bd806..1d37c436 100644
--- a/mteval/ns.cc
+++ b/mteval/ns.cc
@@ -65,38 +65,41 @@ string EvaluationMetric::DetailedScore(const SufficientStats& stats) const {
}
enum BleuType { IBM, Koehn, NIST, QCRI };
-template <unsigned int N = 4u, BleuType BrevityType = IBM>
+template <unsigned int N = 4u, BleuType BrevityType = IBM, bool CharBased = false>
struct BleuSegmentEvaluator : public SegmentEvaluator {
BleuSegmentEvaluator(const vector<vector<WordID> >& refs, const EvaluationMetric* em) : evaluation_metric(em) {
- assert(refs.size() > 0);
+ const vector<vector<WordID> >& local_refs = (CharBased ? Characterize(refs) : refs);
+
+ assert(local_refs.size() > 0);
float tot = 0;
int smallest = 9999999;
- for (vector<vector<WordID> >::const_iterator ci = refs.begin();
- ci != refs.end(); ++ci) {
+ for (vector<vector<WordID> >::const_iterator ci = local_refs.begin();
+ ci != local_refs.end(); ++ci) {
lengths_.push_back(ci->size());
tot += lengths_.back();
if (lengths_.back() < smallest) smallest = lengths_.back();
CountRef(*ci);
}
if (BrevityType == Koehn)
- lengths_[0] = tot / refs.size();
+ lengths_[0] = tot / local_refs.size();
if (BrevityType == NIST)
lengths_[0] = smallest;
}
void Evaluate(const vector<WordID>& hyp, SufficientStats* out) const {
+ const vector<WordID>& local_hyp = (CharBased ? Characterize(hyp) : hyp);
out->fields.resize(N + N + 2);
out->id_ = evaluation_metric->MetricId();
for (unsigned i = 0; i < N+N+2; ++i) out->fields[i] = 0;
- ComputeNgramStats(hyp, &out->fields[0], &out->fields[N], true);
+ ComputeNgramStats(local_hyp, &out->fields[0], &out->fields[N], true);
float& hyp_len = out->fields[2*N];
float& ref_len = out->fields[2*N + 1];
- hyp_len = hyp.size();
+ hyp_len = local_hyp.size();
ref_len = lengths_[0];
if (lengths_.size() > 1 && (BrevityType == IBM || BrevityType == QCRI)) {
float bestd = 2000000;
- float hl = hyp.size();
+ float hl = local_hyp.size();
float bl = -1;
for (vector<float>::const_iterator ci = lengths_.begin(); ci != lengths_.end(); ++ci) {
if (fabs(*ci - hl) < bestd) {
@@ -187,12 +190,12 @@ struct BleuSegmentEvaluator : public SegmentEvaluator {
mutable NGramCountMap ngrams_;
};
-template <unsigned int N = 4u, BleuType BrevityType = IBM>
+template <unsigned int N = 4u, BleuType BrevityType = IBM, bool CharBased = false>
struct BleuMetric : public EvaluationMetric {
BleuMetric() : EvaluationMetric(BrevityType == IBM ? "IBM_BLEU" : (BrevityType == Koehn ? "KOEHN_BLEU" : (BrevityType == NIST ? "NIST_BLEU" : "QCRI_BLEU"))) {}
unsigned SufficientStatisticsVectorSize() const { return N*2 + 2; }
boost::shared_ptr<SegmentEvaluator> CreateSegmentEvaluator(const vector<vector<WordID> >& refs) const {
- return boost::shared_ptr<SegmentEvaluator>(new BleuSegmentEvaluator<N,BrevityType>(refs, this));
+ return boost::shared_ptr<SegmentEvaluator>(new BleuSegmentEvaluator<N,BrevityType, CharBased>(refs, this));
}
float ComputeBreakdown(const SufficientStats& stats, float* bp, vector<float>* out) const {
if (out) { out->clear(); }
@@ -290,6 +293,8 @@ EvaluationMetric* EvaluationMetric::Instance(const string& imetric_id) {
m = new CERMetric;
} else if (metric_id == "WER") {
m = new WERMetric;
+ } else if (metric_id == "CBLEU") {
+ return new BleuMetric<5, IBM, true>;
} else {
cerr << "Implement please: " << metric_id << endl;
abort();
@@ -322,4 +327,3 @@ void SufficientStats::Encode(string* out) const {
os << ' ' << fields[i];
*out = os.str();
}
-
diff --git a/mteval/ns.h b/mteval/ns.h
index f6329b65..16edfdf0 100644
--- a/mteval/ns.h
+++ b/mteval/ns.h
@@ -8,6 +8,9 @@
#include "wordid.h"
#include <iostream>
+std::vector<WordID> Characterize(const std::vector<WordID>& reference);
+std::vector<std::vector<WordID> > Characterize(const std::vector<std::vector<WordID> >& references);
+
class SufficientStats {
public:
SufficientStats() : id_() {}
diff --git a/mteval/scorer.cc b/mteval/scorer.cc
index 4c05dbd8..71e05e9c 100644
--- a/mteval/scorer.cc
+++ b/mteval/scorer.cc
@@ -49,12 +49,17 @@ ScoreType ScoreTypeFromString(const string& st) {
return METEOR;
if (sl == "wer")
return WER;
+ if (sl == "cbleu")
+ return CBLEU;
+ if (sl == "bleu_cbleu")
+ return BLEU_plus_CBLEU_over_2;
cerr << "Don't understand score type '" << st << "', defaulting to ibm_bleu.\n";
+ assert (false);
return IBM_BLEU;
}
static char const* score_names[]={
- "IBM_BLEU", "NIST_BLEU", "Koehn_BLEU", "TER", "BLEU_minus_TER_over_2", "SER", "AER", "IBM_BLEU_3", "METEOR", "WER"
+ "IBM_BLEU", "NIST_BLEU", "Koehn_BLEU", "TER", "BLEU_minus_TER_over_2", "SER", "AER", "IBM_BLEU_3", "METEOR", "WER", "CBLEU", "BLEU_plus_CBLEU_over_2"
};
std::string StringFromScoreType(ScoreType st) {
@@ -291,6 +296,21 @@ ScoreP BLEUScorerBase::ScoreFromString(const string& in) {
return ScoreP(r);
}
+class CBLEUScorer : public BLEUScorerBase {
+ public:
+ CBLEUScorer(const vector<vector<WordID> >& references,
+ int n=5) : BLEUScorerBase(Characterize(references), n), lengths_(references.size()) {
+ for (unsigned i=0; i < references.size(); ++i)
+ lengths_[i] = Characterize(references[i]).size();
+ }
+
+ float ComputeRefLength(const vector<WordID>& hyp) const {
+ return 1000;
+ }
+ private:
+ vector<int> lengths_;
+};
+
class IBM_BLEUScorer : public BLEUScorerBase {
public:
IBM_BLEUScorer(const vector<vector<WordID> >& references,
@@ -362,8 +382,10 @@ ScorerP SentenceScorer::CreateSentenceScorer(const ScoreType type,
case TER: r = new TERScorer(refs);break;
case SER: r = new SERScorer(refs);break;
case BLEU_minus_TER_over_2: r = new BLEUTERCombinationScorer(refs);break;
+ case BLEU_plus_CBLEU_over_2: r = new BLEUCBLEUCombinationScorer(refs); break;
case METEOR: r = new ExternalSentenceScorer(ScoreServerManager::Instance("meteor"), refs); break;
case WER: r = new WERScorer(refs);break;
+ case CBLEU: r = new CBLEUScorer(refs, 5); break;
default:
assert(!"Not implemented!");
}
@@ -410,6 +432,10 @@ ScoreP SentenceScorer::CreateScoreFromString(const ScoreType type, const string&
return ExternalSentenceScorer::ScoreFromString(ScoreServerManager::Instance("meteor"), in);
case WER:
return WERScorer::ScoreFromString(in);
+ case CBLEU:
+ return CBLEUScorer::ScoreFromString(in);
+ case BLEU_plus_CBLEU_over_2:
+ return BLEUCBLEUCombinationScorer::ScoreFromString(in);
default:
assert(!"Not implemented!");
}
@@ -685,3 +711,33 @@ void DocStreamScorer::update(const std::string& ref) {
TD::ConvertSentence(ref, &refs[0]);
scorer = ScorerP(SentenceScorer::CreateSentenceScorer(type, refs, src_line));
}
+
+vector<WordID> Characterize(const vector<WordID>& reference) {
+ vector<WordID> r;
+ string space = " ";
+ for (WordID word_id: reference) {
+ string word = TD::Convert(word_id);
+ unsigned i = 0;
+ while (i < word.length()) {
+ unsigned char_length = UTF8Len(word[i]);
+ string c = word.substr(i, char_length);
+ i += char_length;
+ r.push_back(TD::Convert(c));
+ }
+ r.push_back(TD::Convert(space));
+ }
+
+ // Remove the last space
+ if (r.size() > 0) {
+ r.pop_back();
+ }
+ return r;
+}
+
+vector<vector<WordID>> Characterize(const vector<vector<WordID> >& references) {
+ vector<vector<WordID> > r;
+ for (const vector<WordID>& reference : references) {
+ r.push_back(Characterize(reference));
+ }
+ return r;
+}
diff --git a/mteval/scorer.h b/mteval/scorer.h
index a411f14b..e7de0118 100644
--- a/mteval/scorer.h
+++ b/mteval/scorer.h
@@ -17,10 +17,14 @@ class ErrorSurface;
class Hypergraph; // needed for alignment
//TODO: BLEU N (N separate arg, not part of enum)?
-enum ScoreType { IBM_BLEU, NIST_BLEU, Koehn_BLEU, TER, BLEU_minus_TER_over_2, SER, AER, IBM_BLEU_3, METEOR, WER };
+enum ScoreType { IBM_BLEU, NIST_BLEU, Koehn_BLEU, TER, BLEU_minus_TER_over_2, SER, AER, IBM_BLEU_3, METEOR, WER, CBLEU, BLEU_plus_CBLEU_over_2 };
ScoreType ScoreTypeFromString(const std::string& st);
std::string StringFromScoreType(ScoreType st);
+std::vector<WordID> Characterize(const std::vector<WordID>& reference);
+std::vector<std::vector<WordID> > Characterize(const std::vector<std::vector<WordID> >& references);
+
+
class Score : public boost::intrusive_refcount<Score> {
public:
virtual ~Score();
diff --git a/mteval/wer.cc b/mteval/wer.cc
index c806b3be..b8cfd3d8 100644
--- a/mteval/wer.cc
+++ b/mteval/wer.cc
@@ -31,16 +31,17 @@ class WERScore : public ScoreBase<WERScore> {
WERScore() : stats(0,kDUMMY_LAST_ENTRY) {}
float ComputePartialScore() const { return 0.0;}
float ComputeScore() const {
+ if (static_cast<float>(stats[kCHARCOUNT]) < 0.5)
+ return 0;
return static_cast<float>(stats[kEDITDISTANCE]) / static_cast<float>(stats[kCHARCOUNT]);
}
void ScoreDetails(string* details) const;
void PlusPartialEquals(const Score& rhs, int oracle_e_cover, int oracle_f_cover, int src_len){}
void PlusEquals(const Score& delta, const float scale) {
- if (scale==1)
- stats += static_cast<const WERScore&>(delta).stats;
- if (scale==-1)
- stats -= static_cast<const WERScore&>(delta).stats;
- throw std::runtime_error("WERScore::PlusEquals with scale != +-1");
+ const WERScore& delta_stats = static_cast<const WERScore&>(delta);
+ for (unsigned i = 0; i < kDUMMY_LAST_ENTRY; ++i) {
+ stats[i] += scale * static_cast<float>(delta_stats.stats[i]);
+ }
}
void PlusEquals(const Score& delta) {
stats += static_cast<const WERScore&>(delta).stats;
@@ -88,7 +89,7 @@ void WERScore::ScoreDetails(std::string* details) const {
}
WERScorer::~WERScorer() {}
-WERScorer::WERScorer(const vector<vector<WordID> >& refs) {}
+WERScorer::WERScorer(const vector<vector<WordID> >& refs) {this->refs = refs;}
ScoreP WERScorer::ScoreCCandidate(const vector<WordID>& hyp) const {
return ScoreP();
@@ -97,6 +98,9 @@ ScoreP WERScorer::ScoreCCandidate(const vector<WordID>& hyp) const {
float WERScorer::Calculate(const std::vector<WordID>& hyp, const Sentence& ref, int& edits, int& char_count) const {
edits = cdec::LevenshteinDistance(hyp, ref);
char_count = ref.size();
+ if (char_count == 0) {
+ return 0;
+ }
return static_cast<float>(edits) / static_cast<float>(char_count);
}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-11 07:37:58 +0000