summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--dtrain/dtrain.cc41
-rw-r--r--dtrain/kbestget.h20
-rw-r--r--dtrain/ksampler.h8
-rw-r--r--dtrain/pairsampling.h4
-rw-r--r--dtrain/score.cc132
-rw-r--r--dtrain/score.h120
-rw-r--r--dtrain/test/example/cdec.ini2
-rw-r--r--dtrain/test/example/dtrain.ini4
-rw-r--r--dtrain/test/example/weights.gzbin248 -> 12001 bytes
9 files changed, 213 insertions, 118 deletions
diff --git a/dtrain/dtrain.cc b/dtrain/dtrain.cc
index 44090242..35e6cc46 100644
--- a/dtrain/dtrain.cc
+++ b/dtrain/dtrain.cc
@@ -106,7 +106,7 @@ main(int argc, char** argv)
// scoring metric/scorer
string scorer_str = cfg["scorer"].as<string>();
- score_t (*scorer)(NgramCounts&, const unsigned, const unsigned, unsigned, vector<score_t>);
+ /*score_t (*scorer)(NgramCounts&, const unsigned, const unsigned, unsigned, vector<score_t>);
if (scorer_str == "bleu") {
scorer = &bleu;
} else if (scorer_str == "stupid_bleu") {
@@ -122,9 +122,11 @@ main(int argc, char** argv)
NgramCounts global_counts(N); // counts for 1 best translations
unsigned global_hyp_len = 0; // sum hypothesis lengths
unsigned global_ref_len = 0; // sum reference lengths
- // ^^^ global_* for approx_bleu
+ // ^^^ global_* for approx_bleu*/
vector<score_t> bleu_weights; // we leave this empty -> 1/N
- if (!quiet) cerr << setw(26) << "scorer '" << scorer_str << "'" << endl << endl;
+ //if (!quiet) cerr << setw(26) << "scorer '" << scorer_str << "'" << endl << endl;
+ StupidBleuScorer scorer;
+ scorer.Init(N, bleu_weights);
// init weights
Weights weights;
@@ -240,7 +242,6 @@ main(int argc, char** argv)
// handling input
strsplit(in, in_split, '\t', 4);
// getting reference
- ref_ids.clear();
vector<string> ref_tok;
strsplit(in_split[2], ref_tok, ' ');
register_and_convert(ref_tok, ref_ids);
@@ -279,43 +280,23 @@ main(int argc, char** argv)
// (local) scoring
if (t > 0) ref_ids = ref_ids_buf[ii];
- score_t score = 0.;
for (unsigned i = 0; i < samples->size(); i++) {
- NgramCounts counts = make_ngram_counts(ref_ids, (*samples)[i].w, N);
- if (scorer_str == "approx_bleu") {
- unsigned hyp_len = 0;
- if (i == 0) { // 'context of 1best translations'
- global_counts += counts;
- global_hyp_len += (*samples)[i].w.size();
- global_ref_len += ref_ids.size();
- counts.reset();
- } else {
- hyp_len = (*samples)[i].w.size();
- }
- NgramCounts _c = global_counts + counts;
- score = .9 * scorer(_c,
- global_ref_len,
- global_hyp_len + hyp_len, N, bleu_weights);
- } else {
- score = scorer(counts,
- ref_ids.size(),
- (*samples)[i].w.size(), N, bleu_weights);
- }
-
- (*samples)[i].score = (score);
+ //cout << ii << " " << i << endl;
+ cout << _p9;
+ (*samples)[i].score = scorer.Score((*samples)[i], ref_ids, ii);
if (i == 0) {
- score_sum += score;
+ score_sum += (*samples)[i].score;
model_sum += (*samples)[i].model;
}
if (verbose) {
if (i == 0) cerr << "'" << TD::GetString(ref_ids) << "' [ref]" << endl;
cerr << _p5 << _np << "[hyp " << i << "] " << "'" << TD::GetString((*samples)[i].w) << "'";
- cerr << " [SCORE=" << score << ",model="<< (*samples)[i].model << "]" << endl;
+ cerr << " [SCORE=" << (*samples)[i].score << ",model="<< (*samples)[i].model << "]" << endl;
cerr << (*samples)[i].f << endl;
}
- } // sample/scoring loop
+ }
if (verbose) cerr << endl;
diff --git a/dtrain/kbestget.h b/dtrain/kbestget.h
index 935998a0..2a2c6073 100644
--- a/dtrain/kbestget.h
+++ b/dtrain/kbestget.h
@@ -1,11 +1,24 @@
#ifndef _DTRAIN_KBESTGET_H_
#define _DTRAIN_KBESTGET_H_
-#include "kbest.h"
+
+#include <vector>
+#include <string>
+
+using namespace std;
+
+#include "kbest.h" // cdec
+#include "verbose.h"
+#include "viterbi.h"
+#include "ff_register.h"
+#include "decoder.h"
+#include "weights.h"
namespace dtrain
{
+typedef double score_t; // float
+
struct ScoredHyp
{
@@ -13,11 +26,12 @@ struct ScoredHyp
SparseVector<double> f;
score_t model;
score_t score;
+ unsigned rank;
};
struct HypSampler : public DecoderObserver
{
- virtual vector<ScoredHyp>* GetSamples() {}
+ virtual vector<ScoredHyp>* GetSamples()=0;
};
struct KBestGetter : public HypSampler
@@ -62,6 +76,7 @@ struct KBestGetter : public HypSampler
h.w = d->yield;
h.f = d->feature_values;
h.model = log(d->score);
+ h.rank = i;
s_.push_back(h);
}
}
@@ -79,6 +94,7 @@ struct KBestGetter : public HypSampler
h.w = d->yield;
h.f = d->feature_values;
h.model = log(d->score);
+ h.rank = i;
s_.push_back(h);
}
}
diff --git a/dtrain/ksampler.h b/dtrain/ksampler.h
index 17b0ba56..767dc42e 100644
--- a/dtrain/ksampler.h
+++ b/dtrain/ksampler.h
@@ -1,7 +1,13 @@
#ifndef _DTRAIN_KSAMPLER_H_
#define _DTRAIN_KSAMPLER_H_
+#include "kbestget.h"
#include "hgsampler.h"
+#include <vector>
+#include <string>
+
+using namespace std;
+
#include "kbest.h" // cdec
#include "sampler.h"
@@ -14,6 +20,7 @@ struct KSampler : public HypSampler
const unsigned k_;
vector<ScoredHyp> s_;
MT19937* prng_;
+ score_t (*scorer)(NgramCounts&, const unsigned, const unsigned, unsigned, vector<score_t>);
explicit KSampler(const unsigned k, MT19937* prng) :
k_(k), prng_(prng) {}
@@ -35,6 +42,7 @@ struct KSampler : public HypSampler
h.w = samples[i].words;
h.f = samples[i].fmap;
h.model = log(samples[i].model_score);
+ h.rank = i;
s_.push_back(h);
}
}
diff --git a/dtrain/pairsampling.h b/dtrain/pairsampling.h
index 9546a945..4a6d93d1 100644
--- a/dtrain/pairsampling.h
+++ b/dtrain/pairsampling.h
@@ -2,6 +2,10 @@
#define _DTRAIN_PAIRSAMPLING_H_
#include "kbestget.h"
+#include "score.h"
+#include <vector>
+#include <string>
+using namespace std;
#include "sampler.h" // cdec, MT19937
namespace dtrain
diff --git a/dtrain/score.cc b/dtrain/score.cc
index 52644250..9b22508b 100644
--- a/dtrain/score.cc
+++ b/dtrain/score.cc
@@ -4,40 +4,6 @@ namespace dtrain
{
-Ngrams
-make_ngrams(vector<WordID>& s, unsigned N)
-{
- Ngrams ngrams;
- vector<WordID> ng;
- for (size_t i = 0; i < s.size(); i++) {
- ng.clear();
- for (unsigned j = i; j < min(i+N, s.size()); j++) {
- ng.push_back(s[j]);
- ngrams[ng]++;
- }
- }
- return ngrams;
-}
-
-NgramCounts
-make_ngram_counts(vector<WordID> hyp, vector<WordID> ref, unsigned N)
-{
- Ngrams hyp_ngrams = make_ngrams(hyp, N);
- Ngrams ref_ngrams = make_ngrams(ref, N);
- NgramCounts counts(N);
- Ngrams::iterator it;
- Ngrams::iterator ti;
- for (it = hyp_ngrams.begin(); it != hyp_ngrams.end(); it++) {
- ti = ref_ngrams.find(it->first);
- if (ti != ref_ngrams.end()) {
- counts.add(it->second, ti->second, it->first.size() - 1);
- } else {
- counts.add(it->second, 0, it->first.size() - 1);
- }
- }
- return counts;
-}
-
/*
* bleu
*
@@ -48,26 +14,28 @@ make_ngram_counts(vector<WordID> hyp, vector<WordID> ref, unsigned N)
* NOTE: 0 if one n in {1..N} has 0 count
*/
score_t
-brevity_penaly(const unsigned hyp_len, const unsigned ref_len)
-{
- if (hyp_len > ref_len) return 1;
- return exp(1 - (score_t)ref_len/hyp_len);
-}
-score_t
-bleu(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len,
- unsigned N, vector<score_t> weights )
+BleuScorer::Bleu(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len)
{
if (hyp_len == 0 || ref_len == 0) return 0;
- if (ref_len < N) N = ref_len;
- if (weights.empty()) for (unsigned i = 0; i < N; i++) weights.push_back(1./N);
+ unsigned M = N_;
+ if (ref_len < N_) M = ref_len;
score_t sum = 0;
- for (unsigned i = 0; i < N; i++) {
+ for (unsigned i = 0; i < M; i++) {
if (counts.clipped[i] == 0 || counts.sum[i] == 0) return 0;
- sum += weights[i] * log((score_t)counts.clipped[i] / counts.sum[i]);
+ sum += w_[i] * log((score_t)counts.clipped[i] / counts.sum[i]);
}
return brevity_penaly(hyp_len, ref_len) * exp(sum);
}
+score_t
+BleuScorer::Score(ScoredHyp& hyp, vector<WordID>& ref_ids, unsigned id)
+{
+ unsigned hyp_len = hyp.w.size(), ref_len = ref_ids.size();
+ if (hyp_len == 0 || ref_len == 0) return 0;
+ NgramCounts counts = make_ngram_counts(hyp.w, ref_ids, N_);
+ return Bleu(counts, hyp_len, ref_len);
+}
+
/*
* 'stupid' bleu
*
@@ -79,18 +47,31 @@ bleu(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len,
* NOTE: 0 iff no 1gram match
*/
score_t
-stupid_bleu(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len,
- unsigned N, vector<score_t> weights )
+StupidBleuScorer::Score(ScoredHyp& hyp, vector<WordID>& ref_ids, unsigned id)
{
+ unsigned hyp_len = hyp.w.size(), ref_len = ref_ids.size();
if (hyp_len == 0 || ref_len == 0) return 0;
- if (ref_len < N) N = ref_len;
- if (weights.empty()) for (unsigned i = 0; i < N; i++) weights.push_back(1./N);
+ NgramCounts counts = make_ngram_counts(hyp.w, ref_ids, N_);
+ unsigned M = N_;
+ if (ref_len < N_) M = ref_len;
score_t sum = 0, add = 0;
- for (unsigned i = 0; i < N; i++) {
+ for (unsigned i = 0; i < M; i++) {
if (i == 1) add = 1;
- sum += weights[i] * log(((score_t)counts.clipped[i] + add) / ((counts.sum[i] + add)));
+ //cout << ((score_t)counts.clipped[i] + add) << "/" << counts.sum[i] +add << "." << endl;
+ //cout << "w_[i] " << w_[i] << endl;
+ sum += w_[i] * log(((score_t)counts.clipped[i] + add) / ((counts.sum[i] + add)));
+ //cout << "sum += "<< w_[i] * log(((score_t)counts.clipped[i] + add) / ((counts.sum[i] + add))) << endl;
}
- return brevity_penaly(hyp_len, ref_len) * exp(sum);
+ /*cout << ref_ids << endl;
+ cout << hyp.w << endl;
+ cout << "ref_len " << ref_len << endl;
+ cout << "hyp_len " << hyp_len << endl;
+ cout << "bp " << brevity_penaly(hyp_len, ref_len) << endl;
+ cout << "exp(sum) " << exp(sum) << endl;
+ counts.Print();
+ cout << brevity_penaly(hyp_len, ref_len) * exp(sum) << endl;
+ cout << "---" << endl;*/
+ return brevity_penaly(hyp_len, ref_len) * exp(sum);
}
/*
@@ -103,16 +84,16 @@ stupid_bleu(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len,
* NOTE: max is 0.9375
*/
score_t
-smooth_bleu(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len,
- const unsigned N, vector<score_t> weights )
+SmoothBleuScorer::Score(ScoredHyp& hyp, vector<WordID>& ref_ids, unsigned id)
{
+ unsigned hyp_len = hyp.w.size(), ref_len = ref_ids.size();
if (hyp_len == 0 || ref_len == 0) return 0;
- if (weights.empty()) for (unsigned i = 0; i < N; i++) weights.push_back(1./N);
+ NgramCounts counts = make_ngram_counts(hyp.w, ref_ids, N_);
score_t sum = 0;
unsigned j = 1;
- for (unsigned i = 0; i < N; i++) {
+ for (unsigned i = 0; i < N_; i++) {
if (counts.clipped[i] == 0 || counts.sum[i] == 0) continue;
- sum += exp((weights[i] * log((score_t)counts.clipped[i]/counts.sum[i]))) / pow(2, N-j+1);
+ sum += exp((w_[i] * log((score_t)counts.clipped[i]/counts.sum[i]))) / pow(2, N_-j+1);
j++;
}
return brevity_penaly(hyp_len, ref_len) * sum;
@@ -125,14 +106,39 @@ smooth_bleu(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len,
* and Structural Translation Features"
* (Chiang et al. '08)
*/
-score_t
-approx_bleu(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len,
- const unsigned N, vector<score_t> weights)
+/*void
+ApproxBleuScorer::Prep(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len)
+{
+ glob_onebest_counts += counts;
+ glob_hyp_len += hyp_len;
+ glob_ref_len += ref_len;
+}
+
+void
+ApproxBleuScorer::Reset()
{
- return brevity_penaly(hyp_len, ref_len)
- * 0.9 * bleu(counts, hyp_len, ref_len, N, weights);
+ glob_onebest_counts.Zero();
+ glob_hyp_len = 0;
+ glob_ref_len = 0;
}
+score_t
+ApproxBleuScorer::Score(ScoredHyp& hyp, vector<WordID>& ref_ids, unsigned id)
+{
+ NgramCounts counts = make_ngram_counts(hyp.w, ref_ids, N_);
+ if (id == 0) reset();
+ unsigned hyp_len = 0, ref_len = 0;
+ if (hyp.rank == 0) { // 'context of 1best translations'
+ scorer->prep(counts, hyp.w.size(), ref_ids.size());
+ counts.reset();
+ } else {
+ hyp_len = hyp.w.size();
+ ref_len = ref_ids.size();
+ }
+ return 0.9 * BleuScorer::Bleu(glob_onebest_counts + counts,
+ glob_hyp_len + hyp_len, glob_ref_len + ref_len);
+}*/
+
} // namespace
diff --git a/dtrain/score.h b/dtrain/score.h
index 3e5d82a9..f87d708c 100644
--- a/dtrain/score.h
+++ b/dtrain/score.h
@@ -7,6 +7,8 @@
#include <cassert>
#include <cmath>
+#include "kbestget.h"
+
#include "wordid.h" // cdec
using namespace std;
@@ -15,15 +17,13 @@ namespace dtrain
{
-typedef double score_t; // float
-
struct NgramCounts
{
unsigned N_;
map<unsigned, unsigned> clipped;
map<unsigned, unsigned> sum;
- NgramCounts(const unsigned N) : N_(N) { reset(); }
+ NgramCounts(const unsigned N) : N_(N) { Zero(); }
void
operator+=(const NgramCounts& rhs)
@@ -44,20 +44,19 @@ struct NgramCounts
}
void
- add(unsigned count, unsigned ref_count, unsigned i)
+ Add(unsigned count, unsigned ref_count, unsigned i)
{
assert(i < N_);
if (count > ref_count) {
clipped[i] += ref_count;
- sum[i] += count;
} else {
clipped[i] += count;
- sum[i] += count;
}
+ sum[i] += count;
}
void
- reset()
+ Zero()
{
unsigned i;
for (i = 0; i < N_; i++) {
@@ -67,7 +66,7 @@ struct NgramCounts
}
void
- print()
+ Print()
{
for (unsigned i = 0; i < N_; i++) {
cout << i+1 << "grams (clipped):\t" << clipped[i] << endl;
@@ -78,18 +77,99 @@ struct NgramCounts
typedef map<vector<WordID>, unsigned> Ngrams;
-Ngrams make_ngrams(vector<WordID>& s, unsigned N);
-NgramCounts make_ngram_counts(vector<WordID> hyp, vector<WordID> ref, unsigned N);
-
-score_t brevity_penaly(const unsigned hyp_len, const unsigned ref_len);
-score_t bleu(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len, const unsigned N,
- vector<score_t> weights = vector<score_t>());
-score_t stupid_bleu(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len, unsigned N,
- vector<score_t> weights = vector<score_t>());
-score_t smooth_bleu(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len, const unsigned N,
- vector<score_t> weights = vector<score_t>());
-score_t approx_bleu(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len, const unsigned N,
- vector<score_t> weights = vector<score_t>());
+inline Ngrams
+make_ngrams(const vector<WordID>& s, const unsigned N)
+{
+ Ngrams ngrams;
+ vector<WordID> ng;
+ for (size_t i = 0; i < s.size(); i++) {
+ ng.clear();
+ for (unsigned j = i; j < min(i+N, s.size()); j++) {
+ ng.push_back(s[j]);
+ ngrams[ng]++;
+ }
+ }
+ return ngrams;
+}
+
+inline NgramCounts
+make_ngram_counts(const vector<WordID>& hyp, const vector<WordID>& ref, const unsigned N)
+{
+ Ngrams hyp_ngrams = make_ngrams(hyp, N);
+ Ngrams ref_ngrams = make_ngrams(ref, N);
+ NgramCounts counts(N);
+ Ngrams::iterator it;
+ Ngrams::iterator ti;
+ for (it = hyp_ngrams.begin(); it != hyp_ngrams.end(); it++) {
+ ti = ref_ngrams.find(it->first);
+ if (ti != ref_ngrams.end()) {
+ counts.Add(it->second, ti->second, it->first.size() - 1);
+ } else {
+ counts.Add(it->second, 0, it->first.size() - 1);
+ }
+ }
+ return counts;
+}
+
+struct LocalScorer
+{
+ unsigned N_;
+ vector<score_t> w_;
+
+ virtual score_t
+ Score(ScoredHyp& hyp, vector<WordID>& ref_ids, unsigned id)=0;
+
+ void
+ Init(unsigned N, vector<score_t> weights)
+ {
+ assert(N > 0);
+ N_ = N;
+ if (weights.empty()) for (unsigned i = 0; i < N_; i++) w_.push_back(1./N_);
+ else w_ = weights;
+ }
+
+ score_t
+ brevity_penaly(const unsigned hyp_len, const unsigned ref_len)
+ {
+ if (hyp_len > ref_len) return 1;
+ return exp(1 - (score_t)ref_len/hyp_len);
+ }
+};
+
+struct BleuScorer : public LocalScorer
+{
+ score_t Bleu(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len);
+ score_t Score(ScoredHyp& hyp, vector<WordID>& ref_ids, unsigned id);
+};
+
+struct StupidBleuScorer : public LocalScorer
+{
+ score_t Score(ScoredHyp& hyp, vector<WordID>& ref_ids, unsigned id);
+};
+
+struct SmoothBleuScorer : public LocalScorer
+{
+ score_t Score(ScoredHyp& hyp, vector<WordID>& ref_ids, unsigned id);
+};
+
+// FIXME
+/*struct ApproxBleuScorer : public LocalScorer
+{
+ NgramCounts glob_onebest_counts;
+ unsigned glob_hyp_len, glob_ref_len;
+
+ void Prep(NgramCounts& counts, const unsigned hyp_len, const unsigned ref_len);
+ void Reset();
+ score_t Score(ScoredHyp& hyp, vector<WordID>& ref_ids, unsigned id);
+
+ ApproxBleuScorer()
+ {
+ glob_onebest_counts.Zero();
+ glob_hyp_len = 0;
+ glob_ref_len = 0;
+ }
+};*/
+
} // namespace
diff --git a/dtrain/test/example/cdec.ini b/dtrain/test/example/cdec.ini
index 50379afe..31a205c7 100644
--- a/dtrain/test/example/cdec.ini
+++ b/dtrain/test/example/cdec.ini
@@ -4,4 +4,4 @@ cubepruning_pop_limit=30
scfg_max_span_limit=15
feature_function=WordPenalty
feature_function=KLanguageModel test/example/nc-wmt11.en.srilm.gz
-#feature_function=RuleIdentityFeatures
+feature_function=RuleIdentityFeatures
diff --git a/dtrain/test/example/dtrain.ini b/dtrain/test/example/dtrain.ini
index fbddb915..df746e51 100644
--- a/dtrain/test/example/dtrain.ini
+++ b/dtrain/test/example/dtrain.ini
@@ -1,7 +1,7 @@
decoder_config=test/example/cdec.ini
k=100
-N=3
-epochs=1000
+N=4
+epochs=10
input=test/example/nc-1k.gz
scorer=stupid_bleu
output=test/example/weights.gz
diff --git a/dtrain/test/example/weights.gz b/dtrain/test/example/weights.gz
index e2e1ecce..e7baa367 100644
--- a/dtrain/test/example/weights.gz
+++ b/dtrain/test/example/weights.gz
Binary files differ
generated by cgit v1.2.3 (git 2.39.1) at 2024-09-21 19:40:57 +0000