From 5a8ef8ba65b244837e9cedbd64793b82bf284f93 Mon Sep 17 00:00:00 2001 From: Patrick Simianer Date: Wed, 2 May 2012 15:02:59 +0200 Subject: small improvements --- dtrain/dtrain.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'dtrain/dtrain.h') diff --git a/dtrain/dtrain.h b/dtrain/dtrain.h index 94d149ce..ac0345a4 100644 --- a/dtrain/dtrain.h +++ b/dtrain/dtrain.h @@ -32,7 +32,7 @@ inline void register_and_convert(const vector& strs, vector& ids inline string gettmpf(const string path, const string infix) { - char fn[1024]; + char fn[path.size() + infix.size() + 8]; strcpy(fn, path.c_str()); strcat(fn, "/"); strcat(fn, infix.c_str()); -- cgit v1.2.3 From eb3ee28dc0eb1d3e5ed01ba0df843be329ae450d Mon Sep 17 00:00:00 2001 From: Patrick Simianer Date: Wed, 2 May 2012 15:33:58 +0200 Subject: defines --- dtrain/dtrain.h | 12 ++-- dtrain/pairsampling.h | 3 - dtrain/test/example/expected-output | 124 ++++++++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 dtrain/test/example/expected-output (limited to 'dtrain/dtrain.h') diff --git a/dtrain/dtrain.h b/dtrain/dtrain.h index ac0345a4..d8dc14b6 100644 --- a/dtrain/dtrain.h +++ b/dtrain/dtrain.h @@ -1,6 +1,14 @@ #ifndef _DTRAIN_H_ #define _DTRAIN_H_ +#undef DTRAIN_FASTER_PERCEPTRON // only look at misranked pairs + // DO NOT USE WITH SVM! +#undef DTRAIN_LOCAL +#define DTRAIN_DOTS 10 // after how many inputs to display a '.' +#define DTRAIN_GRAMMAR_DELIM "########EOS########" +#define DTRAIN_SCALE 100000 + + #include #include #include @@ -13,11 +21,7 @@ #include "filelib.h" -#undef DTRAIN_LOCAL -#define DTRAIN_DOTS 10 // after how many inputs to display a '.' -#define DTRAIN_GRAMMAR_DELIM "########EOS########" -#define DTRAIN_SCALE 100000 using namespace std; using namespace dtrain; diff --git a/dtrain/pairsampling.h b/dtrain/pairsampling.h index 52eeedd6..5085738e 100644 --- a/dtrain/pairsampling.h +++ b/dtrain/pairsampling.h @@ -1,9 +1,6 @@ #ifndef _DTRAIN_PAIRSAMPLING_H_ #define _DTRAIN_PAIRSAMPLING_H_ -#define DTRAIN_FASTER_PERCEPTRON // only look at misranked pairs - // DO NOT USE WITH SVM! - namespace dtrain { diff --git a/dtrain/test/example/expected-output b/dtrain/test/example/expected-output new file mode 100644 index 00000000..08733dd4 --- /dev/null +++ b/dtrain/test/example/expected-output @@ -0,0 +1,124 @@ + cdec cfg 'test/example/cdec.ini' +feature: WordPenalty (no config parameters) +State is 0 bytes for feature WordPenalty +feature: KLanguageModel (with config parameters 'test/example/nc-wmt11.en.srilm.gz') +Loading the LM will be faster if you build a binary file. +Reading test/example/nc-wmt11.en.srilm.gz +----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100 +**************************************************************************************************** +Loaded 5-gram KLM from test/example/nc-wmt11.en.srilm.gz (MapSize=49581) +State is 98 bytes for feature KLanguageModel test/example/nc-wmt11.en.srilm.gz +feature: RuleIdentityFeatures (no config parameters) +State is 0 bytes for feature RuleIdentityFeatures +feature: RuleNgramFeatures (no config parameters) +State is 0 bytes for feature RuleNgramFeatures +feature: RuleShape (no config parameters) + Example feature: Shape_S00000_T00000 +State is 0 bytes for feature RuleShape +Seeding random number sequence to 380245307 + +dtrain +Parameters: + k 100 + N 4 + T 3 + scorer 'stupid_bleu' + sample from 'kbest' + filter 'uniq' + learning rate 0.0001 + gamma 0 + pairs 'XYX' + hi lo 0.1 + pair threshold 0 + select weights 'VOID' + l1 reg 0 'none' + cdec cfg 'test/example/cdec.ini' + input 'test/example/nc-wmt11.1k.gz' + output '-' + stop_after 20 +(a dot represents 10 inputs) +Iteration #1 of 3. + .. 20 +Stopping after 20 input sentences. +WEIGHTS + Glue = -0.1015 + WordPenalty = -0.0152 + LanguageModel = +0.21493 + LanguageModel_OOV = -0.3257 + PhraseModel_0 = -0.050844 + PhraseModel_1 = +0.25074 + PhraseModel_2 = +0.27944 + PhraseModel_3 = -0.038384 + PhraseModel_4 = -0.12041 + PhraseModel_5 = +0.1047 + PhraseModel_6 = -0.1289 + PassThrough = -0.3094 + --- + 1best avg score: 0.17508 (+0.17508) + 1best avg model score: -1.2392 (-1.2392) + avg # pairs: 1329.8 + avg # rank err: 649.1 + avg # margin viol: 677.5 + non0 feature count: 874 + avg list sz: 88.6 + avg f count: 85.643 +(time 0.25 min, 0.75 s/S) + +Iteration #2 of 3. + .. 20 +WEIGHTS + Glue = -0.0792 + WordPenalty = -0.056198 + LanguageModel = +0.31038 + LanguageModel_OOV = -0.4011 + PhraseModel_0 = +0.072188 + PhraseModel_1 = +0.11473 + PhraseModel_2 = +0.049774 + PhraseModel_3 = -0.18448 + PhraseModel_4 = -0.12092 + PhraseModel_5 = +0.1599 + PhraseModel_6 = -0.0606 + PassThrough = -0.3848 + --- + 1best avg score: 0.24015 (+0.065075) + 1best avg model score: -10.131 (-8.8914) + avg # pairs: 1324.7 + avg # rank err: 558.65 + avg # margin viol: 752.85 + non0 feature count: 1236 + avg list sz: 84.9 + avg f count: 88.306 +(time 0.22 min, 0.65 s/S) + +Iteration #3 of 3. + .. 20 +WEIGHTS + Glue = -0.051 + WordPenalty = -0.077956 + LanguageModel = +0.33699 + LanguageModel_OOV = -0.4726 + PhraseModel_0 = +0.040228 + PhraseModel_1 = +0.18 + PhraseModel_2 = +0.15618 + PhraseModel_3 = -0.098908 + PhraseModel_4 = -0.036555 + PhraseModel_5 = +0.1619 + PhraseModel_6 = +0.0078 + PassThrough = -0.4563 + --- + 1best avg score: 0.25527 (+0.015113) + 1best avg model score: -13.906 (-3.7756) + avg # pairs: 1356.3 + avg # rank err: 562.1 + avg # margin viol: 757.35 + non0 feature count: 1482 + avg list sz: 86.65 + avg f count: 87.475 +(time 0.23 min, 0.7 s/S) + +Writing weights file to '-' ... +done + +--- +Best iteration: 3 [SCORE 'stupid_bleu'=0.25527]. +This took 0.7 min. -- cgit v1.2.3 From 1d1172a3f85ab7423ae9537cf3c73afdfe7dc693 Mon Sep 17 00:00:00 2001 From: Patrick Simianer Date: Thu, 14 Jun 2012 20:11:57 +0200 Subject: 'sum bleu' --- dtrain/README.md | 2 ++ dtrain/dtrain.cc | 8 ++++++-- dtrain/dtrain.h | 2 +- dtrain/score.cc | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++---- dtrain/score.h | 12 ++++++++++- 5 files changed, 78 insertions(+), 8 deletions(-) (limited to 'dtrain/dtrain.h') diff --git a/dtrain/README.md b/dtrain/README.md index 7aefcc55..843874fa 100644 --- a/dtrain/README.md +++ b/dtrain/README.md @@ -58,6 +58,8 @@ Next + bigger LM, feats (target side Ng., word alignments etc.) + merge kbest lists + proper eval, pairwise ranking, forced transl ++ smmoth bleu variants X ++ MMERT exp Legal ----- diff --git a/dtrain/dtrain.cc b/dtrain/dtrain.cc index 7f44d4cf..eea58393 100644 --- a/dtrain/dtrain.cc +++ b/dtrain/dtrain.cc @@ -165,8 +165,12 @@ main(int argc, char** argv) scorer = dynamic_cast(new StupidBleuScorer); } else if (scorer_str == "smooth_bleu") { scorer = dynamic_cast(new SmoothBleuScorer); - } else if (scorer_str == "smooth_single_bleu") { - scorer = dynamic_cast(new SmoothSingleBleuScorer); + } else if (scorer_str == "sum_bleu") { + scorer = dynamic_cast(new SumBleuScorer); + } else if (scorer_str == "sumexp_bleu") { + scorer = dynamic_cast(new SumExpBleuScorer); + } else if (scorer_str == "sumwhatever_bleu") { + scorer = dynamic_cast(new SumWhateverBleuScorer); } else if (scorer_str == "approx_bleu") { scorer = dynamic_cast(new ApproxBleuScorer(N, approx_bleu_d)); } else if (scorer_str == "lc_bleu") { diff --git a/dtrain/dtrain.h b/dtrain/dtrain.h index d8dc14b6..7e084a79 100644 --- a/dtrain/dtrain.h +++ b/dtrain/dtrain.h @@ -3,7 +3,7 @@ #undef DTRAIN_FASTER_PERCEPTRON // only look at misranked pairs // DO NOT USE WITH SVM! -#undef DTRAIN_LOCAL +#define DTRAIN_LOCAL #define DTRAIN_DOTS 10 // after how many inputs to display a '.' #define DTRAIN_GRAMMAR_DELIM "########EOS########" #define DTRAIN_SCALE 100000 diff --git a/dtrain/score.cc b/dtrain/score.cc index 5bb0bcaa..4a7cac6e 100644 --- a/dtrain/score.cc +++ b/dtrain/score.cc @@ -80,7 +80,7 @@ StupidBleuScorer::Score(vector& hyp, vector& ref, * to Machine Translation" * (Liang et al. '06) * - * NOTE: max is 0.9375 + * NOTE: max is 0.9375 (with N=4) */ score_t SmoothBleuScorer::Score(vector& hyp, vector& ref, @@ -108,9 +108,13 @@ SmoothBleuScorer::Score(vector& hyp, vector& ref, return brevity_penalty(hyp_len, ref_len) * sum; } -// variant of smooth_bleu; i-Bleu scores only single 'i' +/* + * 'sum' bleu + * + * sum up Ngram precisions + */ score_t -SmoothSingleBleuScorer::Score(vector& hyp, vector& ref, +SumBleuScorer::Score(vector& hyp, vector& ref, const unsigned /*rank*/, const unsigned /*src_len*/) { unsigned hyp_len = hyp.size(), ref_len = ref.size(); @@ -128,7 +132,57 @@ SmoothSingleBleuScorer::Score(vector& hyp, vector& ref, return brevity_penalty(hyp_len, ref_len) * sum; } -// TODO single variants! +/* + * 'sum' (exp) bleu + * + * sum up exp(Ngram precisions) + */ +score_t +SumExpBleuScorer::Score(vector& hyp, vector& ref, + const unsigned /*rank*/, const unsigned /*src_len*/) +{ + unsigned hyp_len = hyp.size(), ref_len = ref.size(); + if (hyp_len == 0 || ref_len == 0) return 0.; + NgramCounts counts = make_ngram_counts(hyp, ref, N_); + unsigned M = N_; + if (ref_len < N_) M = ref_len; + score_t sum = 0.; + unsigned j = 1; + for (unsigned i = 0; i < M; i++) { + if (counts.sum_[i] == 0 || counts.clipped_[i] == 0) break; + sum += exp(((score_t)counts.clipped_[i]/counts.sum_[i]))/pow(2., N_-j+1); + j++; + } + return brevity_penalty(hyp_len, ref_len) * sum; +} + +/* + * 'sum' (whatever) bleu + * + * sum up exp(weight * log(Ngram precisions)) + */ +score_t +SumWhateverBleuScorer::Score(vector& hyp, vector& ref, + const unsigned /*rank*/, const unsigned /*src_len*/) +{ + unsigned hyp_len = hyp.size(), ref_len = ref.size(); + if (hyp_len == 0 || ref_len == 0) return 0.; + NgramCounts counts = make_ngram_counts(hyp, ref, N_); + unsigned M = N_; + vector v = w_; + if (ref_len < N_) { + M = ref_len; + for (unsigned i = 0; i < M; i++) v[i] = 1/((score_t)M); + } + score_t sum = 0.; + unsigned j = 1; + for (unsigned i = 0; i < M; i++) { + if (counts.sum_[i] == 0 || counts.clipped_[i] == 0) break; + sum += exp(v[i] * log(((score_t)counts.clipped_[i]/counts.sum_[i])))/pow(2., N_-j+1); + j++; + } + return brevity_penalty(hyp_len, ref_len) * sum; +} /* * approx. bleu diff --git a/dtrain/score.h b/dtrain/score.h index c5be2829..f317c903 100644 --- a/dtrain/score.h +++ b/dtrain/score.h @@ -153,7 +153,17 @@ struct SmoothBleuScorer : public LocalScorer score_t Score(vector& hyp, vector& ref, const unsigned /*rank*/, const unsigned /*src_len*/); }; -struct SmoothSingleBleuScorer : public LocalScorer +struct SumBleuScorer : public LocalScorer +{ + score_t Score(vector& hyp, vector& ref, const unsigned /*rank*/, const unsigned /*src_len*/); +}; + +struct SumExpBleuScorer : public LocalScorer +{ + score_t Score(vector& hyp, vector& ref, const unsigned /*rank*/, const unsigned /*src_len*/); +}; + +struct SumWhateverBleuScorer : public LocalScorer { score_t Score(vector& hyp, vector& ref, const unsigned /*rank*/, const unsigned /*src_len*/); }; -- cgit v1.2.3