diff options
-rwxr-xr-x | decoder/oracle_bleu.h | 15 | ||||
-rw-r--r-- | vest/mr_vest_generate_mapper_input.cc | 47 | ||||
-rw-r--r-- | vest/scorer.cc | 7 | ||||
-rw-r--r-- | vest/scorer.h | 5 |
4 files changed, 59 insertions, 15 deletions
diff --git a/decoder/oracle_bleu.h b/decoder/oracle_bleu.h index 07d83b7f..66d155d3 100755 --- a/decoder/oracle_bleu.h +++ b/decoder/oracle_bleu.h @@ -35,7 +35,7 @@ struct Translation { ViterbiESentence(hg,&sentence); features=ViterbiFeatures(hg,feature_weights,true); } - void Print(std::ostream &out,std::string pre=" +Oracle BLEU ") { + void Print(std::ostream &out,std::string pre=" +Oracle BLEU ") const { out<<pre<<"Viterbi: "<<TD::GetString(sentence)<<"\n"; out<<pre<<"features: "<<features<<std::endl; } @@ -46,11 +46,10 @@ struct Translation { }; struct Oracle { + Translation model,fear,hope; bool is_null() { return model.is_null() /* && fear.is_null() && hope.is_null() */; } - - Translation model,fear,hope; // feature 0 will be the error rate in fear and hope // move toward hope FeatureVector ModelHopeGradient() const { @@ -64,6 +63,16 @@ struct Oracle { r.set_value(0,0); return r; } + void Print(std::ostream &out) const { + hope.Print(out,"hope "); + model.Print(out,"model "); + fear.Print(out,"fear "); + } + friend inline std::ostream & operator<<(std::ostream &out,Oracle const& o) { + o.Print(out); + return out; + } + }; diff --git a/vest/mr_vest_generate_mapper_input.cc b/vest/mr_vest_generate_mapper_input.cc index ac826ecf..062d57b5 100644 --- a/vest/mr_vest_generate_mapper_input.cc +++ b/vest/mr_vest_generate_mapper_input.cc @@ -1,3 +1,4 @@ +//TODO: debug segfault when references supplied, null shared_ptr when oracle #include <iostream> #include <vector> #include <sstream> @@ -15,6 +16,8 @@ #include "oracle_bleu.h" #include "ff_bleu.h" +const bool DEBUG_ORACLE=true; + boost::shared_ptr<FFRegistry> global_ff_registry; namespace { struct init_ff { @@ -32,19 +35,18 @@ namespace po = boost::program_options; typedef SparseVector<double> Dir; typedef Dir Point; - void compress_similar(vector<Dir> &dirs,double min_dist,ostream *log=&cerr,bool avg=true,bool verbose=true) { // return; //TODO: debug if (min_dist<=0) return; double max_s=1.-min_dist; - if (log&&verbose) *log<<"max allowed S="<<max_s<<' '; + if (log&&verbose) *log<<"max allowed S="<<max_s<<endl; unsigned N=dirs.size(); for (int i=0;i<N;++i) { for (int j=i+1;j<N;++j) { double s=dirs[i].tanimoto_coef(dirs[j]); if (log&&verbose) *log<<"S["<<i<<","<<j<<"]="<<s<<' '; if (s>max_s) { - if (log) *log << "Collapsing similar directions (T="<<s<<" > "<<max_s<<"). dirs["<<i<<"]="<<dirs[i]<<" dirs["<<j<<"]"; + if (log) *log << "Collapsing similar directions (T="<<s<<" > "<<max_s<<"). dirs["<<i<<"]="<<dirs[i]<<" dirs["<<j<<"]"<<endl; if (avg) { dirs[i]+=dirs[j]; dirs[i]/=2.; @@ -96,7 +98,9 @@ struct oracle_directions { ("max_similarity,m",po::value<double>(&max_similarity)->default_value(0),"remove directions that are too similar (Tanimoto coeff. less than (1-this)). 0 means don't filter, 1 means only 1 direction allowed?") ("fear_to_hope,f",po::bool_switch(&fear_to_hope),"for each of the oracle_directions, also include a direction from fear to hope (as well as origin to hope)") ("no_old_to_hope,n","don't emit the usual old -> hope oracle") - ("decoder_translations",po::value<string>(&decoder_translations_file)->default_value(""),"one per line decoder 1best translations for computing document BLEU vs. sentences-seen-so-far BLEU"); + ("decoder_translations",po::value<string>(&decoder_translations_file)->default_value(""),"one per line decoder 1best translations for computing document BLEU vs. sentences-seen-so-far BLEU") + ("verbose",po::bool_switch(&verbose),"detailed logs") + ; } void InitCommandLine(int argc, char *argv[], po::variables_map *conf) { po::options_description opts("Configuration options"); @@ -140,11 +144,11 @@ struct oracle_directions { Run(); return 0; } - + bool verbose; void Run() { AddPrimaryAndRandomDirections(); AddOracleDirections(); - compress_similar(directions,max_similarity); + compress_similar(directions,max_similarity,&cerr,true,verbose); Print(); } @@ -186,11 +190,20 @@ struct oracle_directions { oracle_directions() { } Sentences model_hyps; + + vector<OracleBleu::ScoreP> model_scores; bool have_doc; void Init() { have_doc=!decoder_translations_file.empty(); if (have_doc) { model_hyps.Load(decoder_translations_file); + model_scores.resize(model_hyps.size()); + for (int i=0;i<model_hyps.size();++i) { + //FIXME: what is scoreccand? with / without clipping? do without for consistency w/ oracle + Score *s=oracle.ds[i]->ScoreCandidate(model_hyps[i]); + model_scores[i].reset(s); + oracle.doc_score->PlusEquals(*s); + } //TODO: compute doc bleu stats for each sentence, then when getting oracle temporarily exclude stats for that sentence (skip regular score updating) } start_random=false; @@ -217,11 +230,20 @@ struct oracle_directions { std::string decoder_translations_file; // one per line //TODO: is it worthwhile to get a complete document bleu first? would take a list of 1best translations one per line from the decoders, rather than loading all the forests (expensive). translations are in run.raw.N.gz - new arg + void adjust_doc(unsigned i,double scale=1.) { + oracle.doc_score->PlusEquals(*model_scores[i],scale); + } + + Score &ds() { + return *oracle.doc_score; + } + Oracle const& ComputeOracle(unsigned i) { Oracle &o=oracles[i]; if (o.is_null()) { if (have_doc) { - //TODO: + if (verbose) cerr<<"Before removing i="<<i<<" "<<ds().ScoreDetails()<<"\n"; + adjust_doc(i,-1); } ReadFile rf(forest_file(i)); Hypergraph hg; @@ -229,9 +251,18 @@ struct oracle_directions { Timer t("Loading forest from JSON "+forest_file(i)); HypergraphIO::ReadFromJSON(rf.stream(), &hg); } + if (verbose) cerr<<"Before oracle["<<i<<"]: "<<ds().ScoreDetails(); o=oracle.ComputeOracle(oracle.MakeMetadata(hg,i),&hg,origin,&cerr); + if (verbose) { + cerr << o; + cerr<<" ; after: "<<ds().ScoreDetails() + <<" oracle="<<oracle.GetScore(o.hope.sentence,i)->ScoreDetails() + <<" model="<<oracle.GetScore(o.model.sentence,i)->ScoreDetails()<<endl; + if (have_doc) + cerr<<" doc (should = model): "<<model_scores[i]->ScoreDetails()<<endl; + } if (have_doc) { - //TODO: + adjust_doc(i,1); } else oracle.IncludeLastScore(); } diff --git a/vest/scorer.cc b/vest/scorer.cc index 30015bb2..e8e9608a 100644 --- a/vest/scorer.cc +++ b/vest/scorer.cc @@ -96,9 +96,9 @@ class SERScore : public Score { } void PlusPartialEquals(const Score& /* delta */, int /* oracle_e_cover */, int /* oracle_f_cover */, int /* src_len */){} - void PlusEquals(const Score& delta, const float /* scale */) { - correct += static_cast<const SERScore&>(delta).correct; - total += static_cast<const SERScore&>(delta).total; + void PlusEquals(const Score& delta, const float scale) { + correct += scale*static_cast<const SERScore&>(delta).correct; + total += scale*static_cast<const SERScore&>(delta).total; } void PlusEquals(const Score& delta) { correct += static_cast<const SERScore&>(delta).correct; @@ -538,7 +538,6 @@ void BLEUScore::PlusEquals(const Score& delta, const float scale) { hyp_ngram_counts = ( hyp_ngram_counts + d.hyp_ngram_counts) * scale; ref_len = (ref_len + d.ref_len) * scale; hyp_len = ( hyp_len + d.hyp_len) * scale; - } void BLEUScore::PlusPartialEquals(const Score& delta, int oracle_e_cover, int oracle_f_cover, int src_len){ diff --git a/vest/scorer.h b/vest/scorer.h index 5bfeee0f..9c8aebcc 100644 --- a/vest/scorer.h +++ b/vest/scorer.h @@ -20,6 +20,11 @@ class Score { virtual float ComputeScore() const = 0; virtual float ComputePartialScore() const =0; virtual void ScoreDetails(std::string* details) const = 0; + std::string ScoreDetails() { + std::string d; + ScoreDetails(&d); + return d; + } virtual void PlusEquals(const Score& rhs, const float scale) = 0; virtual void PlusEquals(const Score& rhs) = 0; virtual void PlusPartialEquals(const Score& rhs, int oracle_e_cover, int oracle_f_cover, int src_len) = 0; |