Merge pull request #59 from kho/mrmira

Mr.MIRA compatibility

2 files changed, 58 insertions, 11 deletions

diff --git a/decoder/decoder.cc b/decoder/decoder.cc
index 737201e7..1e6c3194 100644
--- a/decoder/decoder.cc
+++ b/decoder/decoder.cc
@@ -17,6 +17,7 @@ namespace std { using std::tr1::unordered_map; }
 #include "fdict.h"
 #include "timing_stats.h"
 #include "verbose.h"
+#include "b64featvector.h"
 
 #include "translator.h"
 #include "phrasebased_translator.h"
@@ -301,6 +302,7 @@ struct DecoderImpl {
   bool feature_expectations; // TODO Observer
   bool output_training_vector; // TODO Observer
   bool remove_intersected_rule_annotations;
+  bool mr_mira_compat;  // Mr.MIRA compatibility mode.
   boost::scoped_ptr<IncrementalBase> incremental;
 
 
@@ -415,7 +417,8 @@ DecoderImpl::DecoderImpl(po::variables_map& conf, int argc, char** argv, istream
         ("vector_format",po::value<string>()->default_value("b64"), "Sparse vector serialization format for feature expectations or gradients, includes (text or b64)")
         ("combine_size,C",po::value<int>()->default_value(1), "When option -G is used, process this many sentence pairs before writing the gradient (1=emit after every sentence pair)")
         ("forest_output,O",po::value<string>(),"Directory to write forests to")
-        ("remove_intersected_rule_annotations", "After forced decoding is completed, remove nonterminal annotations (i.e., the source side spans)");
+        ("remove_intersected_rule_annotations", "After forced decoding is completed, remove nonterminal annotations (i.e., the source side spans)")
+        ("mr_mira_compat", "Mr.MIRA compatibility mode (applies weight delta if available; outputs number of lines before k-best)");
 
   // ob.AddOptions(&opts);
   po::options_description clo("Command line options");
@@ -668,6 +671,7 @@ DecoderImpl::DecoderImpl(po::variables_map& conf, int argc, char** argv, istream
   oracle.show_derivation=conf.count("show_derivations");
   oracle.show_derivation_mask=conf["show_derivations_mask"].as<int>();
   remove_intersected_rule_annotations = conf.count("remove_intersected_rule_annotations");
+  mr_mira_compat = conf.count("mr_mira_compat");
 
   combine_size = conf["combine_size"].as<int>();
   if (combine_size < 1) combine_size = 1;
@@ -701,6 +705,24 @@ void Decoder::AddSupplementalGrammarFromString(const std::string& grammar_string
   static_cast<SCFGTranslator&>(*pimpl_->translator).AddSupplementalGrammarFromString(grammar_string);
 }
 
+static inline void ApplyWeightDelta(const string &delta_b64, vector<weight_t> *weights) {
+  SparseVector<weight_t> delta;
+  DecodeFeatureVector(delta_b64, &delta);
+  if (delta.empty()) return;
+  // Apply updates
+  for (SparseVector<weight_t>::iterator dit = delta.begin();
+       dit != delta.end(); ++dit) {
+    int feat_id = dit->first;
+    union { weight_t weight; unsigned long long repr; } feat_delta;
+    feat_delta.weight = dit->second;
+    if (!SILENT)
+      cerr << "[decoder weight update] " << FD::Convert(feat_id) << " " << feat_delta.weight
+           << " = " << hex << feat_delta.repr << endl;
+    if (weights->size() <= feat_id) weights->resize(feat_id + 1);
+    (*weights)[feat_id] += feat_delta.weight;
+  }
+}
+
 bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
   string buf = input;
   NgramCache::Clear();   // clear ngram cache for remote LM (if used)
@@ -711,6 +733,10 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
   if (sgml.find("id") != sgml.end())
     sent_id = atoi(sgml["id"].c_str());
 
+  // Add delta from input to weights before decoding
+  if (mr_mira_compat)
+    ApplyWeightDelta(sgml["delta"], init_weights.get());
+
   if (!SILENT) {
     cerr << "\nINPUT: ";
     if (buf.size() < 100)
@@ -949,7 +975,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
     if (kbest && !has_ref) {
       //TODO: does this work properly?
       const string deriv_fname = conf.count("show_derivations") ? str("show_derivations",conf) : "-";
-      oracle.DumpKBest(sent_id, forest, conf["k_best"].as<int>(), unique_kbest,"-", deriv_fname);
+      oracle.DumpKBest(sent_id, forest, conf["k_best"].as<int>(), unique_kbest,mr_mira_compat, smeta.GetSourceLength(), "-", deriv_fname);
     } else if (csplit_output_plf) {
       cout << HypergraphIO::AsPLF(forest, false) << endl;
     } else {
@@ -1080,7 +1106,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
       if (conf.count("graphviz")) forest.PrintGraphviz();
       if (kbest) {
         const string deriv_fname = conf.count("show_derivations") ? str("show_derivations",conf) : "-";
-        oracle.DumpKBest(sent_id, forest, conf["k_best"].as<int>(), unique_kbest,"-", deriv_fname);
+        oracle.DumpKBest(sent_id, forest, conf["k_best"].as<int>(), unique_kbest, mr_mira_compat, smeta.GetSourceLength(), "-", deriv_fname);
       }
       if (conf.count("show_conditional_prob")) {
         const prob_t ref_z = Inside<prob_t, EdgeProb>(forest);
diff --git a/decoder/oracle_bleu.h b/decoder/oracle_bleu.h
index 893e36ca..cd587833 100644
--- a/decoder/oracle_bleu.h
+++ b/decoder/oracle_bleu.h
@@ -21,6 +21,7 @@
 #include "kbest.h"
 #include "timing_stats.h"
 #include "sentences.h"
+#include "b64featvector.h"
 
 //TODO: put function impls into .cc
 //TODO: move Translation into its own .h and use in cdec
@@ -255,18 +256,28 @@ struct OracleBleu {
   int show_derivation_mask;
 
   template <class Filter>
-  void kbest(int sent_id,Hypergraph const& forest,int k,std::ostream &kbest_out=std::cout,std::ostream &deriv_out=std::cerr) {
+  void kbest(int sent_id, Hypergraph const& forest, int k, bool mr_mira_compat,
+             int src_len, std::ostream& kbest_out = std::cout,
+             std::ostream& deriv_out = std::cerr) {
     using namespace std;
     using namespace boost;
     typedef KBest::KBestDerivations<Sentence, ESentenceTraversal,Filter> K;
     K kbest(forest,k);
     //add length (f side) src length of this sentence to the psuedo-doc src length count
     float curr_src_length = doc_src_length + tmp_src_length;
-    for (int i = 0; i < k; ++i) {
+    if (mr_mira_compat) kbest_out << k << "\n";
+    int i = 0;
+    for (; i < k; ++i) {
       typename K::Derivation *d = kbest.LazyKthBest(forest.nodes_.size() - 1, i);
       if (!d) break;
-      kbest_out << sent_id << " ||| " << TD::GetString(d->yield) << " ||| "
-                << d->feature_values << " ||| " << log(d->score);
+      kbest_out << sent_id << " ||| ";
+      if (mr_mira_compat) kbest_out << src_len << " ||| ";
+      kbest_out << TD::GetString(d->yield) << " ||| ";
+      if (mr_mira_compat)
+        kbest_out << EncodeFeatureVector(d->feature_values);
+      else
+        kbest_out << d->feature_values;
+      kbest_out << " ||| " << log(d->score);
       if (!refs.empty()) {
         ScoreP sentscore = GetScore(d->yield,sent_id);
         sentscore->PlusEquals(*doc_score,float(1));
@@ -281,10 +292,17 @@ struct OracleBleu {
         deriv_out<<"\n"<<flush;
       }
     }
+    if (mr_mira_compat) {
+      for (; i < k; ++i) kbest_out << "\n";
+      kbest_out << flush;
+    }
   }
 
 // TODO decoder output should probably be moved to another file - how about oracle_bleu.h
-  void DumpKBest(const int sent_id, const Hypergraph& forest, const int k, const bool unique, std::string const &kbest_out_filename_, std::string const &deriv_out_filename_) {
+  void DumpKBest(const int sent_id, const Hypergraph& forest, const int k,
+                 const bool unique, const bool mr_mira_compat,
+                 const int src_len, std::string const& kbest_out_filename_,
+                 std::string const& deriv_out_filename_) {
 
     WriteFile ko(kbest_out_filename_);
     std::cerr << "Output kbest to " << kbest_out_filename_ <<std::endl;
@@ -297,9 +315,11 @@ struct OracleBleu {
     WriteFile oderiv(sderiv.str());
 
     if (!unique)
-      kbest<KBest::NoFilter<std::vector<WordID> > >(sent_id,forest,k,ko.get(),oderiv.get());
+      kbest<KBest::NoFilter<std::vector<WordID> > >(
+          sent_id, forest, k, mr_mira_compat, src_len, ko.get(), oderiv.get());
     else {
-      kbest<KBest::FilterUnique>(sent_id,forest,k,ko.get(),oderiv.get());
+      kbest<KBest::FilterUnique>(sent_id, forest, k, mr_mira_compat, src_len,
+                                 ko.get(), oderiv.get());
     }
   }
 
@@ -307,7 +327,8 @@ void DumpKBest(std::string const& suffix,const int sent_id, const Hypergraph& fo
   {
     std::ostringstream kbest_string_stream;
     kbest_string_stream << forest_output << "/kbest_"<<suffix<< "." << sent_id;
-    DumpKBest(sent_id, forest, k, unique, kbest_string_stream.str(), "-");
+    DumpKBest(sent_id, forest, k, unique, false, -1, kbest_string_stream.str(),
+              "-");
   }
 
 };