RuleTargetBigramFeatures, parallelize.rb

8 files changed, 198 insertions, 8 deletions

diff --git a/decoder/cdec_ff.cc b/decoder/cdec_ff.cc
index b516c386..d64bdada 100644
--- a/decoder/cdec_ff.cc
+++ b/decoder/cdec_ff.cc
@@ -47,8 +47,9 @@ void register_feature_functions() {
   ff_registry.Register("RuleIdentityFeatures", new FFFactory<RuleIdentityFeatures>());
   ff_registry.Register("SourceSyntaxFeatures", new FFFactory<SourceSyntaxFeatures>);
   ff_registry.Register("SourceSpanSizeFeatures", new FFFactory<SourceSpanSizeFeatures>);
-  ff_registry.Register("RuleNgramFeatures", new FFFactory<RuleNgramFeatures>());
   ff_registry.Register("CMR2008ReorderingFeatures", new FFFactory<CMR2008ReorderingFeatures>());
+  ff_registry.Register("RuleSourceBigramFeatures", new FFFactory<RuleSourceBigramFeatures>());
+  ff_registry.Register("RuleTargetBigramFeatures", new FFFactory<RuleTargetBigramFeatures>());
   ff_registry.Register("KLanguageModel", new KLanguageModelFactory());
   ff_registry.Register("NonLatinCount", new FFFactory<NonLatinCount>);
   ff_registry.Register("RuleShape", new FFFactory<RuleShapeFeatures>);
diff --git a/decoder/ff_rules.cc b/decoder/ff_rules.cc
index bd4c4cc0..3d0e514a 100644
--- a/decoder/ff_rules.cc
+++ b/decoder/ff_rules.cc
@@ -66,15 +66,15 @@ void RuleIdentityFeatures::TraversalFeaturesImpl(const SentenceMetadata& smeta,
   features->add_value(it->second, 1);
 }
 
-RuleNgramFeatures::RuleNgramFeatures(const std::string& param) {
+RuleSourceBigramFeatures::RuleSourceBigramFeatures(const std::string& param) {
 }
 
-void RuleNgramFeatures::PrepareForInput(const SentenceMetadata& smeta) {
+void RuleSourceBigramFeatures::PrepareForInput(const SentenceMetadata& smeta) {
 //  std::map<const TRule*, SparseVector<double> >
   rule2_feats_.clear();
 }
 
-void RuleNgramFeatures::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+void RuleSourceBigramFeatures::TraversalFeaturesImpl(const SentenceMetadata& smeta,
                                          const Hypergraph::Edge& edge,
                                          const vector<const void*>& ant_contexts,
                                          SparseVector<double>* features,
@@ -92,14 +92,52 @@ void RuleNgramFeatures::TraversalFeaturesImpl(const SentenceMetadata& smeta,
       assert(w > 0);
       const string& cur = TD::Convert(w);
       ostringstream os;
-      os << "RB:" << prev << '_' << cur;
+      os << "RBS:" << prev << '_' << cur;
       const int fid = FD::Convert(Escape(os.str()));
       if (fid <= 0) return;
       f.add_value(fid, 1.0);
       prev = cur;
     }
     ostringstream os;
-    os << "RB:" << prev << '_' << "</r>";
+    os << "RBS:" << prev << '_' << "</r>";
+    f.set_value(FD::Convert(Escape(os.str())), 1.0);
+  }
+  (*features) += it->second;
+}
+
+RuleTargetBigramFeatures::RuleTargetBigramFeatures(const std::string& param) {
+}
+
+void RuleTargetBigramFeatures::PrepareForInput(const SentenceMetadata& smeta) {
+  rule2_feats_.clear();
+}
+
+void RuleTargetBigramFeatures::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                         const Hypergraph::Edge& edge,
+                                         const vector<const void*>& ant_contexts,
+                                         SparseVector<double>* features,
+                                         SparseVector<double>* estimated_features,
+                                         void* context) const {
+  map<const TRule*, SparseVector<double> >::iterator it = rule2_feats_.find(edge.rule_.get());
+  if (it == rule2_feats_.end()) {
+    const TRule& rule = *edge.rule_;
+    it = rule2_feats_.insert(make_pair(&rule, SparseVector<double>())).first;
+    SparseVector<double>& f = it->second;
+    string prev = "<r>";
+    for (int i = 0; i < rule.e_.size(); ++i) {
+      WordID w = rule.e_[i];
+      if (w < 0) w = -w;
+      if (w == 0) return;
+      const string& cur = TD::Convert(w);
+      ostringstream os;
+      os << "RBT:" << prev << '_' << cur;
+      const int fid = FD::Convert(Escape(os.str()));
+      if (fid <= 0) return;
+      f.add_value(fid, 1.0);
+      prev = cur;
+    }
+    ostringstream os;
+    os << "RBT:" << prev << '_' << "</r>";
     f.set_value(FD::Convert(Escape(os.str())), 1.0);
   }
   (*features) += it->second;
diff --git a/decoder/ff_rules.h b/decoder/ff_rules.h
index 48d8bd05..08b168b0 100644
--- a/decoder/ff_rules.h
+++ b/decoder/ff_rules.h
@@ -22,9 +22,24 @@ class RuleIdentityFeatures : public FeatureFunction {
   mutable std::map<const TRule*, int> rule2_fid_;
 };
 
-class RuleNgramFeatures : public FeatureFunction {
+class RuleSourceBigramFeatures : public FeatureFunction {
  public:
-  RuleNgramFeatures(const std::string& param);
+  RuleSourceBigramFeatures(const std::string& param);
+ protected:
+  virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_contexts,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* context) const;
+  virtual void PrepareForInput(const SentenceMetadata& smeta);
+ private:
+  mutable std::map<const TRule*, SparseVector<double> > rule2_feats_;
+};
+
+class RuleTargetBigramFeatures : public FeatureFunction {
+ public:
+  RuleTargetBigramFeatures(const std::string& param);
  protected:
   virtual void TraversalFeaturesImpl(const SentenceMetadata& smeta,
                                      const Hypergraph::Edge& edge,
diff --git a/dtrain/parallelize/parallelize.rb b/dtrain/parallelize/parallelize.rb
new file mode 100755
index 00000000..e88d9eef
--- /dev/null
+++ b/dtrain/parallelize/parallelize.rb
@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+
+
+if ARGV.size != 5
+  STDERR.write "Usage: "
+  STDERR.write "ruby parallelize.rb <#shards> <input> <refs> <epochs> <dtrain.ini>\n"
+  exit
+end
+
+dtrain_bin = '/home/pks/mt/cdec-dtrain/dtrain/dtrain_local_new_f'
+ruby       = '/usr/bin/ruby'
+lplp_rb    = '/home/pks/mt/cdec-dtrain/dtrain/hstreaming/lplp.rb'
+lplp_args  = 'l2 select_k 100000'
+gzip       = '/bin/gzip'
+
+num_shards = ARGV[0].to_i
+input      = ARGV[1]
+refs       = ARGV[2]
+epochs     = ARGV[3].to_i
+ini        = ARGV[4]
+
+
+`mkdir work`
+
+def make_shards(input, refs, num_shards)
+  lc = `wc -l #{input}`.split.first.to_i
+  shard_sz = lc / num_shards
+  leftover = lc % num_shards
+  in_f = File.new input, 'r'
+  refs_f = File.new refs, 'r'
+  shard_in_files = []
+  shard_refs_files = []
+  0.upto(num_shards-1) { |shard|
+    shard_in = File.new "work/shard.#{shard}.in", 'w+'
+    shard_refs = File.new "work/shard.#{shard}.refs", 'w+'
+    0.upto(shard_sz-1) { |i|
+      shard_in.write in_f.gets
+      shard_refs.write refs_f.gets
+    }
+    shard_in_files << shard_in
+    shard_refs_files << shard_refs
+  }
+  while leftover > 0
+    shard_in_files[-1].write in_f.gets
+    shard_refs_files[-1].write refs_f.gets
+    leftover -= 1
+  end
+  (shard_in_files + shard_refs_files).each do |f| f.close end
+  in_f.close
+  refs_f.close
+end
+
+make_shards input, refs, num_shards
+
+0.upto(epochs-1) { |epoch|
+  pids = []
+  input_weights = ''
+  if epoch > 0 then input_weights = "--input_weights work/weights.#{epoch-1}" end
+  weights_files = []
+  0.upto(num_shards-1) { |shard|
+    pids << Kernel.fork {
+      `#{dtrain_bin} -c #{ini}\
+        --input work/shard.#{shard}.in\
+        --refs work/shard.#{shard}.refs #{input_weights}\
+        --output work/weights.#{shard}.#{epoch}\
+        &> work/out.#{shard}.#{epoch}`
+    }
+    weights_files << "work/weights.#{shard}.#{epoch}"
+  }
+  pids.each { |pid| Process.wait(pid) }
+  cat = File.new('work/weights_cat', 'w+')
+  weights_files.each { |f| cat.write File.new(f, 'r').read }
+  cat.close
+  `#{ruby} #{lplp_rb} #{lplp_args} #{num_shards} < work/weights_cat &> work/weights.#{epoch}`
+}
+
+`rm work/weights_cat`
+`#{gzip} work/*`
+
diff --git a/dtrain/parallelize/test/cdec.ini b/dtrain/parallelize/test/cdec.ini
new file mode 100644
index 00000000..72e99dc5
--- /dev/null
+++ b/dtrain/parallelize/test/cdec.ini
@@ -0,0 +1,22 @@
+formalism=scfg
+add_pass_through_rules=true
+intersection_strategy=cube_pruning
+cubepruning_pop_limit=200
+scfg_max_span_limit=15
+feature_function=WordPenalty
+feature_function=KLanguageModel /stor/dat/wmt12/en/news_only/m/wmt12.news.en.3.kenv5
+#feature_function=ArityPenalty
+#feature_function=CMR2008ReorderingFeatures
+#feature_function=Dwarf
+#feature_function=InputIndicator
+#feature_function=LexNullJump
+#feature_function=NewJump
+#feature_function=NgramFeatures
+#feature_function=NonLatinCount
+#feature_function=OutputIndicator
+#feature_function=RuleIdentityFeatures
+#feature_function=RuleNgramFeatures
+#feature_function=RuleShape
+#feature_function=SourceSpanSizeFeatures
+#feature_function=SourceWordPenalty
+#feature_function=SpanFeatures
diff --git a/dtrain/parallelize/test/dtrain.ini b/dtrain/parallelize/test/dtrain.ini
new file mode 100644
index 00000000..03f9d240
--- /dev/null
+++ b/dtrain/parallelize/test/dtrain.ini
@@ -0,0 +1,15 @@
+k=100
+N=4
+learning_rate=0.0001
+gamma=0
+loss_margin=0
+epochs=1
+scorer=stupid_bleu
+sample_from=kbest
+filter=uniq
+pair_sampling=XYX
+hi_lo=0.1
+select_weights=last
+print_weights=Glue WordPenalty LanguageModel LanguageModel_OOV PhraseModel_0 PhraseModel_1 PhraseModel_2 PhraseModel_3 PhraseModel_4 PhraseModel_5 PhraseModel_6 PassThrough
+tmp=/tmp
+decoder_config=cdec.ini
diff --git a/dtrain/parallelize/test/in b/dtrain/parallelize/test/in
new file mode 100644
index 00000000..a312809f
--- /dev/null
+++ b/dtrain/parallelize/test/in
@@ -0,0 +1,10 @@
+<seg grammar="/stor/dat/wmt12/dev/newstest2010/g/grammar.out.0.gz" id="0">barack obama erhält als vierter us @-@ präsident den frieden nobelpreis</seg>
+<seg grammar="/stor/dat/wmt12/dev/newstest2010/g/grammar.out.1.gz" id="1">der amerikanische präsident barack obama kommt für 26 stunden nach oslo , norwegen , um hier als vierter us @-@ präsident in der geschichte den frieden nobelpreis entgegen zunehmen .</seg>
+<seg grammar="/stor/dat/wmt12/dev/newstest2010/g/grammar.out.2.gz" id="2">darüber hinaus erhält er das diplom sowie die medaille und einen scheck über 1,4 mio. dollar für seine außer gewöhnlichen bestrebungen um die intensivierung der welt diplomatie und zusammen arbeit unter den völkern .</seg>
+<seg grammar="/stor/dat/wmt12/dev/newstest2010/g/grammar.out.3.gz" id="3">der chef des weißen hauses kommt morgen zusammen mit seiner frau michelle in der nordwegischen metropole an und wird die ganze zeit beschäftigt sein .</seg>
+<seg grammar="/stor/dat/wmt12/dev/newstest2010/g/grammar.out.4.gz" id="4">zunächst stattet er dem nobel @-@ institut einen besuch ab , wo er überhaupt zum ersten mal mit den fünf ausschuss mitglieder zusammen trifft , die ihn im oktober aus 172 leuten und 33 organisationen gewählt haben .</seg>
+<seg grammar="/stor/dat/wmt12/dev/newstest2010/g/grammar.out.5.gz" id="5">das präsidenten paar hat danach ein treffen mit dem norwegischen könig harald v. und königin sonja eingeplant .</seg>
+<seg grammar="/stor/dat/wmt12/dev/newstest2010/g/grammar.out.6.gz" id="6">nachmittags erreicht dann der besuch seinen höhepunkt mit der zeremonie , bei der obama den prestige preis übernimmt .</seg>
+<seg grammar="/stor/dat/wmt12/dev/newstest2010/g/grammar.out.7.gz" id="7">diesen erhält er als der vierte us @-@ präsident , aber erst als der dritte , der den preis direkt im amt entgegen nimmt .</seg>
+<seg grammar="/stor/dat/wmt12/dev/newstest2010/g/grammar.out.8.gz" id="8">das weiße haus avisierte schon , dass obama bei der übernahme des preises über den afghanistan krieg sprechen wird .</seg>
+<seg grammar="/stor/dat/wmt12/dev/newstest2010/g/grammar.out.9.gz" id="9">der präsident will diesem thema nicht ausweichen , weil er weiß , dass er den preis als ein präsident übernimmt , der zur zeit krieg in zwei ländern führt .</seg>
diff --git a/dtrain/parallelize/test/refs b/dtrain/parallelize/test/refs
new file mode 100644
index 00000000..4d3128cb
--- /dev/null
+++ b/dtrain/parallelize/test/refs
@@ -0,0 +1,10 @@
+barack obama becomes the fourth american president to receive the nobel peace prize
+the american president barack obama will fly into oslo , norway for 26 hours to receive the nobel peace prize , the fourth american president in history to do so .
+he will receive a diploma , medal and cheque for 1.4 million dollars for his exceptional efforts to improve global diplomacy and encourage international cooperation , amongst other things .
+the head of the white house will be flying into the norwegian city in the morning with his wife michelle and will have a busy schedule .
+first , he will visit the nobel institute , where he will have his first meeting with the five committee members who selected him from 172 people and 33 organisations .
+the presidential couple then has a meeting scheduled with king harald v and queen sonja of norway .
+then , in the afternoon , the visit will culminate in a grand ceremony , at which obama will receive the prestigious award .
+he will be the fourth american president to be awarded the prize , and only the third to have received it while actually in office .
+the white house has stated that , when he accepts the prize , obama will speak about the war in afghanistan .
+the president does not want to skirt around this topic , as he realises that he is accepting the prize as a president whose country is currently at war in two countries .