word aligner cleanup, new features

17 files changed, 201 insertions, 64 deletions

diff --git a/decoder/cdec_ff.cc b/decoder/cdec_ff.cc
index 34499398..b4381dda 100644
--- a/decoder/cdec_ff.cc
+++ b/decoder/cdec_ff.cc
@@ -16,6 +16,7 @@ void register_feature_functions() {
   global_ff_registry->Register("RelativeSentencePosition", new FFFactory<RelativeSentencePosition>);
   global_ff_registry->Register("Model2BinaryFeatures", new FFFactory<Model2BinaryFeatures>);
   global_ff_registry->Register("MarkovJump", new FFFactory<MarkovJump>);
+  global_ff_registry->Register("MarkovJumpFClass", new FFFactory<MarkovJumpFClass>);
   global_ff_registry->Register("SourcePOSBigram", new FFFactory<SourcePOSBigram>);
   global_ff_registry->Register("BlunsomSynchronousParseHack", new FFFactory<BlunsomSynchronousParseHack>);
   global_ff_registry->Register("AlignerResults", new FFFactory<AlignerResults>);
diff --git a/decoder/ff_wordalign.cc b/decoder/ff_wordalign.cc
index e3fa91d4..fb90df62 100644
--- a/decoder/ff_wordalign.cc
+++ b/decoder/ff_wordalign.cc
@@ -1,5 +1,6 @@
 #include "ff_wordalign.h"
 
+#include <set>
 #include <sstream>
 #include <string>
 #include <cmath>
@@ -12,20 +13,20 @@
 #include "tdict.h"   // Blunsom hack
 #include "filelib.h" // Blunsom hack
 
-static const size_t MAX_SENTENCE_SIZE = 100;
+static const int MAX_SENTENCE_SIZE = 100;
 
 using namespace std;
 
 Model2BinaryFeatures::Model2BinaryFeatures(const string& param) :
     fids_(boost::extents[MAX_SENTENCE_SIZE][MAX_SENTENCE_SIZE][MAX_SENTENCE_SIZE]) {
-  for (int i = 0; i < MAX_SENTENCE_SIZE; ++i) {
-    for (int j = 0; j < MAX_SENTENCE_SIZE; ++j) {
+  for (int i = 1; i < MAX_SENTENCE_SIZE; ++i) {
+    for (int j = 0; j < i; ++j) {
       for (int k = 0; k < MAX_SENTENCE_SIZE; ++k) {
         int& val = fids_[i][j][k];
         val = -1;
         if (j < i) {
           ostringstream os;
-          os << "M2_" << i << '_' << j << ':' << k;
+          os << "M2_FL:" << i << "_SI:" << j << "_TI:" << k;
           val = FD::Convert(os.str());
         }
       }
@@ -56,8 +57,24 @@ RelativeSentencePosition::RelativeSentencePosition(const string& param) :
   if (!param.empty()) {
     cerr << "  Loading word classes from " << param << endl;
     condition_on_fclass_ = true;
-    template_ = "RSP:FC000";
-    assert(!"not implemented");
+    ReadFile rf(param);
+    istream& in = *rf.stream();
+    set<WordID> classes;
+    while(in) {
+      string line;
+      getline(in, line);
+      if (line.empty()) continue;
+      vector<WordID> v;
+      TD::ConvertSentence(line, &v);
+      pos_.push_back(v);
+      for (int i = 0; i < v.size(); ++i)
+        classes.insert(v[i]);
+      for (set<WordID>::iterator i = classes.begin(); i != classes.end(); ++i) {
+        ostringstream os;
+        os << "RelPos_FC:" << TD::Convert(*i);
+        fids_[*i] = FD::Convert(os.str());
+      }
+    }
   } else {
     condition_on_fclass_ = false;
   }
@@ -79,17 +96,22 @@ void RelativeSentencePosition::TraversalFeaturesImpl(const SentenceMetadata& sme
                           static_cast<double>(edge.prev_i_) / smeta.GetTargetLength());
   features->set_value(fid_, val);
   if (condition_on_fclass_) {
-    assert(!"not implemented");
+    assert(smeta.GetSentenceID() < pos_.size());
+    const WordID cur_fclass = pos_[smeta.GetSentenceID()][edge.i_];
+    const int fid = fids_.find(cur_fclass)->second;
+    features->set_value(fid, val);
   }
 //  cerr << f_len_ << " " << e_len_ << " [" << edge.i_ << "," << edge.j_ << "|" << edge.prev_i_ << "," << edge.prev_j_ << "]\t" << edge.rule_->AsString() << "\tVAL=" << val << endl;
 }
 
 MarkovJumpFClass::MarkovJumpFClass(const string& param) :
-    FeatureFunction(1) {
+    FeatureFunction(1),
+    fids_(MAX_SENTENCE_SIZE) {
   cerr << "    MarkovJumpFClass" << endl;
   cerr << "Reading source POS tags from " << param << endl;
   ReadFile rf(param);
   istream& in = *rf.stream();
+  set<WordID> classes;
   while(in) {
     string line;
     getline(in, line);
@@ -97,8 +119,66 @@ MarkovJumpFClass::MarkovJumpFClass(const string& param) :
     vector<WordID> v;
     TD::ConvertSentence(line, &v);
     pos_.push_back(v);
+    for (int i = 0; i < v.size(); ++i)
+      classes.insert(v[i]);
   }
   cerr << "  (" << pos_.size() << " lines)\n";
+  cerr << "  Classes: " << classes.size() << endl;
+  for (int ss = 1; ss < MAX_SENTENCE_SIZE; ++ss) {
+    map<WordID, map<int, int> >& cfids = fids_[ss];
+    for (set<WordID>::iterator i = classes.begin(); i != classes.end(); ++i) {
+      map<int, int> &fids = cfids[*i];
+      for (int j = -ss; j <= ss; ++j) {
+        ostringstream os;
+        os << "Jump_FL:" << ss << "_FC:" << TD::Convert(*i) << "_J:" << j;
+        fids[j] = FD::Convert(os.str());
+      }
+    }
+  }
+}
+
+void MarkovJumpFClass::FireFeature(const SentenceMetadata& smeta,
+                                   int prev_src_pos,
+                                   int cur_src_pos,
+                                   SparseVector<double>* features) const {
+  const int jumpsize = cur_src_pos - prev_src_pos;
+  assert(smeta.GetSentenceID() < pos_.size());
+  const WordID cur_fclass = pos_[smeta.GetSentenceID()][cur_src_pos];
+  const int fid = fids_[smeta.GetSourceLength()].find(cur_fclass)->second.find(jumpsize)->second;
+  features->set_value(fid, 1.0);
+}
+
+void MarkovJumpFClass::FinalTraversalFeatures(const void* context,
+                                      SparseVector<double>* features) const {
+  int left_index = *static_cast<const unsigned char*>(context);
+//  int right_index = cur_flen;
+  // TODO
+}
+
+void MarkovJumpFClass::TraversalFeaturesImpl(const SentenceMetadata& smeta,
+                                     const Hypergraph::Edge& edge,
+                                     const std::vector<const void*>& ant_states,
+                                     SparseVector<double>* features,
+                                     SparseVector<double>* estimated_features,
+                                     void* state) const {
+  unsigned char& dpstate = *((unsigned char*)state);
+  if (edge.Arity() == 0) {
+    dpstate = static_cast<unsigned int>(edge.i_);
+  } else if (edge.Arity() == 1) {
+    dpstate = *((unsigned char*)ant_states[0]);
+  } else if (edge.Arity() == 2) {
+    int left_index = *((unsigned char*)ant_states[0]);
+    int right_index = *((unsigned char*)ant_states[1]);
+    if (right_index == -1)
+      dpstate = static_cast<unsigned int>(left_index);
+    else
+      dpstate = static_cast<unsigned int>(right_index);
+//    const WordID cur_fclass = pos_[smeta.GetSentenceID()][right_index];
+//    cerr << edge.i_ << "," << edge.j_ << ": fclass=" << TD::Convert(cur_fclass) << " j=" << jumpsize << endl;
+//    const int fid = fids_[smeta.GetSourceLength()].find(cur_fclass)->second.find(jumpsize)->second;
+//    features->set_value(fid, 1.0);
+    FireFeature(smeta, left_index, right_index, features);
+  }
 }
 
 MarkovJump::MarkovJump(const string& param) :
diff --git a/decoder/ff_wordalign.h b/decoder/ff_wordalign.h
index c5404887..688750de 100644
--- a/decoder/ff_wordalign.h
+++ b/decoder/ff_wordalign.h
@@ -19,7 +19,8 @@ class RelativeSentencePosition : public FeatureFunction {
  private:
   const int fid_;
   bool condition_on_fclass_;
-  std::string template_;
+  std::vector<std::vector<WordID> > pos_;
+  std::map<WordID, int> fids_;  // fclass -> fid
 };
 
 class Model2BinaryFeatures : public FeatureFunction {
@@ -66,10 +67,14 @@ class MarkovJumpFClass : public FeatureFunction {
                                      SparseVector<double>* features,
                                      SparseVector<double>* estimated_features,
                                      void* context) const;
- private:
-  void FireFeature(WordID src,
-                   WordID trg,
+
+  void FireFeature(const SentenceMetadata& smeta,
+                   int prev_src_pos,
+                   int cur_src_pos,
                    SparseVector<double>* features) const;
+
+ private:
+  std::vector<std::map<WordID, std::map<int, int> > > fids_;  // flen -> fclass -> jumpsize -> fid
   std::vector<std::vector<WordID> > pos_;
 };
 
diff --git a/tests/system_tests/unsup-align/cdec.ini b/tests/system_tests/unsup-align/cdec.ini
index 4016a201..37a37214 100644
--- a/tests/system_tests/unsup-align/cdec.ini
+++ b/tests/system_tests/unsup-align/cdec.ini
@@ -1,6 +1,6 @@
 aligner=true
 grammar=unsup-align.lex-grammar
-cubepruning_pop_limit=1000000
+intersection_strategy=full
 formalism=lexcrf
 feature_function=RelativeSentencePosition
 feature_function=MarkovJump
diff --git a/tests/system_tests/unsup-align/gold.statistics b/tests/system_tests/unsup-align/gold.statistics
index 2f37c2db..975c9d4e 100644
--- a/tests/system_tests/unsup-align/gold.statistics
+++ b/tests/system_tests/unsup-align/gold.statistics
@@ -90,7 +90,7 @@ constr_paths 4
 +lm_nodes 7
 +lm_edges 14
 +lm_paths 16
-+lm_trans end thet
++lm_trans thet thet
 constr_nodes 7
 constr_edges 10
 constr_paths 4
diff --git a/training/cluster-ptrain.pl b/training/cluster-ptrain.pl
index 7643d4e5..03122df9 100755
--- a/training/cluster-ptrain.pl
+++ b/training/cluster-ptrain.pl
@@ -29,10 +29,12 @@ my $PRIOR;
 my $OALG = "lbfgs";
 my $sigsq = 1;
 my $means_file;
+my $mem_buffers = 20;
 my $RESTART_IF_NECESSARY;
 GetOptions("cdec=s" => \$DECODER,
            "distributed" => \$DISTRIBUTED,
            "sigma_squared=f" => \$sigsq,
+           "lbfgs_memory_buffers=i" => \$mem_buffers,
            "max_iteration=i" => \$max_iteration,
            "means=s" => \$means_file,
            "optimizer=s" => \$OALG,
@@ -133,7 +135,7 @@ while ($iter < $max_iteration) {
   my $start = time;
   my $next_iter = $iter + 1;
   my $dec_cmd="$DECODER -G $CFLAG -c $config_file -w $dir/weights.$iter.gz < $training_corpus 2> $dir/deco.log.$iter";
-  my $opt_cmd = "$OPTIMIZER $PRIOR_FLAG -M 50 $OALG -s $dir/opt.state -i $dir/weights.$iter.gz -o $dir/weights.$next_iter.gz";
+  my $opt_cmd = "$OPTIMIZER $PRIOR_FLAG -M $mem_buffers $OALG -s $dir/opt.state -i $dir/weights.$iter.gz -o $dir/weights.$next_iter.gz";
   my $pcmd = "$PARALLEL -e $dir/err -p $pmem --nodelist \"$nodelist\" -- ";
   my $cmd = "";
   if ($parallel) { $cmd = $pcmd; }
@@ -183,6 +185,8 @@ Usage: $0 [OPTIONS] cdec.ini training.corpus weights.init
     --means FILE       if you want means other than 0
     --sigma_squared S  variance on prior
     --pmem MEM         Memory required for decoder
+    --lbfgs_memory_buffers Number of buffers to use
+                           with LBFGS optimizer
 
 EOT
 }
diff --git a/word-aligner/aligner.pl b/word-aligner/aligner.pl
index d203fc53..7eec0e42 100755
--- a/word-aligner/aligner.pl
+++ b/word-aligner/aligner.pl
@@ -6,15 +6,20 @@ use Getopt::Long;
 my $training_dir = "$SCRIPT_DIR/../training";
 die "Can't find training dir: $training_dir" unless -d $training_dir;
 
+my $mkcls = '/Users/redpony/software/giza/giza-pp/mkcls-v2/mkcls';
 my $num_classes = 50;
 my $nodes = 40;
 my $pmem = "2500mb";
 my $DECODER = "cdec";
 GetOptions("cdec=s" => \$DECODER,
            "jobs=i" => \$nodes,
-           "pmem=s" => \$pmem
+           "pmem=s" => \$pmem,
+           "mkcls=s" => \$mkcls,
           ) or usage();
 usage() unless (scalar @ARGV == 1);
+die "Cannot find mkcls (specify with --mkcls=/path/to/mkcls) at $mkcls\n" unless -f $mkcls;
+die "Cannot execute mkcls at $mkcls\n" unless -x $mkcls;
+
 my $in_file = shift @ARGV;
 die "Expected format corpus.l1-l2 where l1 & l2 are two-letter abbreviations\nfor the source and target language respectively\n" unless ($in_file =~ /^.+\.([a-z][a-z])-([a-z][a-z])$/);
 my $f_lang = $1;
@@ -22,13 +27,13 @@ my $e_lang = $2;
 
 print STDERR "Source language: $f_lang\n";
 print STDERR "Target language: $e_lang\n";
+print STDERR " Using mkcls in: $mkcls\n\n";
 die "Don't have an orthographic normalizer for $f_lang\n" unless -f "$SCRIPT_DIR/ortho-norm/$f_lang.pl";
 die "Don't have an orthographic normalizer for $e_lang\n" unless -f "$SCRIPT_DIR/ortho-norm/$e_lang.pl";
 
 my @stages = qw(nopos relpos markov);
 my @directions = qw(f-e e-f);
 
-my $mkcls = '/Users/redpony/software/giza/giza-pp/mkcls-v2/mkcls';
 my $corpus = 'c';
 
 my $cwd = getcwd();
@@ -75,7 +80,7 @@ NCLASSES = $num_classes
 
 TARGETS = @targets
 PTRAIN = \$(TRAINING_DIR)/cluster-ptrain.pl --restart_if_necessary
-PTRAIN_PARAMS = --gaussian_prior --sigma_squared 1.0 --max_iteration 5
+PTRAIN_PARAMS = --gaussian_prior --sigma_squared 1.0 --max_iteration 15
 
 export
 
@@ -95,12 +100,16 @@ clean:
 EOT
 close TOPLEVEL;
 
+print STDERR "Created alignment task. chdir to talign/ then type make.\n\n";
+exit 0;
+
 sub make_stage {
   my ($stage, $direction, $prev_stage) = @_;
   my $stage_dir = "$align_dir/$stage-$direction";
   my $first = $direction;
   $first =~ s/^(.+)-.*$/$1/;
   mkdir $stage_dir;
+  my $RELPOS = "feature_function=RelativeSentencePosition $align_dir/grammars/corpus.class.$first\n";
   open CDEC, ">$stage_dir/cdec.ini" or die;
   print CDEC <<EOT;
 formalism=lexcrf
@@ -108,10 +117,11 @@ intersection_strategy=full
 grammar=$align_dir/grammars/corpus.$direction.lex-grammar.gz
 EOT
   if ($stage =~ /relpos/) {
-    print CDEC "feature_function=RelativeSentencePosition\n";
+    print CDEC "$RELPOS\n";
   } elsif ($stage =~ /markov/) {
-    print CDEC "feature_function=RelativeSentencePosition\n";
+    print CDEC "$RELPOS\n";
     print CDEC "feature_function=MarkovJump\n";
+    print CDEC "feature_function=MarkovJumpFClass $align_dir/grammars/corpus.class.$first\n";
     print CDEC "feature_function=SourcePOSBigram $align_dir/grammars/corpus.class.$first\n";
   }
   close CDEC;
diff --git a/word-aligner/makefiles/makefile.grammars b/word-aligner/makefiles/makefile.grammars
index a6167010..b89937c1 100644
--- a/word-aligner/makefiles/makefile.grammars
+++ b/word-aligner/makefiles/makefile.grammars
@@ -3,18 +3,19 @@ all: corpus.f-e.lex-grammar.gz corpus.e-f.lex-grammar.gz corpus.class.e corpus.c
 clean:
 	$(RM) orthonorm-dict.* voc2class* corpus.class.* corpus.e-f corpus.f-e weights* corpus.f-e.lex-grammar* *.model1 *voc corpus.e-f.lex-grammar*
 
+SUPPORT_DIR = $(SCRIPT_DIR)/support
 GZIP = /usr/bin/gzip
 ZCAT = zcat
-EXTRACT_WEIGHTS = $(SCRIPT_DIR)/extract_weights.pl
-EXTRACT_GRAMMAR = $(SCRIPT_DIR)/extract_grammar.pl
-SUPPLEMENT_WEIGHTS = $(SCRIPT_DIR)/supplement_weights_file.pl
-EXTRACT_VOCAB = $(SCRIPT_DIR)/extract_vocab.pl
+EXTRACT_WEIGHTS = $(SUPPORT_DIR)/extract_weights.pl
+EXTRACT_GRAMMAR = $(SUPPORT_DIR)/extract_grammar.pl
+SUPPLEMENT_WEIGHTS = $(SUPPORT_DIR)/supplement_weights_file.pl
+EXTRACT_VOCAB = $(SUPPORT_DIR)/extract_vocab.pl
 ORTHONORM_E = $(SCRIPT_DIR)/ortho-norm/$(E_LANG).pl
 ORTHONORM_F = $(SCRIPT_DIR)/ortho-norm/$(F_LANG).pl
-CLASSIFY = $(SCRIPT_DIR)/classify.pl
-MAKE_LEX_GRAMMAR = $(SCRIPT_DIR)/make_lex_grammar.pl
+CLASSIFY = $(SUPPORT_DIR)/classify.pl
+MAKE_LEX_GRAMMAR = $(SUPPORT_DIR)/make_lex_grammar.pl
 MODEL1 = $(TRAINING_DIR)/model1
-MERGE_CORPUS = $(SCRIPT_DIR)/merge_corpus.pl
+MERGE_CORPUS = $(SUPPORT_DIR)/merge_corpus.pl
 
 orthonorm-dict.e: corpus.e
 	$(EXTRACT_VOCAB) corpus.e > e.voc
diff --git a/word-aligner/supplement_weights_file.pl b/word-aligner/supplement_weights_file.pl
deleted file mode 100755
index 76f668e2..00000000
--- a/word-aligner/supplement_weights_file.pl
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/perl -w
-use strict;
-
-my ($f_classes) = @ARGV;
-
-die "Usage: $0 f-classes.file" unless $f_classes && -f $f_classes;
-
-print <<EOT;
-MarkovJump 0
-RelativeSentencePosition 0
-EOT
-
-# !	8
-# "	11
-# 's	18
-
-my %dcats = ();
-$dcats{'BOS'} = 1;
-$dcats{'EOS'} = 1;
-
-open FC, "<$f_classes" or die;
-while(<FC>) {
-  chomp;
-  my ($x, $cat) = split /\s+/;
-  $dcats{$cat} = 1;
-}
-
-my @cats = sort keys %dcats;
-
-for (my $i=0; $i < scalar @cats; $i++) {
-  my $c1 = $cats[$i];
-  for (my $j=0; $j < scalar @cats; $j++) {
-    my $c2 = $cats[$j];
-    print "SP:${c1}_${c2} 0\n";
-  }
-}
-
diff --git a/word-aligner/classify.pl b/word-aligner/support/classify.pl
index 893c7b22..893c7b22 100755
--- a/word-aligner/classify.pl
+++ b/word-aligner/support/classify.pl
diff --git a/word-aligner/extract_grammar.pl b/word-aligner/support/extract_grammar.pl
index d7275ef5..d7275ef5 100755
--- a/word-aligner/extract_grammar.pl
+++ b/word-aligner/support/extract_grammar.pl
diff --git a/word-aligner/extract_vocab.pl b/word-aligner/support/extract_vocab.pl
index 070d4202..070d4202 100755
--- a/word-aligner/extract_vocab.pl
+++ b/word-aligner/support/extract_vocab.pl
diff --git a/word-aligner/extract_weights.pl b/word-aligner/support/extract_weights.pl
index dfedd12e..dfedd12e 100755
--- a/word-aligner/extract_weights.pl
+++ b/word-aligner/support/extract_weights.pl
diff --git a/word-aligner/invert_grammar.pl b/word-aligner/support/invert_grammar.pl
index 3988388d..3988388d 100755
--- a/word-aligner/invert_grammar.pl
+++ b/word-aligner/support/invert_grammar.pl
diff --git a/word-aligner/make_lex_grammar.pl b/word-aligner/support/make_lex_grammar.pl
index bdb2752c..bdb2752c 100755
--- a/word-aligner/make_lex_grammar.pl
+++ b/word-aligner/support/make_lex_grammar.pl
diff --git a/word-aligner/merge_corpus.pl b/word-aligner/support/merge_corpus.pl
index 02827903..02827903 100755
--- a/word-aligner/merge_corpus.pl
+++ b/word-aligner/support/merge_corpus.pl
diff --git a/word-aligner/support/supplement_weights_file.pl b/word-aligner/support/supplement_weights_file.pl
new file mode 100755
index 00000000..7f804b90
--- /dev/null
+++ b/word-aligner/support/supplement_weights_file.pl
@@ -0,0 +1,73 @@
+#!/usr/bin/perl -w
+use strict;
+
+my $ADD_FCLASS_JUMP = 1;
+my $ADD_MODEL2_BINARY = 0;
+my $ADD_FC_RELPOS = 1;
+
+my ($f_classes) = @ARGV;
+
+die "Usage: $0 f-classes.file" unless $f_classes && -f $f_classes;
+
+print <<EOT;
+MarkovJump 0
+RelativeSentencePosition 0
+EOT
+
+# !	8
+# "	11
+# 's	18
+
+my %dcats = ();
+$dcats{'BOS'} = 1;
+$dcats{'EOS'} = 1;
+
+open FC, "<$f_classes" or die;
+while(<FC>) {
+  chomp;
+  my ($x, $cat) = split /\s+/;
+  $dcats{$cat} = 1;
+}
+
+my @cats = sort keys %dcats;
+
+my $added = 0;
+for (my $i=0; $i < scalar @cats; $i++) {
+  my $c1 = $cats[$i];
+  for (my $j=0; $j < scalar @cats; $j++) {
+    my $c2 = $cats[$j];
+    print "SP:${c1}_${c2} 0\n";
+    $added++;
+  }
+}
+
+for (my $ss=1; $ss < 100; $ss++) {
+  if ($ADD_FCLASS_JUMP) {
+    for (my $i=0; $i < scalar @cats; $i++) {
+      my $cat = $cats[$i];
+      for (my $j = -$ss; $j <= $ss; $j++) {
+        print "Jump_FL:${ss}_FC:${cat}_J:$j 0\n";
+        $added++;
+      }
+    }
+  }
+  if ($ADD_MODEL2_BINARY) {
+    # M2_FL:8_SI:3_TI:2=1
+    for (my $i = 0; $i < $ss; $i++) {
+      for (my $j = 0; $j < 100; $j++) {
+        print "M2_FL:${ss}_SI:${i}_TI:${j} 0\n";
+        $added++;
+      }
+    }
+  }
+}
+if ($ADD_FC_RELPOS) {
+  #RelPos_FC:11
+  for (my $i=0; $i < scalar @cats; $i++) {
+    my $cat = $cats[$i];
+    print "RelPos_FC:$cat 0\n";
+    $added++;
+  }
+}
+
+print STDERR "Added $added weights\n";