Merge remote-tracking branch 'upstream/master'

5 files changed, 54 insertions, 45 deletions

diff --git a/klm/lm/left_test.cc b/klm/lm/left_test.cc
index b23e6a0f..b4561461 100644
--- a/klm/lm/left_test.cc
+++ b/klm/lm/left_test.cc
@@ -16,13 +16,16 @@ namespace {
 #define Term(word) score.Terminal(m.GetVocabulary().Index(word));
 #define VCheck(word, value) BOOST_CHECK_EQUAL(m.GetVocabulary().Index(word), value);
 
+// Apparently some Boost versions use templates and are pretty strict about types matching.  
+#define SLOPPY_CHECK_CLOSE(ref, value, tol) BOOST_CHECK_CLOSE(static_cast<double>(ref), static_cast<double>(value), static_cast<double>(tol));
+
 template <class M> void Short(const M &m) {
   ChartState base;
   {
     RuleScore<M> score(m, base);
     Term("more");
     Term("loin");
-    BOOST_CHECK_CLOSE(-1.206319 - 0.3561665, score.Finish(), 0.001);
+    SLOPPY_CHECK_CLOSE(-1.206319 - 0.3561665, score.Finish(), 0.001);
   }
   BOOST_CHECK(base.left.full);
   BOOST_CHECK_EQUAL(2, base.left.length);
@@ -35,7 +38,7 @@ template <class M> void Short(const M &m) {
     Term("little");
     score.NonTerminal(base, -1.206319 - 0.3561665);
     // p(little more loin | null context)
-    BOOST_CHECK_CLOSE(-1.56538, score.Finish(), 0.001);
+    SLOPPY_CHECK_CLOSE(-1.56538, score.Finish(), 0.001);
   }
   BOOST_CHECK_EQUAL(3, more_left.left.length);
   BOOST_CHECK_EQUAL(1, more_left.right.length);
@@ -47,7 +50,7 @@ template <class M> void Short(const M &m) {
     RuleScore<M> score(m, shorter);
     Term("to");
     score.NonTerminal(base, -1.206319 - 0.3561665);
-    BOOST_CHECK_CLOSE(-0.30103 - 1.687872 - 1.206319 - 0.3561665, score.Finish(), 0.01);
+    SLOPPY_CHECK_CLOSE(-0.30103 - 1.687872 - 1.206319 - 0.3561665, score.Finish(), 0.01);
   }
   BOOST_CHECK_EQUAL(1, shorter.left.length);
   BOOST_CHECK_EQUAL(1, shorter.right.length);
@@ -61,7 +64,7 @@ template <class M> void Charge(const M &m) {
     RuleScore<M> score(m, base);
     Term("on");
     Term("more");
-    BOOST_CHECK_CLOSE(-1.509559 -0.4771212 -1.206319, score.Finish(), 0.001);
+    SLOPPY_CHECK_CLOSE(-1.509559 -0.4771212 -1.206319, score.Finish(), 0.001);
   }
   BOOST_CHECK_EQUAL(1, base.left.length);
   BOOST_CHECK_EQUAL(1, base.right.length);
@@ -73,7 +76,7 @@ template <class M> void Charge(const M &m) {
     RuleScore<M> score(m, extend);
     Term("looking");
     score.NonTerminal(base, -1.509559 -0.4771212 -1.206319);
-    BOOST_CHECK_CLOSE(-3.91039, score.Finish(), 0.001);
+    SLOPPY_CHECK_CLOSE(-3.91039, score.Finish(), 0.001);
   }
   BOOST_CHECK_EQUAL(2, extend.left.length);
   BOOST_CHECK_EQUAL(1, extend.right.length);
@@ -85,7 +88,7 @@ template <class M> void Charge(const M &m) {
     RuleScore<M> score(m, tobos);
     score.BeginSentence();
     score.NonTerminal(extend, -3.91039);
-    BOOST_CHECK_CLOSE(-3.471169, score.Finish(), 0.001);
+    SLOPPY_CHECK_CLOSE(-3.471169, score.Finish(), 0.001);
   }
   BOOST_CHECK_EQUAL(0, tobos.left.length);
   BOOST_CHECK_EQUAL(1, tobos.right.length);
@@ -169,8 +172,8 @@ template <class M> void LookupVocab(const M &m, const StringPiece &str, std::vec
 #define TEXT_TEST(str) \
   LookupVocab(m, str, words); \
   expect = LeftToRight(m, words, rest); \
-  BOOST_CHECK_CLOSE(expect, RightToLeft(m, words, rest), 0.001); \
-  BOOST_CHECK_CLOSE(expect, TreeMiddle(m, words, rest), 0.001); \
+  SLOPPY_CHECK_CLOSE(expect, RightToLeft(m, words, rest), 0.001); \
+  SLOPPY_CHECK_CLOSE(expect, TreeMiddle(m, words, rest), 0.001); \
 
 // Build sentences, or parts thereof, from right to left.  
 template <class M> void GrowBig(const M &m, bool rest = false) {
@@ -202,20 +205,20 @@ template <class M> void AlsoWouldConsiderHigher(const M &m) {
   {
     RuleScore<M> score(m, also);
     score.Terminal(m.GetVocabulary().Index("also"));
-    BOOST_CHECK_CLOSE(-1.687872, score.Finish(), 0.001);
+    SLOPPY_CHECK_CLOSE(-1.687872, score.Finish(), 0.001);
   }
   ChartState would;
   {
     RuleScore<M> score(m, would);
     score.Terminal(m.GetVocabulary().Index("would"));
-    BOOST_CHECK_CLOSE(-1.687872, score.Finish(), 0.001);
+    SLOPPY_CHECK_CLOSE(-1.687872, score.Finish(), 0.001);
   }
   ChartState combine_also_would;
   {
     RuleScore<M> score(m, combine_also_would);
     score.NonTerminal(also, -1.687872);
     score.NonTerminal(would, -1.687872);
-    BOOST_CHECK_CLOSE(-1.687872 - 2.0, score.Finish(), 0.001);
+    SLOPPY_CHECK_CLOSE(-1.687872 - 2.0, score.Finish(), 0.001);
   }
   BOOST_CHECK_EQUAL(2, combine_also_would.right.length);
 
@@ -224,7 +227,7 @@ template <class M> void AlsoWouldConsiderHigher(const M &m) {
     RuleScore<M> score(m, also_would);
     score.Terminal(m.GetVocabulary().Index("also"));
     score.Terminal(m.GetVocabulary().Index("would"));
-    BOOST_CHECK_CLOSE(-1.687872 - 2.0, score.Finish(), 0.001);
+    SLOPPY_CHECK_CLOSE(-1.687872 - 2.0, score.Finish(), 0.001);
   }
   BOOST_CHECK_EQUAL(2, also_would.right.length);
 
@@ -232,7 +235,7 @@ template <class M> void AlsoWouldConsiderHigher(const M &m) {
   {
     RuleScore<M> score(m, consider);
     score.Terminal(m.GetVocabulary().Index("consider"));
-    BOOST_CHECK_CLOSE(-1.687872, score.Finish(), 0.001);
+    SLOPPY_CHECK_CLOSE(-1.687872, score.Finish(), 0.001);
   }
   BOOST_CHECK_EQUAL(1, consider.left.length);
   BOOST_CHECK_EQUAL(1, consider.right.length);
@@ -245,19 +248,19 @@ template <class M> void AlsoWouldConsiderHigher(const M &m) {
     score.Terminal(m.GetVocabulary().Index("higher"));
     higher_score = score.Finish();
   }
-  BOOST_CHECK_CLOSE(-1.509559, higher_score, 0.001);
+  SLOPPY_CHECK_CLOSE(-1.509559, higher_score, 0.001);
   BOOST_CHECK_EQUAL(1, higher.left.length);
   BOOST_CHECK_EQUAL(1, higher.right.length);
   BOOST_CHECK(!higher.left.full);
   VCheck("higher", higher.right.words[0]);
-  BOOST_CHECK_CLOSE(-0.30103, higher.right.backoff[0], 0.001);
+  SLOPPY_CHECK_CLOSE(-0.30103, higher.right.backoff[0], 0.001);
 
   ChartState consider_higher;
   {
     RuleScore<M> score(m, consider_higher);
     score.NonTerminal(consider, -1.687872);
     score.NonTerminal(higher, higher_score);
-    BOOST_CHECK_CLOSE(-1.509559 - 1.687872 - 0.30103, score.Finish(), 0.001);
+    SLOPPY_CHECK_CLOSE(-1.509559 - 1.687872 - 0.30103, score.Finish(), 0.001);
   }
   BOOST_CHECK_EQUAL(2, consider_higher.left.length);
   BOOST_CHECK(!consider_higher.left.full);
@@ -267,7 +270,7 @@ template <class M> void AlsoWouldConsiderHigher(const M &m) {
     RuleScore<M> score(m, full);
     score.NonTerminal(combine_also_would, -1.687872 - 2.0);
     score.NonTerminal(consider_higher, -1.509559 - 1.687872 - 0.30103);
-    BOOST_CHECK_CLOSE(-10.6879, score.Finish(), 0.001);
+    SLOPPY_CHECK_CLOSE(-10.6879, score.Finish(), 0.001);
   }
   BOOST_CHECK_EQUAL(4, full.right.length);
 }
@@ -277,7 +280,7 @@ template <class M> void AlsoWouldConsiderHigher(const M &m) {
   float got = val; \
   std::vector<WordIndex> indices; \
   LookupVocab(m, str, indices); \
-  BOOST_CHECK_CLOSE(LeftToRight(m, indices), got, 0.001); \
+  SLOPPY_CHECK_CLOSE(LeftToRight(m, indices), got, 0.001); \
 }
 
 template <class M> void FullGrow(const M &m) {
diff --git a/klm/lm/model_test.cc b/klm/lm/model_test.cc
index 8a122c60..32084b5b 100644
--- a/klm/lm/model_test.cc
+++ b/klm/lm/model_test.cc
@@ -6,6 +6,9 @@
 #include <boost/test/unit_test.hpp>
 #include <boost/test/floating_point_comparison.hpp>
 
+// Apparently some Boost versions use templates and are pretty strict about types matching.  
+#define SLOPPY_CHECK_CLOSE(ref, value, tol) BOOST_CHECK_CLOSE(static_cast<double>(ref), static_cast<double>(value), static_cast<double>(tol));
+
 namespace lm {
 namespace ngram {
 
@@ -46,7 +49,7 @@ template <class Model> State GetState(const Model &model, const char *word, cons
       state, \
       model.GetVocabulary().Index(word), \
       out);\
-  BOOST_CHECK_CLOSE(score, ret.prob, 0.001); \
+  SLOPPY_CHECK_CLOSE(score, ret.prob, 0.001); \
   BOOST_CHECK_EQUAL(static_cast<unsigned int>(ngram), ret.ngram_length); \
   BOOST_CHECK_GE(std::min<unsigned char>(ngram, 5 - 1), out.length); \
   BOOST_CHECK_EQUAL(indep_left, ret.independent_left); \
@@ -176,14 +179,14 @@ template <class M> void ExtendLeftTest(const M &model) {
   State right;
   FullScoreReturn little(model.FullScore(model.NullContextState(), model.GetVocabulary().Index("little"), right));
   const float kLittleProb = -1.285941;
-  BOOST_CHECK_CLOSE(kLittleProb, little.prob, 0.001);
+  SLOPPY_CHECK_CLOSE(kLittleProb, little.prob, 0.001);
   unsigned char next_use;
   float backoff_out[4];
 
   FullScoreReturn extend_none(model.ExtendLeft(NULL, NULL, NULL, little.extend_left, 1, NULL, next_use));
   BOOST_CHECK_EQUAL(0, next_use);
   BOOST_CHECK_EQUAL(little.extend_left, extend_none.extend_left);
-  BOOST_CHECK_CLOSE(little.prob - little.rest, extend_none.prob, 0.001);
+  SLOPPY_CHECK_CLOSE(little.prob - little.rest, extend_none.prob, 0.001);
   BOOST_CHECK_EQUAL(1, extend_none.ngram_length);
 
   const WordIndex a = model.GetVocabulary().Index("a");
@@ -191,16 +194,16 @@ template <class M> void ExtendLeftTest(const M &model) {
   // a little
   FullScoreReturn extend_a(model.ExtendLeft(&a, &a + 1, &backoff_in, little.extend_left, 1, backoff_out, next_use));
   BOOST_CHECK_EQUAL(1, next_use);
-  BOOST_CHECK_CLOSE(-0.69897, backoff_out[0], 0.001);
-  BOOST_CHECK_CLOSE(-0.09132547 - little.rest, extend_a.prob, 0.001);
+  SLOPPY_CHECK_CLOSE(-0.69897, backoff_out[0], 0.001);
+  SLOPPY_CHECK_CLOSE(-0.09132547 - little.rest, extend_a.prob, 0.001);
   BOOST_CHECK_EQUAL(2, extend_a.ngram_length);
   BOOST_CHECK(!extend_a.independent_left);
 
   const WordIndex on = model.GetVocabulary().Index("on");
   FullScoreReturn extend_on(model.ExtendLeft(&on, &on + 1, &backoff_in, extend_a.extend_left, 2, backoff_out, next_use));
   BOOST_CHECK_EQUAL(1, next_use);
-  BOOST_CHECK_CLOSE(-0.4771212, backoff_out[0], 0.001);
-  BOOST_CHECK_CLOSE(-0.0283603 - (extend_a.rest + little.rest), extend_on.prob, 0.001);
+  SLOPPY_CHECK_CLOSE(-0.4771212, backoff_out[0], 0.001);
+  SLOPPY_CHECK_CLOSE(-0.0283603 - (extend_a.rest + little.rest), extend_on.prob, 0.001);
   BOOST_CHECK_EQUAL(3, extend_on.ngram_length);
   BOOST_CHECK(!extend_on.independent_left);
 
@@ -208,9 +211,9 @@ template <class M> void ExtendLeftTest(const M &model) {
   float backoff_in_arr[4];
   FullScoreReturn extend_both(model.ExtendLeft(both, both + 2, backoff_in_arr, little.extend_left, 1, backoff_out, next_use));
   BOOST_CHECK_EQUAL(2, next_use);
-  BOOST_CHECK_CLOSE(-0.69897, backoff_out[0], 0.001);
-  BOOST_CHECK_CLOSE(-0.4771212, backoff_out[1], 0.001);
-  BOOST_CHECK_CLOSE(-0.0283603 - little.rest, extend_both.prob, 0.001);
+  SLOPPY_CHECK_CLOSE(-0.69897, backoff_out[0], 0.001);
+  SLOPPY_CHECK_CLOSE(-0.4771212, backoff_out[1], 0.001);
+  SLOPPY_CHECK_CLOSE(-0.0283603 - little.rest, extend_both.prob, 0.001);
   BOOST_CHECK_EQUAL(3, extend_both.ngram_length);
   BOOST_CHECK(!extend_both.independent_left);
   BOOST_CHECK_EQUAL(extend_on.extend_left, extend_both.extend_left);
@@ -218,12 +221,12 @@ template <class M> void ExtendLeftTest(const M &model) {
 
 #define StatelessTest(word, provide, ngram, score) \
   ret = model.FullScoreForgotState(indices + num_words - word, indices + num_words - word + provide, indices[num_words - word - 1], state); \
-  BOOST_CHECK_CLOSE(score, ret.prob, 0.001); \
+  SLOPPY_CHECK_CLOSE(score, ret.prob, 0.001); \
   BOOST_CHECK_EQUAL(static_cast<unsigned int>(ngram), ret.ngram_length); \
   model.GetState(indices + num_words - word, indices + num_words - word + provide, before); \
   ret = model.FullScore(before, indices[num_words - word - 1], out); \
   BOOST_CHECK(state == out); \
-  BOOST_CHECK_CLOSE(score, ret.prob, 0.001); \
+  SLOPPY_CHECK_CLOSE(score, ret.prob, 0.001); \
   BOOST_CHECK_EQUAL(static_cast<unsigned int>(ngram), ret.ngram_length);
 
 template <class M> void Stateless(const M &model) {
@@ -238,7 +241,7 @@ template <class M> void Stateless(const M &model) {
   State state, out, before;
 
   ret = model.FullScoreForgotState(indices + num_words - 1, indices + num_words, indices[num_words - 2], state);
-  BOOST_CHECK_CLOSE(-0.484652, ret.prob, 0.001);
+  SLOPPY_CHECK_CLOSE(-0.484652, ret.prob, 0.001);
   StatelessTest(1, 1, 2, -0.484652);
 
   // looking
@@ -276,7 +279,7 @@ template <class M> void NoUnkCheck(const M &model) {
   State state;
 
   FullScoreReturn ret = model.FullScoreForgotState(&unk_index, &unk_index + 1, unk_index, state);
-  BOOST_CHECK_CLOSE(-100.0, ret.prob, 0.001);
+  SLOPPY_CHECK_CLOSE(-100.0, ret.prob, 0.001);
 }
 
 template <class M> void Everything(const M &m) {
@@ -426,8 +429,8 @@ BOOST_AUTO_TEST_CASE(rest_max) {
   RestProbingModel model(TestLocation(), config);
   State state, out;
   FullScoreReturn ret(model.FullScore(model.NullContextState(), model.GetVocabulary().Index("."), state));
-  BOOST_CHECK_CLOSE(-0.2705918, ret.rest, 0.001);
-  BOOST_CHECK_CLOSE(-0.01916512, model.FullScore(state, model.GetVocabulary().EndSentence(), out).rest, 0.001);
+  SLOPPY_CHECK_CLOSE(-0.2705918, ret.rest, 0.001);
+  SLOPPY_CHECK_CLOSE(-0.01916512, model.FullScore(state, model.GetVocabulary().EndSentence(), out).rest, 0.001);
 }
 
 } // namespace
diff --git a/klm/util/probing_hash_table_test.cc b/klm/util/probing_hash_table_test.cc
index ef68e5f2..be0fa859 100644
--- a/klm/util/probing_hash_table_test.cc
+++ b/klm/util/probing_hash_table_test.cc
@@ -1,10 +1,12 @@
 #include "util/probing_hash_table.hh"
 
-#include <stdint.h>
-
 #define BOOST_TEST_MODULE ProbingHashTableTest
 #include <boost/test/unit_test.hpp>
+#include <boost/scoped_array.hpp>
 #include <boost/functional/hash.hpp>
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
 
 namespace util {
 namespace {
@@ -27,10 +29,11 @@ struct Entry {
 typedef ProbingHashTable<Entry, boost::hash<unsigned char> > Table;
 
 BOOST_AUTO_TEST_CASE(simple) {
-  char mem[Table::Size(10, 1.2)];
-  memset(mem, 0, sizeof(mem));
+  size_t size = Table::Size(10, 1.2);
+  boost::scoped_array<char> mem(new char[size]);
+  memset(mem.get(), 0, size);
 
-  Table table(mem, sizeof(mem));
+  Table table(mem.get(), size);
   const Entry *i = NULL;
   BOOST_CHECK(!table.Find(2, i));
   Entry to_ins;
diff --git a/python/pkg/cdec/sa/compile.py b/python/pkg/cdec/sa/compile.py
index 2a89243b..393c72a4 100644
--- a/python/pkg/cdec/sa/compile.py
+++ b/python/pkg/cdec/sa/compile.py
@@ -93,11 +93,11 @@ def main():
     
     # Write configuration
     config = cdec.configobj.ConfigObj(args.config, unrepr=True)
-    config['f_sa_file'] = f_sa_bin
-    config['e_file'] = e_bin
-    config['a_file'] = a_bin
-    config['lex_file'] = lex_bin
-    config['precompute_file'] = precomp_bin
+    config['f_sa_file'] = os.path.abspath(f_sa_bin)
+    config['e_file'] = os.path.abspath(e_bin)
+    config['a_file'] = os.path.abspath(a_bin)
+    config['lex_file'] = os.path.abspath(lex_bin)
+    config['precompute_file'] = os.path.abspath(precomp_bin)
     for name, value in zip(param_names, params):
         config[name] = value
     config.write()
diff --git a/python/pkg/cdec/sa/extract.py b/python/pkg/cdec/sa/extract.py
index 918aa3bb..875bf42e 100644
--- a/python/pkg/cdec/sa/extract.py
+++ b/python/pkg/cdec/sa/extract.py
@@ -25,7 +25,7 @@ def main():
             for rule in extractor.grammar(sentence):
                 output.write(str(rule)+'\n')
         grammar_file = os.path.abspath(grammar_file)
-        print('<seg grammar="{0}">{1}</seg>'.format(grammar_file, sentence))
+        print('<seg grammar="{0}" id="{1}">{2}</seg>'.format(grammar_file, i, sentence))
 
 if __name__ == '__main__':
     main()