From e5d6f4ae41009c26978ecd62668501af9762b0bc Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Fri, 3 Aug 2012 07:46:33 -0400 Subject: Zero-initialize newed memory in a test --- klm/util/probing_hash_table_test.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/klm/util/probing_hash_table_test.cc b/klm/util/probing_hash_table_test.cc index ef68e5f2..37ffeb53 100644 --- a/klm/util/probing_hash_table_test.cc +++ b/klm/util/probing_hash_table_test.cc @@ -1,11 +1,12 @@ #include "util/probing_hash_table.hh" -#include - #define BOOST_TEST_MODULE ProbingHashTableTest #include +#include #include +#include + namespace util { namespace { @@ -27,10 +28,11 @@ struct Entry { typedef ProbingHashTable > Table; BOOST_AUTO_TEST_CASE(simple) { - char mem[Table::Size(10, 1.2)]; - memset(mem, 0, sizeof(mem)); + size_t size = Table::Size(10, 1.2); + boost::scoped_array mem(new char[size]); + memset(mem.get(), 0, size); - Table table(mem, sizeof(mem)); + Table table(mem.get(), size); const Entry *i = NULL; BOOST_CHECK(!table.Find(2, i)); Entry to_ins; -- cgit v1.2.3 From 611501431a53e507cdc15bb091d3bcc94ca7f39d Mon Sep 17 00:00:00 2001 From: Victor Chahuneau Date: Fri, 3 Aug 2012 14:55:18 -0400 Subject: [python] Fix some details in cdec.sa tools --- python/pkg/cdec/sa/compile.py | 10 +++++----- python/pkg/cdec/sa/extract.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/pkg/cdec/sa/compile.py b/python/pkg/cdec/sa/compile.py index 2a89243b..393c72a4 100644 --- a/python/pkg/cdec/sa/compile.py +++ b/python/pkg/cdec/sa/compile.py @@ -93,11 +93,11 @@ def main(): # Write configuration config = cdec.configobj.ConfigObj(args.config, unrepr=True) - config['f_sa_file'] = f_sa_bin - config['e_file'] = e_bin - config['a_file'] = a_bin - config['lex_file'] = lex_bin - config['precompute_file'] = precomp_bin + config['f_sa_file'] = os.path.abspath(f_sa_bin) + config['e_file'] = os.path.abspath(e_bin) + config['a_file'] = os.path.abspath(a_bin) + config['lex_file'] = os.path.abspath(lex_bin) + config['precompute_file'] = os.path.abspath(precomp_bin) for name, value in zip(param_names, params): config[name] = value config.write() diff --git a/python/pkg/cdec/sa/extract.py b/python/pkg/cdec/sa/extract.py index 918aa3bb..875bf42e 100644 --- a/python/pkg/cdec/sa/extract.py +++ b/python/pkg/cdec/sa/extract.py @@ -25,7 +25,7 @@ def main(): for rule in extractor.grammar(sentence): output.write(str(rule)+'\n') grammar_file = os.path.abspath(grammar_file) - print('{1}'.format(grammar_file, sentence)) + print('{2}'.format(grammar_file, i, sentence)) if __name__ == '__main__': main() -- cgit v1.2.3 From 461c632c09b776955737c0390d8ba34b7e41d081 Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Mon, 6 Aug 2012 08:19:00 -0400 Subject: More headers --- klm/util/probing_hash_table_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/klm/util/probing_hash_table_test.cc b/klm/util/probing_hash_table_test.cc index 37ffeb53..be0fa859 100644 --- a/klm/util/probing_hash_table_test.cc +++ b/klm/util/probing_hash_table_test.cc @@ -4,7 +4,8 @@ #include #include #include - +#include +#include #include namespace util { -- cgit v1.2.3 From 9b96ffa040f6c05c4e597a34c316743425839fe0 Mon Sep 17 00:00:00 2001 From: Kenneth Heafield Date: Mon, 6 Aug 2012 08:35:23 -0400 Subject: Fix test compilation for Yvette's gcc and Boost combination --- klm/lm/left_test.cc | 39 +++++++++++++++++++++------------------ klm/lm/model_test.cc | 35 +++++++++++++++++++---------------- 2 files changed, 40 insertions(+), 34 deletions(-) diff --git a/klm/lm/left_test.cc b/klm/lm/left_test.cc index b23e6a0f..b4561461 100644 --- a/klm/lm/left_test.cc +++ b/klm/lm/left_test.cc @@ -16,13 +16,16 @@ namespace { #define Term(word) score.Terminal(m.GetVocabulary().Index(word)); #define VCheck(word, value) BOOST_CHECK_EQUAL(m.GetVocabulary().Index(word), value); +// Apparently some Boost versions use templates and are pretty strict about types matching. +#define SLOPPY_CHECK_CLOSE(ref, value, tol) BOOST_CHECK_CLOSE(static_cast(ref), static_cast(value), static_cast(tol)); + template void Short(const M &m) { ChartState base; { RuleScore score(m, base); Term("more"); Term("loin"); - BOOST_CHECK_CLOSE(-1.206319 - 0.3561665, score.Finish(), 0.001); + SLOPPY_CHECK_CLOSE(-1.206319 - 0.3561665, score.Finish(), 0.001); } BOOST_CHECK(base.left.full); BOOST_CHECK_EQUAL(2, base.left.length); @@ -35,7 +38,7 @@ template void Short(const M &m) { Term("little"); score.NonTerminal(base, -1.206319 - 0.3561665); // p(little more loin | null context) - BOOST_CHECK_CLOSE(-1.56538, score.Finish(), 0.001); + SLOPPY_CHECK_CLOSE(-1.56538, score.Finish(), 0.001); } BOOST_CHECK_EQUAL(3, more_left.left.length); BOOST_CHECK_EQUAL(1, more_left.right.length); @@ -47,7 +50,7 @@ template void Short(const M &m) { RuleScore score(m, shorter); Term("to"); score.NonTerminal(base, -1.206319 - 0.3561665); - BOOST_CHECK_CLOSE(-0.30103 - 1.687872 - 1.206319 - 0.3561665, score.Finish(), 0.01); + SLOPPY_CHECK_CLOSE(-0.30103 - 1.687872 - 1.206319 - 0.3561665, score.Finish(), 0.01); } BOOST_CHECK_EQUAL(1, shorter.left.length); BOOST_CHECK_EQUAL(1, shorter.right.length); @@ -61,7 +64,7 @@ template void Charge(const M &m) { RuleScore score(m, base); Term("on"); Term("more"); - BOOST_CHECK_CLOSE(-1.509559 -0.4771212 -1.206319, score.Finish(), 0.001); + SLOPPY_CHECK_CLOSE(-1.509559 -0.4771212 -1.206319, score.Finish(), 0.001); } BOOST_CHECK_EQUAL(1, base.left.length); BOOST_CHECK_EQUAL(1, base.right.length); @@ -73,7 +76,7 @@ template void Charge(const M &m) { RuleScore score(m, extend); Term("looking"); score.NonTerminal(base, -1.509559 -0.4771212 -1.206319); - BOOST_CHECK_CLOSE(-3.91039, score.Finish(), 0.001); + SLOPPY_CHECK_CLOSE(-3.91039, score.Finish(), 0.001); } BOOST_CHECK_EQUAL(2, extend.left.length); BOOST_CHECK_EQUAL(1, extend.right.length); @@ -85,7 +88,7 @@ template void Charge(const M &m) { RuleScore score(m, tobos); score.BeginSentence(); score.NonTerminal(extend, -3.91039); - BOOST_CHECK_CLOSE(-3.471169, score.Finish(), 0.001); + SLOPPY_CHECK_CLOSE(-3.471169, score.Finish(), 0.001); } BOOST_CHECK_EQUAL(0, tobos.left.length); BOOST_CHECK_EQUAL(1, tobos.right.length); @@ -169,8 +172,8 @@ template void LookupVocab(const M &m, const StringPiece &str, std::vec #define TEXT_TEST(str) \ LookupVocab(m, str, words); \ expect = LeftToRight(m, words, rest); \ - BOOST_CHECK_CLOSE(expect, RightToLeft(m, words, rest), 0.001); \ - BOOST_CHECK_CLOSE(expect, TreeMiddle(m, words, rest), 0.001); \ + SLOPPY_CHECK_CLOSE(expect, RightToLeft(m, words, rest), 0.001); \ + SLOPPY_CHECK_CLOSE(expect, TreeMiddle(m, words, rest), 0.001); \ // Build sentences, or parts thereof, from right to left. template void GrowBig(const M &m, bool rest = false) { @@ -202,20 +205,20 @@ template void AlsoWouldConsiderHigher(const M &m) { { RuleScore score(m, also); score.Terminal(m.GetVocabulary().Index("also")); - BOOST_CHECK_CLOSE(-1.687872, score.Finish(), 0.001); + SLOPPY_CHECK_CLOSE(-1.687872, score.Finish(), 0.001); } ChartState would; { RuleScore score(m, would); score.Terminal(m.GetVocabulary().Index("would")); - BOOST_CHECK_CLOSE(-1.687872, score.Finish(), 0.001); + SLOPPY_CHECK_CLOSE(-1.687872, score.Finish(), 0.001); } ChartState combine_also_would; { RuleScore score(m, combine_also_would); score.NonTerminal(also, -1.687872); score.NonTerminal(would, -1.687872); - BOOST_CHECK_CLOSE(-1.687872 - 2.0, score.Finish(), 0.001); + SLOPPY_CHECK_CLOSE(-1.687872 - 2.0, score.Finish(), 0.001); } BOOST_CHECK_EQUAL(2, combine_also_would.right.length); @@ -224,7 +227,7 @@ template void AlsoWouldConsiderHigher(const M &m) { RuleScore score(m, also_would); score.Terminal(m.GetVocabulary().Index("also")); score.Terminal(m.GetVocabulary().Index("would")); - BOOST_CHECK_CLOSE(-1.687872 - 2.0, score.Finish(), 0.001); + SLOPPY_CHECK_CLOSE(-1.687872 - 2.0, score.Finish(), 0.001); } BOOST_CHECK_EQUAL(2, also_would.right.length); @@ -232,7 +235,7 @@ template void AlsoWouldConsiderHigher(const M &m) { { RuleScore score(m, consider); score.Terminal(m.GetVocabulary().Index("consider")); - BOOST_CHECK_CLOSE(-1.687872, score.Finish(), 0.001); + SLOPPY_CHECK_CLOSE(-1.687872, score.Finish(), 0.001); } BOOST_CHECK_EQUAL(1, consider.left.length); BOOST_CHECK_EQUAL(1, consider.right.length); @@ -245,19 +248,19 @@ template void AlsoWouldConsiderHigher(const M &m) { score.Terminal(m.GetVocabulary().Index("higher")); higher_score = score.Finish(); } - BOOST_CHECK_CLOSE(-1.509559, higher_score, 0.001); + SLOPPY_CHECK_CLOSE(-1.509559, higher_score, 0.001); BOOST_CHECK_EQUAL(1, higher.left.length); BOOST_CHECK_EQUAL(1, higher.right.length); BOOST_CHECK(!higher.left.full); VCheck("higher", higher.right.words[0]); - BOOST_CHECK_CLOSE(-0.30103, higher.right.backoff[0], 0.001); + SLOPPY_CHECK_CLOSE(-0.30103, higher.right.backoff[0], 0.001); ChartState consider_higher; { RuleScore score(m, consider_higher); score.NonTerminal(consider, -1.687872); score.NonTerminal(higher, higher_score); - BOOST_CHECK_CLOSE(-1.509559 - 1.687872 - 0.30103, score.Finish(), 0.001); + SLOPPY_CHECK_CLOSE(-1.509559 - 1.687872 - 0.30103, score.Finish(), 0.001); } BOOST_CHECK_EQUAL(2, consider_higher.left.length); BOOST_CHECK(!consider_higher.left.full); @@ -267,7 +270,7 @@ template void AlsoWouldConsiderHigher(const M &m) { RuleScore score(m, full); score.NonTerminal(combine_also_would, -1.687872 - 2.0); score.NonTerminal(consider_higher, -1.509559 - 1.687872 - 0.30103); - BOOST_CHECK_CLOSE(-10.6879, score.Finish(), 0.001); + SLOPPY_CHECK_CLOSE(-10.6879, score.Finish(), 0.001); } BOOST_CHECK_EQUAL(4, full.right.length); } @@ -277,7 +280,7 @@ template void AlsoWouldConsiderHigher(const M &m) { float got = val; \ std::vector indices; \ LookupVocab(m, str, indices); \ - BOOST_CHECK_CLOSE(LeftToRight(m, indices), got, 0.001); \ + SLOPPY_CHECK_CLOSE(LeftToRight(m, indices), got, 0.001); \ } template void FullGrow(const M &m) { diff --git a/klm/lm/model_test.cc b/klm/lm/model_test.cc index 8a122c60..32084b5b 100644 --- a/klm/lm/model_test.cc +++ b/klm/lm/model_test.cc @@ -6,6 +6,9 @@ #include #include +// Apparently some Boost versions use templates and are pretty strict about types matching. +#define SLOPPY_CHECK_CLOSE(ref, value, tol) BOOST_CHECK_CLOSE(static_cast(ref), static_cast(value), static_cast(tol)); + namespace lm { namespace ngram { @@ -46,7 +49,7 @@ template State GetState(const Model &model, const char *word, cons state, \ model.GetVocabulary().Index(word), \ out);\ - BOOST_CHECK_CLOSE(score, ret.prob, 0.001); \ + SLOPPY_CHECK_CLOSE(score, ret.prob, 0.001); \ BOOST_CHECK_EQUAL(static_cast(ngram), ret.ngram_length); \ BOOST_CHECK_GE(std::min(ngram, 5 - 1), out.length); \ BOOST_CHECK_EQUAL(indep_left, ret.independent_left); \ @@ -176,14 +179,14 @@ template void ExtendLeftTest(const M &model) { State right; FullScoreReturn little(model.FullScore(model.NullContextState(), model.GetVocabulary().Index("little"), right)); const float kLittleProb = -1.285941; - BOOST_CHECK_CLOSE(kLittleProb, little.prob, 0.001); + SLOPPY_CHECK_CLOSE(kLittleProb, little.prob, 0.001); unsigned char next_use; float backoff_out[4]; FullScoreReturn extend_none(model.ExtendLeft(NULL, NULL, NULL, little.extend_left, 1, NULL, next_use)); BOOST_CHECK_EQUAL(0, next_use); BOOST_CHECK_EQUAL(little.extend_left, extend_none.extend_left); - BOOST_CHECK_CLOSE(little.prob - little.rest, extend_none.prob, 0.001); + SLOPPY_CHECK_CLOSE(little.prob - little.rest, extend_none.prob, 0.001); BOOST_CHECK_EQUAL(1, extend_none.ngram_length); const WordIndex a = model.GetVocabulary().Index("a"); @@ -191,16 +194,16 @@ template void ExtendLeftTest(const M &model) { // a little FullScoreReturn extend_a(model.ExtendLeft(&a, &a + 1, &backoff_in, little.extend_left, 1, backoff_out, next_use)); BOOST_CHECK_EQUAL(1, next_use); - BOOST_CHECK_CLOSE(-0.69897, backoff_out[0], 0.001); - BOOST_CHECK_CLOSE(-0.09132547 - little.rest, extend_a.prob, 0.001); + SLOPPY_CHECK_CLOSE(-0.69897, backoff_out[0], 0.001); + SLOPPY_CHECK_CLOSE(-0.09132547 - little.rest, extend_a.prob, 0.001); BOOST_CHECK_EQUAL(2, extend_a.ngram_length); BOOST_CHECK(!extend_a.independent_left); const WordIndex on = model.GetVocabulary().Index("on"); FullScoreReturn extend_on(model.ExtendLeft(&on, &on + 1, &backoff_in, extend_a.extend_left, 2, backoff_out, next_use)); BOOST_CHECK_EQUAL(1, next_use); - BOOST_CHECK_CLOSE(-0.4771212, backoff_out[0], 0.001); - BOOST_CHECK_CLOSE(-0.0283603 - (extend_a.rest + little.rest), extend_on.prob, 0.001); + SLOPPY_CHECK_CLOSE(-0.4771212, backoff_out[0], 0.001); + SLOPPY_CHECK_CLOSE(-0.0283603 - (extend_a.rest + little.rest), extend_on.prob, 0.001); BOOST_CHECK_EQUAL(3, extend_on.ngram_length); BOOST_CHECK(!extend_on.independent_left); @@ -208,9 +211,9 @@ template void ExtendLeftTest(const M &model) { float backoff_in_arr[4]; FullScoreReturn extend_both(model.ExtendLeft(both, both + 2, backoff_in_arr, little.extend_left, 1, backoff_out, next_use)); BOOST_CHECK_EQUAL(2, next_use); - BOOST_CHECK_CLOSE(-0.69897, backoff_out[0], 0.001); - BOOST_CHECK_CLOSE(-0.4771212, backoff_out[1], 0.001); - BOOST_CHECK_CLOSE(-0.0283603 - little.rest, extend_both.prob, 0.001); + SLOPPY_CHECK_CLOSE(-0.69897, backoff_out[0], 0.001); + SLOPPY_CHECK_CLOSE(-0.4771212, backoff_out[1], 0.001); + SLOPPY_CHECK_CLOSE(-0.0283603 - little.rest, extend_both.prob, 0.001); BOOST_CHECK_EQUAL(3, extend_both.ngram_length); BOOST_CHECK(!extend_both.independent_left); BOOST_CHECK_EQUAL(extend_on.extend_left, extend_both.extend_left); @@ -218,12 +221,12 @@ template void ExtendLeftTest(const M &model) { #define StatelessTest(word, provide, ngram, score) \ ret = model.FullScoreForgotState(indices + num_words - word, indices + num_words - word + provide, indices[num_words - word - 1], state); \ - BOOST_CHECK_CLOSE(score, ret.prob, 0.001); \ + SLOPPY_CHECK_CLOSE(score, ret.prob, 0.001); \ BOOST_CHECK_EQUAL(static_cast(ngram), ret.ngram_length); \ model.GetState(indices + num_words - word, indices + num_words - word + provide, before); \ ret = model.FullScore(before, indices[num_words - word - 1], out); \ BOOST_CHECK(state == out); \ - BOOST_CHECK_CLOSE(score, ret.prob, 0.001); \ + SLOPPY_CHECK_CLOSE(score, ret.prob, 0.001); \ BOOST_CHECK_EQUAL(static_cast(ngram), ret.ngram_length); template void Stateless(const M &model) { @@ -238,7 +241,7 @@ template void Stateless(const M &model) { State state, out, before; ret = model.FullScoreForgotState(indices + num_words - 1, indices + num_words, indices[num_words - 2], state); - BOOST_CHECK_CLOSE(-0.484652, ret.prob, 0.001); + SLOPPY_CHECK_CLOSE(-0.484652, ret.prob, 0.001); StatelessTest(1, 1, 2, -0.484652); // looking @@ -276,7 +279,7 @@ template void NoUnkCheck(const M &model) { State state; FullScoreReturn ret = model.FullScoreForgotState(&unk_index, &unk_index + 1, unk_index, state); - BOOST_CHECK_CLOSE(-100.0, ret.prob, 0.001); + SLOPPY_CHECK_CLOSE(-100.0, ret.prob, 0.001); } template void Everything(const M &m) { @@ -426,8 +429,8 @@ BOOST_AUTO_TEST_CASE(rest_max) { RestProbingModel model(TestLocation(), config); State state, out; FullScoreReturn ret(model.FullScore(model.NullContextState(), model.GetVocabulary().Index("."), state)); - BOOST_CHECK_CLOSE(-0.2705918, ret.rest, 0.001); - BOOST_CHECK_CLOSE(-0.01916512, model.FullScore(state, model.GetVocabulary().EndSentence(), out).rest, 0.001); + SLOPPY_CHECK_CLOSE(-0.2705918, ret.rest, 0.001); + SLOPPY_CHECK_CLOSE(-0.01916512, model.FullScore(state, model.GetVocabulary().EndSentence(), out).rest, 0.001); } } // namespace -- cgit v1.2.3