cfg test

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@552 ec762483-ff6d-05da-a07a-a48fb63a330f

9 files changed, 223 insertions, 76 deletions

diff --git a/decoder/Makefile.am b/decoder/Makefile.am
index 86912b49..0c434665 100644
--- a/decoder/Makefile.am
+++ b/decoder/Makefile.am
@@ -6,13 +6,16 @@ noinst_PROGRAMS = \
   hg_test \
   ff_test \
   parser_test \
-  grammar_test
-TESTS = trule_test ff_test parser_test grammar_test hg_test
+  grammar_test \
+  cfg_test
+TESTS = trule_test ff_test parser_test grammar_test hg_test cfg_test
 endif
 
 cdec_SOURCES = cdec.cc forest_writer.cc maxtrans_blunsom.cc cdec_ff.cc
 cdec_LDADD = libcdec.a ../mteval/libmteval.a ../utils/libutils.a -lz
 
+cfg_test_SOURCES = cfg_test.cc
+cfg_test_LDADD = $(GTEST_LDFLAGS) $(GTEST_LIBS) libcdec.a ../mteval/libmteval.a ../utils/libutils.a -lz
 parser_test_SOURCES = parser_test.cc
 parser_test_LDADD = $(GTEST_LDFLAGS) $(GTEST_LIBS) libcdec.a ../mteval/libmteval.a ../utils/libutils.a -lz
 ff_test_SOURCES = ff_test.cc
diff --git a/decoder/cfg.h b/decoder/cfg.h
index b214d8dc..21e03e2c 100755
--- a/decoder/cfg.h
+++ b/decoder/cfg.h
@@ -34,11 +34,15 @@
 #include "nt_span.h"
 #include <algorithm>
 #include "indices_after.h"
+#include <boost/functional/hash.hpp>
 
 class Hypergraph;
 class CFGFormat; // #include "cfg_format.h"
 class CFGBinarize; // #include "cfg_binarize.h"
 
+#undef CFG_MUST_EQ
+#define CFG_MUST_EQ(f) if (!(o.f==f)) return false;
+
 struct CFG {
   typedef int RuleHandle;
   typedef int NTHandle;
@@ -52,6 +56,23 @@ struct CFG {
   typedef std::pair<int,int> BinRhs;
 
   struct Rule {
+    std::size_t hash_impl() const {
+      using namespace boost;
+      std::size_t h=lhs;
+      hash_combine(h,rhs);
+      hash_combine(h,p);
+      hash_combine(h,f);
+      return h;
+    }
+    bool operator ==(Rule const &o) const {
+      CFG_MUST_EQ(lhs)
+      CFG_MUST_EQ(rhs)
+      CFG_MUST_EQ(p)
+      CFG_MUST_EQ(f)
+      return true;
+    }
+    inline bool operator!=(Rule const& o) const { return !(o==*this); }
+
     // for binarizing - no costs/probs
     Rule() : lhs(-1) {  }
     bool is_null() const { return lhs<0; }
@@ -123,6 +144,11 @@ struct CFG {
   };
 
   struct NT {
+    std::size_t hash_impl() const { using namespace boost; return hash_value(ruleids); }
+    bool operator ==(NT const &o) const {
+      return ruleids==o.ruleids; // don't care about from
+    }
+    inline bool operator!=(NT const& o) const { return !(o==*this); }
     Ruleids ruleids; // index into CFG rules with lhs = this NT.  aka in_edges_
     NTSpan from; // optional name - still needs id to disambiguate
     void Swap(NT &o) {
@@ -166,6 +192,18 @@ struct CFG {
     swap(goal_nt,o.goal_nt);
   }
 
+  //NOTE: this checks exact equality of data structures only.  it's well known that CFG equivalence (and intersection==empty) test is undecidable.
+  bool operator ==(CFG const &o) const {
+    // doesn't matter: hg, goal_inside
+    CFG_MUST_EQ(uninit)
+    CFG_MUST_EQ(pushed_inside)
+    CFG_MUST_EQ(goal_nt)
+    CFG_MUST_EQ(nts)
+    CFG_MUST_EQ(rules)
+    return true;
+  }
+  inline bool operator!=(CFG const& o) const { return !(o==*this); }
+
   typedef std::vector<NTHandle> NTOrder; // a list of nts, in definition-before-use order.
 
   //perhaps: give up on templated Order and move the below to .cc (NTOrder should be fine)
@@ -251,6 +289,15 @@ protected:
   // rules/nts will have same index as hg edges/nodes
 };
 
+inline std::size_t hash_value(CFG::Rule const& r) {
+  return r.hash_impl();
+}
+
+inline std::size_t hash_value(CFG::NT const& r) {
+  return r.hash_impl();
+}
+
+
 inline void swap(CFG &a,CFG &b) {
   a.Swap(b);
 }
diff --git a/decoder/cfg_options.h b/decoder/cfg_options.h
index 10321a7f..5dca168e 100755
--- a/decoder/cfg_options.h
+++ b/decoder/cfg_options.h
@@ -9,19 +9,19 @@
 struct CFGOptions {
   CFGFormat format;
   CFGBinarize binarize;
-  std::string cfg_output,source_cfg_output,cfg_unbin_output;
+  std::string out,source_out,unbin_out;
   void set_defaults() {
     format.set_defaults();
     binarize.set_defaults();
-    cfg_output="";
+    out="";
   }
   CFGOptions() { set_defaults(); }
   template <class Opts> // template to support both printable_opts and boost nonprintable
   void AddOptions(Opts *opts) {
     opts->add_options()
-      ("cfg_output", defaulted_value(&cfg_output),"write final target CFG (before FSA rescoring) to this file")
-      ("source_cfg_output", defaulted_value(&source_cfg_output),"write source CFG (after prelm-scoring, prelm-prune) to this file")
-      ("cfg_unbin_output", defaulted_value(&cfg_unbin_output),"write pre-binarization CFG to this file") //TODO:
+      ("cfg_output", defaulted_value(&out),"write final target CFG (before FSA rescoring) to this file")
+      ("source_cfg_output", defaulted_value(&source_out),"write source CFG (after prelm-scoring, prelm-prune) to this file")
+      ("cfg_unbin_output", defaulted_value(&unbin_out),"write pre-binarization CFG to this file") //TODO:
     ;
     binarize.AddOptions(opts);
     format.AddOptions(opts);
@@ -29,28 +29,36 @@ struct CFGOptions {
   void Validate() {
     format.Validate();
     binarize.Validate();
-//    if (cfg_output.empty()) binarize.bin_name_nts=false;
+//    if (out.empty()) binarize.bin_name_nts=false;
   }
   char const* description() const {
     return "CFG output options";
   }
   void maybe_output_source(Hypergraph const& hg) {
-    if (source_cfg_output.empty()) return;
-    std::cerr<<"Printing source CFG to "<<source_cfg_output<<": "<<format<<'\n';
-    WriteFile o(source_cfg_output);
+    if (source_out.empty()) return;
+    std::cerr<<"Printing source CFG to "<<source_out<<": "<<format<<'\n';
+    WriteFile o(source_out);
     CFG cfg(hg,false,format.features,format.goal_nt());
     cfg.Print(o.get(),format);
   }
+  void maybe_print(CFG &cfg,std::string cfg_output,char const* desc=" unbinarized") {
+      WriteFile o(cfg_output);
+      std::cerr<<"Printing target"<<desc<<" CFG to "<<cfg_output<<": "<<format<<'\n';
+      cfg.Print(o.get(),format);
+  }
+
   void maybe_output(HgCFG &hgcfg) {
-    if (cfg_output.empty()) return;
-    std::cerr<<"Printing target CFG to "<<cfg_output<<": "<<format<<'\n';
-    WriteFile o(cfg_output);
+    if (out.empty() && unbin_out.empty()) return;
+    CFG &cfg=hgcfg.GetCFG();
+    maybe_print(cfg,unbin_out);
     maybe_binarize(hgcfg);
-    hgcfg.GetCFG().Print(o.get(),format);
+    maybe_print(cfg,out,"");
   }
+
   void maybe_binarize(HgCFG &hgcfg) {
     if (hgcfg.binarized) return;
-    hgcfg.GetCFG().Binarize(binarize);
+    CFG &cfg=hgcfg.GetCFG();
+    cfg.Binarize(binarize);
     hgcfg.binarized=true;
   }
 };
diff --git a/decoder/cfg_test.cc b/decoder/cfg_test.cc
new file mode 100755
index 00000000..c4c37a2c
--- /dev/null
+++ b/decoder/cfg_test.cc
@@ -0,0 +1,39 @@
+#include <gtest/gtest.h>
+#include "cfg.h"
+#include "hg_test.h"
+#include "cfg_options.h"
+
+struct CFGTest : public HGSetup {
+  CFGTest() {  }
+  ~CFGTest() {  }
+  static void JsonFN(Hypergraph hg,CFG &cfg,std::string file
+                     ,std::string const& wts="Model_0 1 EgivenF 1 f1 1")
+  {
+    FeatureVector v;
+    istringstream ws(wts);
+//    ASSERT_TRUE(ws>>v);
+    HGSetup::JsonTestFile(&hg,file);
+//    hg.Reweight(v);
+    cfg.Init(hg,true,false,false);
+  }
+
+  static void SetUpTestCase() {
+  }
+  static void TearDownTestCase() {
+  }
+};
+
+TEST_F(CFGTest,Binarize) {
+  Hypergraph hg;
+  CFG cfg;
+  JsonFN(hg,cfg,perro_json,perro_wts);
+  CFGFormat form;
+  form.features=true;
+  cerr<<"\nCFG Test.\n\n";
+  cfg.Print(cerr,form);
+}
+
+int main(int argc, char **argv) {
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/decoder/hg_cfg.h b/decoder/hg_cfg.h
index 0a3eb53c..64a0e767 100755
--- a/decoder/hg_cfg.h
+++ b/decoder/hg_cfg.h
@@ -7,12 +7,14 @@ class Hypergraph;
 
 // in case you might want the CFG whether or not you apply FSA models:
 struct HgCFG {
-  HgCFG(Hypergraph const& ih) : ih(ih) { have_cfg=binarized=false; }
+  HgCFG(Hypergraph const& ih) : ih(ih) { have_cfg=binarized=false;have_features=false; }
   Hypergraph const& ih;
   CFG cfg;
   bool have_cfg;
+  bool have_features;
+  bool want_features;
   void InitCFG(CFG &to) {
-    to.Init(ih,true,false,true);
+    to.Init(ih,true,want_features,true);
   }
   bool binarized;
   CFG &GetCFG()
diff --git a/decoder/hg_test.cc b/decoder/hg_test.cc
index 51934ad1..d934036f 100644
--- a/decoder/hg_test.cc
+++ b/decoder/hg_test.cc
@@ -1,68 +1,20 @@
-#include <cassert>
 #include <iostream>
 #include <fstream>
 #include <vector>
-#include <gtest/gtest.h>
 #include "tdict.h"
 
 #include "json_parse.h"
-#include "filelib.h"
-#include "hg.h"
-#include "hg_io.h"
 #include "hg_intersect.h"
 #include "viterbi.h"
 #include "kbest.h"
 #include "inside_outside.h"
 
-using namespace std;
-
-class HGTest : public testing::Test {
- protected:
-  virtual void SetUp() { }
-  virtual void TearDown() { }
-  void CreateHG(Hypergraph* hg) const;
-  void CreateHG_int(Hypergraph* hg) const;
-  void CreateHG_tiny(Hypergraph* hg) const;
-  void CreateHGBalanced(Hypergraph* hg) const;
-  void CreateLatticeHG(Hypergraph* hg) const;
-  void CreateTinyLatticeHG(Hypergraph* hg) const;
-};
-       
-void HGTest::CreateTinyLatticeHG(Hypergraph* hg) const {
-  const string json = "{\"rules\":[1,\"[X] ||| [1] a\",2,\"[X] ||| [1] A\",3,\"[X] ||| [1] b\",4,\"[X] ||| [1] B'\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[],\"node\":{\"in_edges\":[]},\"edges\":[{\"tail\":[0],\"feats\":[0,-0.2],\"rule\":1},{\"tail\":[0],\"feats\":[0,-0.6],\"rule\":2}],\"node\":{\"in_edges\":[0,1]},\"edges\":[{\"tail\":[1],\"feats\":[0,-0.1],\"rule\":3},{\"tail\":[1],\"feats\":[0,-0.9],\"rule\":4}],\"node\":{\"in_edges\":[2,3]}}";
-  istringstream instr(json);
-  EXPECT_TRUE(HypergraphIO::ReadFromJSON(&instr, hg));
-}
+#include "hg_test.h"
 
-void HGTest::CreateLatticeHG(Hypergraph* hg) const {
-  const string json = "{\"rules\":[1,\"[X] ||| [1] a\",2,\"[X] ||| [1] A\",3,\"[X] ||| [1] A A\",4,\"[X] ||| [1] b\",5,\"[X] ||| [1] c\",6,\"[X] ||| [1] B C\",7,\"[X] ||| [1] A B C\",8,\"[X] ||| [1] CC\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[],\"node\":{\"in_edges\":[]},\"edges\":[{\"tail\":[0],\"feats\":[2,-0.3],\"rule\":1},{\"tail\":[0],\"feats\":[2,-0.6],\"rule\":2},{\"tail\":[0],\"feats\":[2,-1.7],\"rule\":3}],\"node\":{\"in_edges\":[0,1,2]},\"edges\":[{\"tail\":[1],\"feats\":[2,-0.5],\"rule\":4}],\"node\":{\"in_edges\":[3]},\"edges\":[{\"tail\":[2],\"feats\":[2,-0.6],\"rule\":5},{\"tail\":[1],\"feats\":[2,-0.8],\"rule\":6},{\"tail\":[0],\"feats\":[2,-0.01],\"rule\":7},{\"tail\":[2],\"feats\":[2,-0.8],\"rule\":8}],\"node\":{\"in_edges\":[4,5,6,7]}}";
-  istringstream instr(json);
-  EXPECT_TRUE(HypergraphIO::ReadFromJSON(&instr, hg));
-}
-
-void HGTest::CreateHG_tiny(Hypergraph* hg) const {
-  const string json = "{\"rules\":[1,\"[X] ||| <s>\",2,\"[X] ||| X [1]\",3,\"[X] ||| Z [1]\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[{\"tail\":[],\"feats\":[0,-2,1,-99],\"rule\":1}],\"node\":{\"in_edges\":[0]},\"edges\":[{\"tail\":[0],\"feats\":[0,-0.5,1,-0.8],\"rule\":2},{\"tail\":[0],\"feats\":[0,-0.7,1,-0.9],\"rule\":3}],\"node\":{\"in_edges\":[1,2]}}";
-  istringstream instr(json);
-  EXPECT_TRUE(HypergraphIO::ReadFromJSON(&instr, hg));
-}
 
-void HGTest::CreateHG_int(Hypergraph* hg) const {
-  const string json = "{\"rules\":[1,\"[X] ||| a\",2,\"[X] ||| b\",3,\"[X] ||| a [1]\",4,\"[X] ||| [1] b\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[{\"tail\":[],\"feats\":[0,0.1],\"rule\":1},{\"tail\":[],\"feats\":[0,0.1],\"rule\":2}],\"node\":{\"in_edges\":[0,1],\"cat\":\"X\"},\"edges\":[{\"tail\":[0],\"feats\":[0,0.3],\"rule\":3},{\"tail\":[0],\"feats\":[0,0.2],\"rule\":4}],\"node\":{\"in_edges\":[2,3],\"cat\":\"Goal\"}}";
-  istringstream instr(json);
-  EXPECT_TRUE(HypergraphIO::ReadFromJSON(&instr, hg));
-}
-
-void HGTest::CreateHG(Hypergraph* hg) const {
-  string json = "{\"rules\":[1,\"[X] ||| a\",2,\"[X] ||| A [1]\",3,\"[X] ||| c\",4,\"[X] ||| C [1]\",5,\"[X] ||| [1] B [2]\",6,\"[X] ||| [1] b [2]\",7,\"[X] ||| X [1]\",8,\"[X] ||| Z [1]\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":1}],\"node\":{\"in_edges\":[0]},\"edges\":[{\"tail\":[0],\"feats\":[0,-0.8,1,-0.1],\"rule\":2}],\"node\":{\"in_edges\":[1]},\"edges\":[{\"tail\":[],\"feats\":[1,-1],\"rule\":3}],\"node\":{\"in_edges\":[2]},\"edges\":[{\"tail\":[2],\"feats\":[0,-0.2,1,-0.1],\"rule\":4}],\"node\":{\"in_edges\":[3]},\"edges\":[{\"tail\":[1,3],\"feats\":[0,-1.2,1,-0.2],\"rule\":5},{\"tail\":[1,3],\"feats\":[0,-0.5,1,-1.3],\"rule\":6}],\"node\":{\"in_edges\":[4,5]},\"edges\":[{\"tail\":[4],\"feats\":[0,-0.5,1,-0.8],\"rule\":7},{\"tail\":[4],\"feats\":[0,-0.7,1,-0.9],\"rule\":8}],\"node\":{\"in_edges\":[6,7]}}";
-  istringstream instr(json);
-  EXPECT_TRUE(HypergraphIO::ReadFromJSON(&instr, hg));
-}
+using namespace std;
 
-void HGTest::CreateHGBalanced(Hypergraph* hg) const {
-  const string json = "{\"rules\":[1,\"[X] ||| i\",2,\"[X] ||| a\",3,\"[X] ||| b\",4,\"[X] ||| [1] [2]\",5,\"[X] ||| [1] [2]\",6,\"[X] ||| c\",7,\"[X] ||| d\",8,\"[X] ||| [1] [2]\",9,\"[X] ||| [1] [2]\",10,\"[X] ||| [1] [2]\",11,\"[X] ||| [1] [2]\",12,\"[X] ||| [1] [2]\",13,\"[X] ||| [1] [2]\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":1}],\"node\":{\"in_edges\":[0]},\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":2}],\"node\":{\"in_edges\":[1]},\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":3}],\"node\":{\"in_edges\":[2]},\"edges\":[{\"tail\":[1,2],\"feats\":[],\"rule\":4},{\"tail\":[2,1],\"feats\":[],\"rule\":5}],\"node\":{\"in_edges\":[3,4]},\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":6}],\"node\":{\"in_edges\":[5]},\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":7}],\"node\":{\"in_edges\":[6]},\"edges\":[{\"tail\":[4,5],\"feats\":[],\"rule\":8},{\"tail\":[5,4],\"feats\":[],\"rule\":9}],\"node\":{\"in_edges\":[7,8]},\"edges\":[{\"tail\":[3,6],\"feats\":[],\"rule\":10},{\"tail\":[6,3],\"feats\":[],\"rule\":11}],\"node\":{\"in_edges\":[9,10]},\"edges\":[{\"tail\":[7,0],\"feats\":[],\"rule\":12},{\"tail\":[0,7],\"feats\":[],\"rule\":13}],\"node\":{\"in_edges\":[11,12]}}";
-  istringstream instr(json);
-  EXPECT_TRUE(HypergraphIO::ReadFromJSON(&instr, hg));
-}
+typedef HGSetup HGTest;
 
 TEST_F(HGTest,Controlled) {
   Hypergraph hg;
@@ -187,7 +139,7 @@ TEST_F(HGTest,PruneInsideOutside) {
   cerr << TD::GetString(trans) << "\n";
   cerr << "cost: " << cost << "\n";
   hg.PrintGraphviz();
-  //hg.DensityPruneInsideOutside(0.5, false, 2.0);
+  hg.DensityPruneInsideOutside(0.5, false, 2.0);
   hg.BeamPruneInsideOutside(0.5, false, 0.5);
   cost = ViterbiESentence(hg, &trans);
   cerr << "Ncst: " << cost << endl;
@@ -274,7 +226,7 @@ TEST_F(HGTest,PLF) {
   string outplf = HypergraphIO::AsPLF(hg);
   cerr << " IN: " << inplf << endl;
   cerr << "OUT: " << outplf << endl;
-  assert(inplf == outplf);
+  EXPECT_EQ(inplf,outplf);
 }
 
 TEST_F(HGTest,PushWeightsToGoal) {
@@ -382,9 +334,8 @@ TEST_F(HGTest,TestAddExpectations) {
 }
 
 TEST_F(HGTest, Small) {
-  ReadFile rf("test_data/small.json.gz");
   Hypergraph hg;
-  assert(HypergraphIO::ReadFromJSON(rf.stream(), &hg));
+  CreateSmallHG(&hg);
   SparseVector<double> wts;
   wts.set_value(FD::Convert("Model_0"), -2.0);
   wts.set_value(FD::Convert("Model_1"), -0.5);
diff --git a/decoder/hg_test.h b/decoder/hg_test.h
new file mode 100755
index 00000000..4f3ae251
--- /dev/null
+++ b/decoder/hg_test.h
@@ -0,0 +1,91 @@
+#ifndef HG_TEST_H
+#define HG_TEST_H
+
+#include "filelib.h"
+#include "hg.h"
+#include "hg_io.h"
+#include <sstream>
+#include <gtest/gtest.h>
+
+using namespace std;
+
+#pragma GCC diagnostic ignore "-Wunused"
+
+namespace {
+
+char const* small_json="small.json.gz";
+
+char const* perro_json="perro.json.gz";
+char const* perro_wts="SameFirstLetter 1 LongerThanPrev 1 ShorterThanPrev 1 GlueTop 0.0 Glue -1.0 EgivenF -0.5 FgivenE -0.5 LexEgivenF -0.5 LexFgivenE -0.5 LM 1";
+
+}
+
+// you can inherit from this or just use the static methods
+struct HGSetup : public testing::Test {
+  enum {
+    HG,
+    HG_int,
+    HG_tiny,
+    HGBalanced,
+    LatticeHG,
+    TinyLatticeHG,
+  };
+  static void CreateHG(Hypergraph* hg);
+  static void CreateHG_int(Hypergraph* hg);
+  static void CreateHG_tiny(Hypergraph* hg);
+  static void CreateHGBalanced(Hypergraph* hg);
+  static void CreateLatticeHG(Hypergraph* hg);
+  static void CreateTinyLatticeHG(Hypergraph* hg);
+
+  static void Json(Hypergraph *hg,std::string const& json) {
+    std::istringstream i(json);
+    ASSERT_TRUE(HypergraphIO::ReadFromJSON(&i, hg));
+  }
+  static void JsonFile(Hypergraph *hg,std::string f) {
+    ReadFile rf(f);
+    ASSERT_TRUE(HypergraphIO::ReadFromJSON(rf.stream(), hg)) << "read from HG file "<<f;
+  }
+  static void JsonTestFile(Hypergraph *hg,std::string n) {
+    JsonFile(hg,"test_data/"+n);
+  }
+  static void CreateSmallHG(Hypergraph *hg) { JsonTestFile(hg,small_json); }
+};
+
+namespace {
+char const* HGjsons[]= {
+  "{\"rules\":[1,\"[X] ||| a\",2,\"[X] ||| A [1]\",3,\"[X] ||| c\",4,\"[X] ||| C [1]\",5,\"[X] ||| [1] B [2]\",6,\"[X] ||| [1] b [2]\",7,\"[X] ||| X [1]\",8,\"[X] ||| Z [1]\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":1}],\"node\":{\"in_edges\":[0]},\"edges\":[{\"tail\":[0],\"feats\":[0,-0.8,1,-0.1],\"rule\":2}],\"node\":{\"in_edges\":[1]},\"edges\":[{\"tail\":[],\"feats\":[1,-1],\"rule\":3}],\"node\":{\"in_edges\":[2]},\"edges\":[{\"tail\":[2],\"feats\":[0,-0.2,1,-0.1],\"rule\":4}],\"node\":{\"in_edges\":[3]},\"edges\":[{\"tail\":[1,3],\"feats\":[0,-1.2,1,-0.2],\"rule\":5},{\"tail\":[1,3],\"feats\":[0,-0.5,1,-1.3],\"rule\":6}],\"node\":{\"in_edges\":[4,5]},\"edges\":[{\"tail\":[4],\"feats\":[0,-0.5,1,-0.8],\"rule\":7},{\"tail\":[4],\"feats\":[0,-0.7,1,-0.9],\"rule\":8}],\"node\":{\"in_edges\":[6,7]}}",
+"{\"rules\":[1,\"[X] ||| a\",2,\"[X] ||| b\",3,\"[X] ||| a [1]\",4,\"[X] ||| [1] b\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[{\"tail\":[],\"feats\":[0,0.1],\"rule\":1},{\"tail\":[],\"feats\":[0,0.1],\"rule\":2}],\"node\":{\"in_edges\":[0,1],\"cat\":\"X\"},\"edges\":[{\"tail\":[0],\"feats\":[0,0.3],\"rule\":3},{\"tail\":[0],\"feats\":[0,0.2],\"rule\":4}],\"node\":{\"in_edges\":[2,3],\"cat\":\"Goal\"}}",
+  "{\"rules\":[1,\"[X] ||| <s>\",2,\"[X] ||| X [1]\",3,\"[X] ||| Z [1]\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[{\"tail\":[],\"feats\":[0,-2,1,-99],\"rule\":1}],\"node\":{\"in_edges\":[0]},\"edges\":[{\"tail\":[0],\"feats\":[0,-0.5,1,-0.8],\"rule\":2},{\"tail\":[0],\"feats\":[0,-0.7,1,-0.9],\"rule\":3}],\"node\":{\"in_edges\":[1,2]}}",
+  "{\"rules\":[1,\"[X] ||| i\",2,\"[X] ||| a\",3,\"[X] ||| b\",4,\"[X] ||| [1] [2]\",5,\"[X] ||| [1] [2]\",6,\"[X] ||| c\",7,\"[X] ||| d\",8,\"[X] ||| [1] [2]\",9,\"[X] ||| [1] [2]\",10,\"[X] ||| [1] [2]\",11,\"[X] ||| [1] [2]\",12,\"[X] ||| [1] [2]\",13,\"[X] ||| [1] [2]\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":1}],\"node\":{\"in_edges\":[0]},\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":2}],\"node\":{\"in_edges\":[1]},\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":3}],\"node\":{\"in_edges\":[2]},\"edges\":[{\"tail\":[1,2],\"feats\":[],\"rule\":4},{\"tail\":[2,1],\"feats\":[],\"rule\":5}],\"node\":{\"in_edges\":[3,4]},\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":6}],\"node\":{\"in_edges\":[5]},\"edges\":[{\"tail\":[],\"feats\":[],\"rule\":7}],\"node\":{\"in_edges\":[6]},\"edges\":[{\"tail\":[4,5],\"feats\":[],\"rule\":8},{\"tail\":[5,4],\"feats\":[],\"rule\":9}],\"node\":{\"in_edges\":[7,8]},\"edges\":[{\"tail\":[3,6],\"feats\":[],\"rule\":10},{\"tail\":[6,3],\"feats\":[],\"rule\":11}],\"node\":{\"in_edges\":[9,10]},\"edges\":[{\"tail\":[7,0],\"feats\":[],\"rule\":12},{\"tail\":[0,7],\"feats\":[],\"rule\":13}],\"node\":{\"in_edges\":[11,12]}}",
+  "{\"rules\":[1,\"[X] ||| [1] a\",2,\"[X] ||| [1] A\",3,\"[X] ||| [1] A A\",4,\"[X] ||| [1] b\",5,\"[X] ||| [1] c\",6,\"[X] ||| [1] B C\",7,\"[X] ||| [1] A B C\",8,\"[X] ||| [1] CC\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[],\"node\":{\"in_edges\":[]},\"edges\":[{\"tail\":[0],\"feats\":[2,-0.3],\"rule\":1},{\"tail\":[0],\"feats\":[2,-0.6],\"rule\":2},{\"tail\":[0],\"feats\":[2,-1.7],\"rule\":3}],\"node\":{\"in_edges\":[0,1,2]},\"edges\":[{\"tail\":[1],\"feats\":[2,-0.5],\"rule\":4}],\"node\":{\"in_edges\":[3]},\"edges\":[{\"tail\":[2],\"feats\":[2,-0.6],\"rule\":5},{\"tail\":[1],\"feats\":[2,-0.8],\"rule\":6},{\"tail\":[0],\"feats\":[2,-0.01],\"rule\":7},{\"tail\":[2],\"feats\":[2,-0.8],\"rule\":8}],\"node\":{\"in_edges\":[4,5,6,7]}}",
+  "{\"rules\":[1,\"[X] ||| [1] a\",2,\"[X] ||| [1] A\",3,\"[X] ||| [1] b\",4,\"[X] ||| [1] B'\"],\"features\":[\"f1\",\"f2\",\"Feature_1\",\"Feature_0\",\"Model_0\",\"Model_1\",\"Model_2\",\"Model_3\",\"Model_4\",\"Model_5\",\"Model_6\",\"Model_7\"],\"edges\":[],\"node\":{\"in_edges\":[]},\"edges\":[{\"tail\":[0],\"feats\":[0,-0.2],\"rule\":1},{\"tail\":[0],\"feats\":[0,-0.6],\"rule\":2}],\"node\":{\"in_edges\":[0,1]},\"edges\":[{\"tail\":[1],\"feats\":[0,-0.1],\"rule\":3},{\"tail\":[1],\"feats\":[0,-0.9],\"rule\":4}],\"node\":{\"in_edges\":[2,3]}}",
+};
+
+}
+
+void HGSetup::CreateTinyLatticeHG(Hypergraph* hg) {
+    Json(hg,HGjsons[TinyLatticeHG]);
+}
+
+void HGSetup::CreateLatticeHG(Hypergraph* hg) {
+  Json(hg,HGjsons[LatticeHG]);
+}
+
+void HGSetup::CreateHG_tiny(Hypergraph* hg) {
+  Json(hg,HGjsons[HG_tiny]);
+}
+
+void HGSetup::CreateHG_int(Hypergraph* hg) {
+  Json(hg,HGjsons[HG_int]);
+}
+
+void HGSetup::CreateHG(Hypergraph* hg) {
+  Json(hg,HGjsons[HG]);
+}
+
+void HGSetup::CreateHGBalanced(Hypergraph* hg) {
+  Json(hg,HGjsons[HGBalanced]);
+}
+
+
+#endif
diff --git a/utils/ftoa.h b/utils/ftoa.h
index 81a685ac..3dba528d 100755
--- a/utils/ftoa.h
+++ b/utils/ftoa.h
@@ -243,6 +243,7 @@ char *prepend_pos_nonsci(char *p,F f) {
       *--p='0';
   } else
     p=utoa(p,u);
+  return p;
 }
 
 // modify p; return true if handled
diff --git a/utils/utoa.h b/utils/utoa.h
index 341965cc..5de490ba 100755
--- a/utils/utoa.h
+++ b/utils/utoa.h
@@ -136,6 +136,9 @@ char *utoa_drop_trailing_0(char *buf,Uint_ n_, unsigned &n_skipped) {
   }
 }
 
+//#include "warning_push.h"
+//#pragma GCC diagnostic ignore "-Wtype-limits" // because sign check on itoa<unsigned> is annoying
+
 // desired feature: itoa(unsigned) = utoa(unsigned)
 // positive sign: 0 -> +0, 1-> +1.  obviously -n -> -n
 template <class Int>
@@ -167,21 +170,23 @@ char * itoa_left_pad(char *buf,char *bufend,Int i,bool positive_sign=false,char
 }
 
 template <class Int>
-inline std::string itos(Int n) {
+inline std::string utos(Int n) {
   char buf[signed_for_int<Int>::toa_bufsize];
   char *end=buf+signed_for_int<Int>::toa_bufsize;
-  char *p=itoa(end,n);
+  char *p=utoa(end,n);
   return std::string(p,end);
 }
 
 template <class Int>
-inline std::string utos(Int n) {
+inline std::string itos(Int n) {
   char buf[signed_for_int<Int>::toa_bufsize];
   char *end=buf+signed_for_int<Int>::toa_bufsize;
   char *p=itoa(end,n);
   return std::string(p,end);
 }
 
+//#include "warning_pop.h"
+
 //returns position of '\0' terminating number written starting at to
 template <class Int>
 inline char* append_utoa(char *to,typename signed_for_int<Int>::unsigned_t n) {