cleanup

19 files changed, 984 insertions, 0 deletions

diff --git a/src/Makefile b/src/Makefile
new file mode 100644
index 0000000..c81c7dd
--- /dev/null
+++ b/src/Makefile
@@ -0,0 +1,61 @@
+COMPILER := g++
+CXXFLAGS := -O3 -march=native -mtune=native -I../lib
+
+
+all: test_gason test_json-cpp test_jsoncpp test_libjson test_picojson test_rapidjson test_sajson test_JsonBox test_jsonxx test_MicroJSON test_nosjob test_cdec_json_parser test_msgpack test_msgpack_streaming
+
+test_gason: test_gason.cc
+	$(COMPILER) $(CXXFLAGS) -std=c++11 test_gason.cc -o test_gason ../lib/gason/gason.o
+
+test_json-cpp: test_json-cpp.cc
+	$(COMPILER) $(CXXFLAGS) -std=c++11 test_json-cpp.cc -o test_json-cpp
+
+test_jsoncpp: test_jsoncpp.cc
+	$(COMPILER) $(CXXFLAGS) test_jsoncpp.cc ../lib/jsoncpp/lib/libjsoncpp.a -o test_jsoncpp
+
+test_libjson: test_libjson.cc
+	$(COMPILER) $(CXXFLAGS) test_libjson.cc ../lib/libjson-7.6.1/libjson.a -o test_libjson
+
+test_picojson: test_picojson.cc
+	$(COMPILER) $(CXXFLAGS) test_picojson.cc -o test_picojson
+
+test_rapidjson: test_rapidjson.cc
+	$(COMPILER) $(CXXFLAGS) test_rapidjson.cc -o test_rapidjson
+
+test_sajson: test_sajson.cc
+	$(COMPILER) $(CXXFLAGS) test_sajson.cc -o test_sajson
+
+test_JsonBox: test_JsonBox.cc
+	$(COMPILER) $(CXXFLAGS) test_JsonBox.cc -I../lib/JsonBox/include/ ../lib/JsonBox/libJsonBox.a -o test_JsonBox
+
+test_jsonxx: test_jsonxx.cc
+	$(COMPILER) $(CXXFLAGS) test_jsonxx.cc ../lib/jsonxx/jsonxx.o -o test_jsonxx
+
+test_MicroJSON: test_MicroJSON.cc
+	#$(COMPILER) $(CXXFLAGS) test_MicroJSON.cc ../lib/MicroJSON-0.3.2/libMicroJSON-0.so -lUTF8Strings-1 -o test_MicroJSON
+
+test_nosjob: test_nosjob.cc
+	$(COMPILER) $(CXXFLAGS) test_nosjob.cc ../lib/nosjob-e1d67401fcda6e05/libnosjob.a -o test_nosjob
+
+test_cdec_json_parser: test_cdec_json_parser.cc
+	$(COMPILER) $(CXXFLAGS) test_cdec_json_parser.cc ../lib/cdec_json_parser/json_parse.o ../lib/cdec_json_parser/JSON_parser.o -o test_cdec_json_parser
+
+test_msgpack: test_msgpack.cc
+	 $(COMPILER) $(CXXFLAGS) test_msgpack.cc -I../lib/msgpack-c/include/ ../lib/msgpack-c/lib/libmsgpack.a -o test_msgpack
+
+test_msgpack_streaming: test_msgpack_streaming.cc
+	 $(COMPILER) $(CXXFLAGS) test_msgpack_streaming.cc -I../lib/msgpack-c/include/ ../lib/msgpack-c/lib/libmsgpack.a -o test_msgpack_streaming
+
+make_pak: make_pak.cc
+	$(COMPILER) $(CXXFLAGS) -std=c++11 make_pak.cc ../lib/msgpack-c/lib/libmsgpack.a -o make_pak
+
+make_pak_s: make_pak_s.cc
+	$(COMPILER) $(CXXFLAGS) -std=c++11 make_pak_s.cc ../lib/msgpack-c/lib/libmsgpack.a -o make_pak_s
+
+clean:
+	rm -f test_gason test_json-cpp test_jsoncpp test_libjson
+	rm -f test_picojson test_rapidjson test_sajson test_JsonBox
+	rm -f test_jsonxx test_MicroJSON test_nosjob test_cdec_json_parser
+	rm -f test_msgpack test_msgpack_streaming
+	rm -f make_pak make_pak_s
+
diff --git a/src/make_pak.cc b/src/make_pak.cc
new file mode 100644
index 0000000..bacdfe3
--- /dev/null
+++ b/src/make_pak.cc
@@ -0,0 +1,125 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <msgpack.hpp>
+#include <msgpack/fbuffer.h>
+#include <msgpack/fbuffer.hpp>
+
+
+/*
+ * https://github.com/ascheglov/json-cpp
+ *
+ */
+#include "json-cpp/single_include/json-cpp.hpp"
+
+using namespace std;
+
+
+struct Node {
+  int id;
+  string cat;
+  vector<int> span;
+
+  MSGPACK_DEFINE(id, cat, span);
+};
+
+struct Vector {
+  double CountEF;
+  double EgivenFCoherent;
+  double Glue;
+  double IsSingletonF;
+  double IsSingletonFE;
+  double LanguageModel;
+  double LanguageModel_OOV;
+  double MaxLexFgivenE;
+  double MaxLexEgivenF;
+  double PassThrough;
+  double PassThrough_1;
+  double PassThrough_2;
+  double PassThrough_3;
+  double PassThrough_4;
+  double PassThrough_5;
+  double PassThrough_6;
+  double SampleCountF;
+  double WordPenalty;
+
+	MSGPACK_DEFINE(CountEF, EgivenFCoherent, Glue, IsSingletonF, IsSingletonFE, LanguageModel, LanguageModel_OOV, MaxLexEgivenF, MaxLexFgivenE, PassThrough, PassThrough_1, PassThrough_2, PassThrough_3, PassThrough_4, PassThrough_5, PassThrough_6, SampleCountF, WordPenalty);
+};
+
+struct Edge {
+  int head;
+  string rule;
+  vector<int> tails;
+  Vector f;
+  double weight;
+
+	MSGPACK_DEFINE(head, rule, tails, f, weight);
+};
+
+struct Hg {
+  Vector weights;
+  vector<Node> nodes;
+  vector<Edge> edges;
+
+  MSGPACK_DEFINE(weights, nodes, edges);
+};
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Hg& o)
+{
+  fields(o, stream, "weights", o.weights, "nodes", o.nodes, "edges", o.edges);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Edge& o)
+{
+  fields(o, stream, "head", o.head, "rule", o.rule, "tails", o.tails, "f", o.f, "weight", o.weight);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Vector& o)
+{
+  fields(o, stream, "EgivenFCoherent", o.EgivenFCoherent, "SampleCountF", o.SampleCountF, "CountEF", o.CountEF, "MaxLexFgivenE", o.MaxLexFgivenE, "MaxLexEgivenF", o.MaxLexEgivenF, "IsSingletonF", o.IsSingletonF, "IsSingletonFE", o.IsSingletonFE, "LanguageModel", o.LanguageModel, "LanguageModel_OOV", o.LanguageModel_OOV, "PassThrough", o.PassThrough, "PassThrough_1", o.PassThrough_1, "PassThrough_2", o.PassThrough_2, "PassThrough_3", o.PassThrough_3, "PassThrough_4", o.PassThrough_4, "PassThrough_5", o.PassThrough_5, "PassThrough_6", o.PassThrough_6, "WordPenalty", o.WordPenalty, "Glue", o.Glue);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Node& o)
+{
+  fields(o, stream, "id", o.id, "cat", o.cat, "span", o.span);
+}
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+  string json_str((istreambuf_iterator<char>(ifs) ),
+                   (istreambuf_iterator<char>()));
+
+  Hg hg;
+  Vector w;
+  hg.weights = w;
+  vector<Node> nodes;
+  hg.nodes = nodes;
+  vector<Edge> edges;
+  hg.edges = edges;
+  jsoncpp::parse(hg, json_str);
+
+  FILE* file = fopen(argv[2], "wb");
+  msgpack::fbuffer fbuf(file);
+  msgpack::pack(fbuf, hg);
+  fclose(file);
+
+  /*ifstream ifs1(argv[2]);
+  string str1((istreambuf_iterator<char>(jfs1)),
+                   (istreambuf_iterator<char>()));
+
+	msgpack::zone zone;
+	msgpack::object obj;
+	msgpack::unpack(str1.data(), str1.size(), NULL, &zone, &obj);
+
+  Hg hg;
+	obj.convert(&hg);*/
+
+  return 0;
+}
+
diff --git a/src/make_pak_s.cc b/src/make_pak_s.cc
new file mode 100644
index 0000000..a72bc60
--- /dev/null
+++ b/src/make_pak_s.cc
@@ -0,0 +1,121 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <msgpack.hpp>
+#include <msgpack/fbuffer.h>
+#include <msgpack/fbuffer.hpp>
+
+
+/*
+ * https://github.com/ascheglov/json-cpp
+ *
+ */
+#include "json-cpp/single_include/json-cpp.hpp"
+
+using namespace std;
+
+
+struct Node {
+  int id;
+  string cat;
+  vector<int> span;
+
+  MSGPACK_DEFINE(id, cat, span);
+};
+
+struct Vector {
+  double CountEF;
+  double EgivenFCoherent;
+  double Glue;
+  double IsSingletonF;
+  double IsSingletonFE;
+  double LanguageModel;
+  double LanguageModel_OOV;
+  double MaxLexFgivenE;
+  double MaxLexEgivenF;
+  double PassThrough;
+  double PassThrough_1;
+  double PassThrough_2;
+  double PassThrough_3;
+  double PassThrough_4;
+  double PassThrough_5;
+  double PassThrough_6;
+  double SampleCountF;
+  double WordPenalty;
+
+	MSGPACK_DEFINE(CountEF, EgivenFCoherent, Glue, IsSingletonF, IsSingletonFE, LanguageModel, LanguageModel_OOV, MaxLexEgivenF, MaxLexFgivenE, PassThrough, PassThrough_1, PassThrough_2, PassThrough_3, PassThrough_4, PassThrough_5, PassThrough_6, SampleCountF, WordPenalty);
+};
+
+struct Edge {
+  int head;
+  string rule;
+  vector<int> tails;
+  Vector f;
+  double weight;
+
+	MSGPACK_DEFINE(head, rule, tails, f, weight);
+};
+
+struct Hg {
+  Vector weights;
+  vector<Node> nodes;
+  vector<Edge> edges;
+
+  MSGPACK_DEFINE(weights, nodes, edges);
+};
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Hg& o)
+{
+  fields(o, stream, "weights", o.weights, "nodes", o.nodes, "edges", o.edges);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Edge& o)
+{
+  fields(o, stream, "head", o.head, "rule", o.rule, "tails", o.tails, "f", o.f, "weight", o.weight);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Vector& o)
+{
+  fields(o, stream, "EgivenFCoherent", o.EgivenFCoherent, "SampleCountF", o.SampleCountF, "CountEF", o.CountEF, "MaxLexFgivenE", o.MaxLexFgivenE, "MaxLexEgivenF", o.MaxLexEgivenF, "IsSingletonF", o.IsSingletonF, "IsSingletonFE", o.IsSingletonFE, "LanguageModel", o.LanguageModel, "LanguageModel_OOV", o.LanguageModel_OOV, "PassThrough", o.PassThrough, "PassThrough_1", o.PassThrough_1, "PassThrough_2", o.PassThrough_2, "PassThrough_3", o.PassThrough_3, "PassThrough_4", o.PassThrough_4, "PassThrough_5", o.PassThrough_5, "PassThrough_6", o.PassThrough_6, "WordPenalty", o.WordPenalty, "Glue", o.Glue);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Node& o)
+{
+  fields(o, stream, "id", o.id, "cat", o.cat, "span", o.span);
+}
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+  string json_str((istreambuf_iterator<char>(ifs) ),
+                   (istreambuf_iterator<char>()));
+
+  Hg hg;
+  Vector w;
+  hg.weights = w;
+  vector<Node> nodes;
+  hg.nodes = nodes;
+  vector<Edge> edges;
+  hg.edges = edges;
+  jsoncpp::parse(hg, json_str);
+
+  FILE* file = fopen(argv[2], "wb");
+  msgpack::fbuffer fbuf(file);
+  msgpack::pack(fbuf, hg.nodes.size());
+  msgpack::pack(fbuf, hg.edges.size());
+  msgpack::pack(fbuf, hg.weights);
+  for (auto it = hg.nodes.begin(); it != hg.nodes.end(); it++)
+    msgpack::pack(fbuf, *it);
+  for (auto it = hg.edges.begin(); it != hg.edges.end(); it++)
+    msgpack::pack(fbuf, *it);
+
+  fclose(file);
+
+  return 0;
+}
+
diff --git a/src/test_JsonBox.cc b/src/test_JsonBox.cc
new file mode 100644
index 0000000..e1b22c4
--- /dev/null
+++ b/src/test_JsonBox.cc
@@ -0,0 +1,24 @@
+#include <iostream>
+#include <string>
+
+/*
+ * https://github.com/anhero/JsonBox
+ *
+ */
+#include "JsonBox/include/JsonBox.h"
+
+using namespace std;
+
+
+int
+main(int argc, char** argv)
+{
+  JsonBox::Value v;
+  v.loadFromFile(argv[1]);
+  JsonBox::Value w = v["edges"].getArray().back();
+  string s = w["rule"].getString();
+  cerr << s.substr(1,4) << endl;
+
+  return 0;
+}
+
diff --git a/src/test_MicroJSON.cc b/src/test_MicroJSON.cc
new file mode 100644
index 0000000..d8d4969
--- /dev/null
+++ b/src/test_MicroJSON.cc
@@ -0,0 +1,28 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+
+/*
+ * http://grigory.info/MicroJSON.About.html
+ *
+ */
+#include "MicroJSON-0.3.2/Node.h"
+
+using namespace std;
+
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+  string json_str((istreambuf_iterator<char>(ifs)),
+                   (istreambuf_iterator<char>()));
+
+  MicroJSON::Node Root;
+  Root.Parse(json_str);
+  MicroJSON::Node* edges = Root.GetSubNode("edges");
+  cerr << edges->GetChildren().back()->GetSubNode("rule") << endl;
+
+  return 0;
+}
+
diff --git a/src/test_MicroJSON.sh b/src/test_MicroJSON.sh
new file mode 100755
index 0000000..89d1d3d
--- /dev/null
+++ b/src/test_MicroJSON.sh
@@ -0,0 +1,5 @@
+#!/bin/sh -x
+
+
+LD_LIBRARY_PATH="/home/pks/z/test/json_test/MicroJSON-0.3.2" ./test_MicroJSON $1
+
diff --git a/src/test_cdec_json_parser.cc b/src/test_cdec_json_parser.cc
new file mode 100644
index 0000000..e805318
--- /dev/null
+++ b/src/test_cdec_json_parser.cc
@@ -0,0 +1,25 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+
+/*
+ * https://github.com/redpony/cdec/tree/master/decoder
+ *
+ */
+#include "cdec_json_parser/json_parse.h"
+
+using namespace std;
+
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+
+  istream& s = ifs;
+  JSONParser p;
+  p.Parse(&s);
+
+  return 0;
+}
+
diff --git a/src/test_gason.cc b/src/test_gason.cc
new file mode 100644
index 0000000..d78c385
--- /dev/null
+++ b/src/test_gason.cc
@@ -0,0 +1,71 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <string.h>
+
+/*
+ * https://github.com/vivkin/gason
+ *
+ */
+#include "gason/gason.h"
+
+using namespace std;
+
+
+void
+print(const char *s)
+{
+  string u(s);
+  u = u.substr(1, 4);
+  if (u == "Goal") {
+    cerr << u << endl;
+  }
+}
+
+void
+walk(JsonValue o)
+{
+	switch (o.getTag()) {
+		case JSON_TAG_NUMBER:
+			break;
+		case JSON_TAG_BOOL:
+			break;
+		case JSON_TAG_STRING:
+			print(o.toString());
+			break;
+		case JSON_TAG_ARRAY:
+			if (!o.toNode())
+				break;
+			for (auto i : o)
+				walk(i->value);
+			break;
+		case JSON_TAG_OBJECT:
+			if (!o.toNode())
+				break;
+			for (auto i : o) {
+				print(i->key);
+				walk(i->value);
+			}
+			break;
+		case JSON_TAG_NULL:
+			break;
+	}
+}
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+  string json_str((istreambuf_iterator<char>(ifs)),
+                   (istreambuf_iterator<char>()));
+
+  char* s = strdup(json_str.c_str());
+  char *p;
+  JsonValue v;
+  JsonAllocator a;
+  JsonParseStatus status = jsonParse(s, &p, &v, a);
+  walk(v);
+
+  return 0;
+}
+
diff --git a/src/test_json-cpp.cc b/src/test_json-cpp.cc
new file mode 100644
index 0000000..c27db06
--- /dev/null
+++ b/src/test_json-cpp.cc
@@ -0,0 +1,100 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+
+/*
+ * https://github.com/ascheglov/json-cpp
+ *
+ */
+#include "json-cpp/single_include/json-cpp.hpp"
+
+using namespace std;
+
+
+struct Node {
+  int id;
+  string cat;
+  vector<int> span;
+};
+
+struct Vector {
+  double CountEF;
+  double EgivenFCoherent;
+  double Glue;
+  double IsSingletonF;
+  double IsSingletonFE;
+  double LanguageModel;
+  double LanguageModel_OOV;
+  double MaxLexFgivenE;
+  double MaxLexEgivenF;
+  double PassThrough;
+  double PassThrough_1;
+  double PassThrough_2;
+  double PassThrough_3;
+  double PassThrough_4;
+  double PassThrough_5;
+  double PassThrough_6;
+  double SampleCountF;
+  double WordPenalty;
+};
+
+struct Edge {
+  int head;
+  string rule;
+  vector<int> tails;
+  Vector f;
+  double weight;
+};
+
+struct Hg {
+  Vector weights;
+  vector<Node> nodes;
+  vector<Edge> edges;
+  vector<string> rules;
+};
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Hg& o)
+{
+  fields(o, stream, "weights", o.weights, "nodes", o.nodes, "edges", o.edges, "rules", o.rules);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Edge& o)
+{
+  fields(o, stream, "head", o.head, "rule", o.rule, "tails", o.tails, "f", o.f, "weight", o.weight);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Vector& o)
+{
+  fields(o, stream, "EgivenFCoherent", o.EgivenFCoherent, "SampleCountF", o.SampleCountF, "CountEF", o.CountEF, "MaxLexFgivenE", o.MaxLexFgivenE, "MaxLexEgivenF", o.MaxLexEgivenF, "IsSingletonF", o.IsSingletonF, "IsSingletonFE", o.IsSingletonFE, "LanguageModel", o.LanguageModel, "LanguageModel_OOV", o.LanguageModel_OOV, "PassThrough", o.PassThrough, "PassThrough_1", o.PassThrough_1, "PassThrough_2", o.PassThrough_2, "PassThrough_3", o.PassThrough_3, "PassThrough_4", o.PassThrough_4, "PassThrough_5", o.PassThrough_5, "PassThrough_6", o.PassThrough_6, "WordPenalty", o.WordPenalty, "Glue", o.Glue);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Node& o)
+{
+  fields(o, stream, "id", o.id, "cat", o.cat, "span", o.span);
+}
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+  string json_str((istreambuf_iterator<char>(ifs) ),
+                   (istreambuf_iterator<char>()));
+
+  Hg hg;
+  Vector w;
+  hg.weights = w;
+  vector<Node> nodes;
+  hg.nodes = nodes;
+  vector<Edge> edges;
+  hg.edges = edges;
+  jsoncpp::parse(hg, json_str);
+  Edge& last_edge = hg.edges.back();
+  cerr << last_edge.rule.substr(1, 4) << endl;
+
+  return 0;
+}
+
diff --git a/src/test_jsoncpp.cc b/src/test_jsoncpp.cc
new file mode 100644
index 0000000..ab3bd0c
--- /dev/null
+++ b/src/test_jsoncpp.cc
@@ -0,0 +1,29 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+
+/*
+ * https://github.com/open-source-parsers/jsoncpp
+ *
+ */
+#include "jsoncpp/include/json/json.h"
+
+using namespace std;
+
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+  string json_str((istreambuf_iterator<char>(ifs)),
+                   (istreambuf_iterator<char>()));
+
+  Json::Value v;
+  Json::Reader reader;
+  reader.parse(json_str, v);
+  Json::Value last_edge = v["edges"][v["edges"].size()-1];
+  cerr << last_edge["rule"].asString().substr(1, 4) << endl;
+
+  return 0;
+}
+
diff --git a/src/test_jsonxx.cc b/src/test_jsonxx.cc
new file mode 100644
index 0000000..d06640e
--- /dev/null
+++ b/src/test_jsonxx.cc
@@ -0,0 +1,35 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+
+/*
+ * https://github.com/hjiang/jsonxx
+ *
+ */
+#include "jsonxx/jsonxx.h"
+
+using namespace std;
+
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+  string json_str((istreambuf_iterator<char>(ifs)),
+                   (istreambuf_iterator<char>()));
+
+  jsonxx::Object o;
+  o.parse(json_str);
+  jsonxx::Array edges = o.get<jsonxx::Array>("edges");
+  jsonxx::Array::container::const_iterator it = edges.values().begin(), end = edges.values().end();
+  while (it != end) {
+    jsonxx::Object e = (*it)->get<jsonxx::Object>();
+    string s = e.get<string>("rule").substr(1, 4);
+    if (s == "Goal")
+      cerr << s << endl;
+    ++it;
+  }
+
+  return 0;
+}
+
diff --git a/src/test_libjson.cc b/src/test_libjson.cc
new file mode 100644
index 0000000..03dc3bf
--- /dev/null
+++ b/src/test_libjson.cc
@@ -0,0 +1,44 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+
+/*
+ * http://sourceforge.net/projects/libjson/
+ *
+ */
+#include "libjson-7.6.1/libjson.h"
+
+using namespace std;
+
+
+void
+walk(const JSONNode & n)
+{
+  JSONNode::const_iterator it = n.begin();
+  while (it != n.end()){
+    if (it->type() == JSON_ARRAY || it->type() == JSON_NODE){
+      walk(*it);
+    }
+    string s = it->as_string();
+    if (s.size() >= 5) {
+      string t = s.substr(1, 4);
+      if (t == "Goal")
+        cerr << t << endl;
+    }
+    ++it;
+  }
+}
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+  string json_str((istreambuf_iterator<char>(ifs)),
+                   (istreambuf_iterator<char>()));
+
+  JSONNode n = libjson::parse(json_str);
+  walk(n);
+
+  return 0;
+}
+
diff --git a/src/test_msgpack.cc b/src/test_msgpack.cc
new file mode 100644
index 0000000..70cf1c8
--- /dev/null
+++ b/src/test_msgpack.cc
@@ -0,0 +1,81 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+
+/*
+ * http://msgpack.org/
+ *
+ */
+#include <msgpack.hpp>
+#include <msgpack/fbuffer.hpp>
+
+using namespace std;
+
+
+struct Node {
+  int id;
+  string cat;
+  vector<int> span;
+
+  MSGPACK_DEFINE(id, cat, span);
+};
+
+struct Vector {
+  double CountEF;
+  double EgivenFCoherent;
+  double Glue;
+  double IsSingletonF;
+  double IsSingletonFE;
+  double LanguageModel;
+  double LanguageModel_OOV;
+  double MaxLexFgivenE;
+  double MaxLexEgivenF;
+  double PassThrough;
+  double PassThrough_1;
+  double PassThrough_2;
+  double PassThrough_3;
+  double PassThrough_4;
+  double PassThrough_5;
+  double PassThrough_6;
+  double SampleCountF;
+  double WordPenalty;
+
+	MSGPACK_DEFINE(CountEF, EgivenFCoherent, Glue, IsSingletonF, IsSingletonFE, LanguageModel, LanguageModel_OOV, MaxLexEgivenF, MaxLexFgivenE, PassThrough, PassThrough_1, PassThrough_2, PassThrough_3, PassThrough_4, PassThrough_5, PassThrough_6, SampleCountF, WordPenalty);
+};
+
+struct Edge {
+  int head;
+  string rule;
+  vector<int> tails;
+  Vector f;
+  double weight;
+
+	MSGPACK_DEFINE(head, rule, tails, f, weight);
+};
+
+struct Hg {
+  Vector weights;
+  vector<Node> nodes;
+  vector<Edge> edges;
+
+  MSGPACK_DEFINE(weights, nodes, edges);
+};
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+  string str((istreambuf_iterator<char>(ifs)),
+                   (istreambuf_iterator<char>()));
+
+	msgpack::zone zone;
+	msgpack::object obj;
+	msgpack::unpack(str.data(), str.size(), NULL, &zone, &obj);
+  Hg hg;
+	obj.convert(&hg);
+  Edge last_edge = hg.edges.back();
+	cerr << last_edge.rule.substr(1, 4) << endl;
+
+  return 0;
+}
+
diff --git a/src/test_msgpack_ruby b/src/test_msgpack_ruby
new file mode 100755
index 0000000..0f2d387
--- /dev/null
+++ b/src/test_msgpack_ruby
@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'msgpack'
+
+
+msg = MessagePack.unpack(File.new(ARGV[0]).read)
+
+STDERR.write "#{msg["edges"].last()["rule"][1..4]}\n"
+
diff --git a/src/test_msgpack_streaming.cc b/src/test_msgpack_streaming.cc
new file mode 100644
index 0000000..1a3cf55
--- /dev/null
+++ b/src/test_msgpack_streaming.cc
@@ -0,0 +1,99 @@
+#include <msgpack.hpp>
+#include <msgpack/fbuffer.hpp>
+#include <iostream>
+#include <string>
+#include <fstream>
+
+using namespace std;
+ 
+
+
+struct Node {
+  int id;
+  string cat;
+  vector<int> span;
+
+  MSGPACK_DEFINE(id, cat, span);
+};
+
+struct Vector {
+  double CountEF;
+  double EgivenFCoherent;
+  double Glue;
+  double IsSingletonF;
+  double IsSingletonFE;
+  double LanguageModel;
+  double LanguageModel_OOV;
+  double MaxLexFgivenE;
+  double MaxLexEgivenF;
+  double PassThrough;
+  double PassThrough_1;
+  double PassThrough_2;
+  double PassThrough_3;
+  double PassThrough_4;
+  double PassThrough_5;
+  double PassThrough_6;
+  double SampleCountF;
+  double WordPenalty;
+
+	MSGPACK_DEFINE(CountEF, EgivenFCoherent, Glue, IsSingletonF, IsSingletonFE, LanguageModel, LanguageModel_OOV, MaxLexEgivenF, MaxLexFgivenE, PassThrough, PassThrough_1, PassThrough_2, PassThrough_3, PassThrough_4, PassThrough_5, PassThrough_6, SampleCountF, WordPenalty);
+};
+
+struct Edge {
+  int head;
+  string rule;
+  vector<int> tails;
+  Vector f;
+  double weight;
+
+	MSGPACK_DEFINE(head, rule, tails, f, weight);
+};
+
+struct Hg {
+  Vector weights;
+  vector<Node> nodes;
+  vector<Edge> edges;
+
+  MSGPACK_DEFINE(weights, nodes, edges);
+};
+
+int
+main(int argc, char** argv) {
+  ifstream ifs(argv[1]);
+
+  size_t count = 0, n_, e_;
+  msgpack::unpacker pac;
+  while(true) {
+      pac.reserve_buffer(32*1024);
+      size_t bytes = ifs.readsome(pac.buffer(), pac.buffer_capacity());
+      pac.buffer_consumed(bytes);
+      msgpack::unpacked result;
+      while(pac.next(&result)) {
+          msgpack::object obj = result.get();
+          if (count ==  0) {
+            obj.convert(&n_);
+            n_ += 2;
+          } else  if (count == 1) {
+            obj.convert(&e_);
+            e_ += 2;
+          } else if (count == 2) {
+            Vector v; 
+            obj.convert(&v);
+          } else if (count > 2 && count <= n_) { 
+            Node n;
+            obj.convert(&n);
+          } else if (count > n_ && count <= n_+e_+1) {
+            Edge e;
+            obj.convert(&e);
+            string s = e.rule.substr(1, 4);
+            if (s == "Goal")
+              cout << s << endl;
+          }
+          count++;
+      }
+      if (!bytes) break;
+  }
+
+  return 0;
+}
+
diff --git a/src/test_nosjob.cc b/src/test_nosjob.cc
new file mode 100644
index 0000000..cf8891f
--- /dev/null
+++ b/src/test_nosjob.cc
@@ -0,0 +1,32 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+
+/*
+ * http://fossil.wanderinghorse.net/repos/nosjob/index.cgi/index
+ *
+ */
+#include "nosjob-e1d67401fcda6e05/include/wh/nosjob/nosjob.hpp"
+
+using namespace std;
+
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+  string json_str((istreambuf_iterator<char>(ifs)),
+                   (istreambuf_iterator<char>()));
+
+  nosjob::Atom root = nosjob::JsonParser().parse(json_str);
+  nosjob::Object o = nosjob::Object::cast(root);
+  nosjob::Atom edges = o.get(nosjob::Utf8String("edges"));
+  nosjob::Array a = nosjob::Array::cast(edges);
+  nosjob::Object last_edge = nosjob::Object::cast(a.get(a.size()-1));
+  nosjob::Utf8String s = nosjob::Utf8String::cast(last_edge.get(nosjob::Utf8String("rule")));
+  string t((char*)s.c_str());
+  cerr << t.substr(1, 4) << endl;
+
+  return 0;
+}
+
diff --git a/src/test_picojson.cc b/src/test_picojson.cc
new file mode 100644
index 0000000..cf3b621
--- /dev/null
+++ b/src/test_picojson.cc
@@ -0,0 +1,32 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <sstream>
+
+/*
+ * https://github.com/kazuho/picojson
+ *
+ */
+#include "picojson/picojson.h"
+
+using namespace std;
+
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+  string json_str((istreambuf_iterator<char>(ifs)),
+                  (istreambuf_iterator<char>()));
+
+  picojson::value v;
+  istringstream iss(json_str);
+  picojson::parse(v, iss);
+  picojson::value::object& obj = v.get<picojson::object>();
+  picojson::value::object& last_edge = obj["edges"].get<picojson::array>().back().get<picojson::object>();
+  string s(last_edge["rule"].get<string>());
+  cerr << s.substr(1, 4) << endl;
+
+  return 0;
+}
+
diff --git a/src/test_rapidjson.cc b/src/test_rapidjson.cc
new file mode 100644
index 0000000..b344ed0
--- /dev/null
+++ b/src/test_rapidjson.cc
@@ -0,0 +1,31 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <string.h>
+
+/*
+ * https://github.com/miloyip/rapidjson
+ *
+ */
+#include "rapidjson/include/rapidjson/rapidjson.h"
+#include "rapidjson/include/rapidjson/document.h"
+#include "rapidjson/include/rapidjson/stringbuffer.h"
+
+using namespace std;
+
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+  string json_str((istreambuf_iterator<char>(ifs)),
+                   (istreambuf_iterator<char>()));
+
+  rapidjson::Document d;
+  d.Parse(json_str.c_str());
+  string s(d["edges"][d["edges"].Size()-1]["rule"].GetString());
+  cerr << s.substr(1, 4) << endl;
+
+  return 0;
+}
+
diff --git a/src/test_sajson.cc b/src/test_sajson.cc
new file mode 100644
index 0000000..4081d43
--- /dev/null
+++ b/src/test_sajson.cc
@@ -0,0 +1,32 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <string.h>
+
+/*
+ * https://github.com/chadaustin/sajson
+ *
+ */
+#include "sajson/include/sajson.h"
+
+using namespace std;
+
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+  string json_str((istreambuf_iterator<char>(ifs)),
+                   (istreambuf_iterator<char>()));
+
+  const sajson::document& document = sajson::parse(sajson::literal(json_str.c_str()));
+  size_t index_a = document.get_root().find_object_key(sajson::literal("edges"));
+  const sajson::value& edges = document.get_root().get_object_value(index_a);
+  const sajson::value& last_edge = edges.get_array_element(edges.get_length()-1);
+  size_t index_r  = last_edge.find_object_key(sajson::literal("rule"));
+  const sajson::value& r = last_edge.get_object_value(index_r);
+  cerr << r.as_string().substr(1, 4) << endl;
+
+  return 0;
+}
+