msgpack streaming

9 files changed, 276 insertions, 31 deletions

diff --git a/.gitignore b/.gitignore
index 80d3144..b740083 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,8 +25,10 @@ test_picojson
 test_rapidjson
 test_sajson
 test_msgpack
+test_msgpack_streaming
 data/make_paks
-data/*.pak
+data/make_paks2
+data/*.pak*
 data/*.json
 data/*.gz
 data/*.in
diff --git a/Makefile b/Makefile
index bf290a4..3013d32 100644
--- a/Makefile
+++ b/Makefile
@@ -43,6 +43,9 @@ test_cdec_json_parser: test_cdec_json_parser.cc
 test_msgpack: test_msgpack.cc
 	 $(COMPILER) $(CXXFLAGS) test_msgpack.cc -I./msgpack-c/include/ ./msgpack-c/lib/libmsgpack.a -o test_msgpack
 
+test_msgpack_streaming: test_msgpack_streaming.cc
+	 $(COMPILER) $(CXXFLAGS) test_msgpack_streaming.cc -I./msgpack-c/include/ ./msgpack-c/lib/libmsgpack.a -o test_msgpack_streaming
+
 clean:
 	rm -f test_gason test_json-cpp test_jsoncpp test_libjson
 	rm -f test_picojson test_rapidjson test_sajson test_JsonBox
diff --git a/README.md b/README.md
index d1c1ea5..808bac8 100644
--- a/README.md
+++ b/README.md
@@ -217,29 +217,43 @@ MSGPACK parsing benchmark
 
 <pre>
 [test_msgpack]
-data/1020.pak: 2.2 s
-data/1570.pak: 0.8 s
-data/1391.pak: 0.5 s
-data/429.pak:  0.15 s
-data/2002.pak: 0.09 s
-data/1889.pak: 0.02 s
-data/1495.pak: 0.0 s
-data/748.pak:  0.0 s
+data/1020.pak:  2.24 s
+data/1570.pak:  0.82 s
+data/1391.pak:  0.51 s
+data/429.pak:   0.15 s
+data/2002.pak:  0.08 s
+data/1889.pak:  0.02 s
+data/1495.pak:  0.0 s
+data/748.pak:   0.0 s
 ---
-overall: 0.47 s
- memory: 451 m
-
-[test_msgpack_ruby]
-data/1020.pak: 1.91 s
-data/1570.pak: 0.76 s
-data/1391.pak: 0.52 s
-data/429.pak:  0.23 s
-data/2002.pak: 0.19 s
-data/1889.pak: 0.14 s
-data/1495.pak: 0.13 s
-data/748.pak:  0.13 s
----
-overall: 0.5 s
- memory: 216 m
+overall:        0.47
+ memory:        446 m
+
+ [test_msgpack_streaming]
+ data/1020.pak2: 0.8 s
+ data/1570.pak2: 0.28 s
+ data/1391.pak2: 0.18 s
+ data/429.pak2:  0.06 s
+ data/2002.pak2: 0.04 s
+ data/1889.pak2: 0.01 s
+ data/1495.pak2: 0.0 s
+ data/748.pak2:  0.0 s
+ ---
+ overall:        0.17
+  memory:        175 m
+
+  [test_msgpack_ruby]
+  data/1020.pak:  1.94 s
+  data/1570.pak:  0.77 s
+  data/1391.pak:  0.52 s
+  data/429.pak:   0.24 s
+  data/2002.pak:  0.19 s
+  data/1889.pak:  0.14 s
+  data/1495.pak:  0.13 s
+  data/748.pak:   0.13 s
+  ---
+  overall:        0.5
+   memory:        224 m
+
 </pre>
 
diff --git a/data/Makefile b/data/Makefile
index e56b229..24d85a3 100644
--- a/data/Makefile
+++ b/data/Makefile
@@ -1,6 +1,9 @@
 make_paks: make_paks.cc
 	g++ -std=c++11 make_paks.cc -I../msgpack-c/include/ ../msgpack-c/lib/libmsgpack.a -o make_paks
 
+make_paks2: make_paks2.cc
+	g++ -std=c++11 make_paks2.cc -I../msgpack-c/include/ ../msgpack-c/lib/libmsgpack.a -o make_paks2
+
 clean:
 	rm -f make_paks
 
diff --git a/data/make_paks.cc b/data/make_paks.cc
index 3477294..ca6c9b2 100644
--- a/data/make_paks.cc
+++ b/data/make_paks.cc
@@ -60,15 +60,14 @@ struct Hg {
   Vector weights;
   vector<Node> nodes;
   vector<Edge> edges;
-  vector<string> rules;
 
-  MSGPACK_DEFINE(weights, nodes, edges, rules);
+  MSGPACK_DEFINE(weights, nodes, edges);
 };
 
 template<typename X> inline void
 serialize(jsoncpp::Stream<X>& stream, Hg& o)
 {
-  fields(o, stream, "weights", o.weights, "nodes", o.nodes, "edges", o.edges, "rules", o.rules);
+  fields(o, stream, "weights", o.weights, "nodes", o.nodes, "edges", o.edges);
 }
 
 template<typename X> inline void
diff --git a/data/make_paks2.cc b/data/make_paks2.cc
new file mode 100644
index 0000000..1b5895b
--- /dev/null
+++ b/data/make_paks2.cc
@@ -0,0 +1,121 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <msgpack.hpp>
+#include <msgpack/fbuffer.h>
+#include <msgpack/fbuffer.hpp>
+
+
+/*
+ * https://github.com/ascheglov/json-cpp
+ *
+ */
+#include "../json-cpp.hpp"
+
+using namespace std;
+
+
+struct Node {
+  int id;
+  string cat;
+  vector<int> span;
+
+  MSGPACK_DEFINE(id, cat, span);
+};
+
+struct Vector {
+  double CountEF;
+  double EgivenFCoherent;
+  double Glue;
+  double IsSingletonF;
+  double IsSingletonFE;
+  double LanguageModel;
+  double LanguageModel_OOV;
+  double MaxLexFgivenE;
+  double MaxLexEgivenF;
+  double PassThrough;
+  double PassThrough_1;
+  double PassThrough_2;
+  double PassThrough_3;
+  double PassThrough_4;
+  double PassThrough_5;
+  double PassThrough_6;
+  double SampleCountF;
+  double WordPenalty;
+
+	MSGPACK_DEFINE(CountEF, EgivenFCoherent, Glue, IsSingletonF, IsSingletonFE, LanguageModel, LanguageModel_OOV, MaxLexEgivenF, MaxLexFgivenE, PassThrough, PassThrough_1, PassThrough_2, PassThrough_3, PassThrough_4, PassThrough_5, PassThrough_6, SampleCountF, WordPenalty);
+};
+
+struct Edge {
+  int head;
+  string rule;
+  vector<int> tails;
+  Vector f;
+  double weight;
+
+	MSGPACK_DEFINE(head, rule, tails, f, weight);
+};
+
+struct Hg {
+  Vector weights;
+  vector<Node> nodes;
+  vector<Edge> edges;
+
+  MSGPACK_DEFINE(weights, nodes, edges);
+};
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Hg& o)
+{
+  fields(o, stream, "weights", o.weights, "nodes", o.nodes, "edges", o.edges);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Edge& o)
+{
+  fields(o, stream, "head", o.head, "rule", o.rule, "tails", o.tails, "f", o.f, "weight", o.weight);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Vector& o)
+{
+  fields(o, stream, "EgivenFCoherent", o.EgivenFCoherent, "SampleCountF", o.SampleCountF, "CountEF", o.CountEF, "MaxLexFgivenE", o.MaxLexFgivenE, "MaxLexEgivenF", o.MaxLexEgivenF, "IsSingletonF", o.IsSingletonF, "IsSingletonFE", o.IsSingletonFE, "LanguageModel", o.LanguageModel, "LanguageModel_OOV", o.LanguageModel_OOV, "PassThrough", o.PassThrough, "PassThrough_1", o.PassThrough_1, "PassThrough_2", o.PassThrough_2, "PassThrough_3", o.PassThrough_3, "PassThrough_4", o.PassThrough_4, "PassThrough_5", o.PassThrough_5, "PassThrough_6", o.PassThrough_6, "WordPenalty", o.WordPenalty, "Glue", o.Glue);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Node& o)
+{
+  fields(o, stream, "id", o.id, "cat", o.cat, "span", o.span);
+}
+
+int
+main(int argc, char** argv)
+{
+  ifstream ifs(argv[1]);
+  string json_str((istreambuf_iterator<char>(ifs) ),
+                   (istreambuf_iterator<char>()));
+
+  Hg hg;
+  Vector w;
+  hg.weights = w;
+  vector<Node> nodes;
+  hg.nodes = nodes;
+  vector<Edge> edges;
+  hg.edges = edges;
+  jsoncpp::parse(hg, json_str);
+
+  FILE* file = fopen(argv[2], "wb");
+  msgpack::fbuffer fbuf(file);
+  msgpack::pack(fbuf, hg.nodes.size());
+  msgpack::pack(fbuf, hg.edges.size());
+  msgpack::pack(fbuf, hg.weights);
+  for (auto it = hg.nodes.begin(); it != hg.nodes.end(); it++)
+    msgpack::pack(fbuf, *it);
+  for (auto it = hg.edges.begin(); it != hg.edges.end(); it++)
+    msgpack::pack(fbuf, *it);
+
+  fclose(file);
+
+  return 0;
+}
+
diff --git a/run_msgpack.sh b/run_msgpack.sh
index 46c8127..b2f6eb5 100755
--- a/run_msgpack.sh
+++ b/run_msgpack.sh
@@ -15,12 +15,18 @@ echo
 
 for prg in \
   test_msgpack \
+  test_msgpack_streaming \
   test_msgpack_ruby
 do
   echo "[$prg]"
   sync; echo 3 > /proc/sys/vm/drop_caches
   echo > .overall_msgpack
-  for file in `ls -S data/*.pak`; do
+  if [[ $prg == test_msgpack_streaming ]]; then
+    A="2" 
+  else
+    A=""
+  fi
+  for file in `ls -S data/*.pak$A`; do
     echo "$file:\t$(./benchmark.rb $REPEAT ./$prg $file 2>/dev/null | tee -a .overall_msgpack | avg | round 2) s"
   done
   echo "---"
diff --git a/test_msgpack.cc b/test_msgpack.cc
index 1204b05..70cf1c8 100644
--- a/test_msgpack.cc
+++ b/test_msgpack.cc
@@ -7,7 +7,6 @@
  *
  */
 #include <msgpack.hpp>
-#include <msgpack/fbuffer.h>
 #include <msgpack/fbuffer.hpp>
 
 using namespace std;
@@ -58,9 +57,8 @@ struct Hg {
   Vector weights;
   vector<Node> nodes;
   vector<Edge> edges;
-  vector<string> rules;
 
-  MSGPACK_DEFINE(weights, nodes, edges, rules);
+  MSGPACK_DEFINE(weights, nodes, edges);
 };
 
 int
diff --git a/test_msgpack_streaming.cc b/test_msgpack_streaming.cc
new file mode 100644
index 0000000..1a3cf55
--- /dev/null
+++ b/test_msgpack_streaming.cc
@@ -0,0 +1,99 @@
+#include <msgpack.hpp>
+#include <msgpack/fbuffer.hpp>
+#include <iostream>
+#include <string>
+#include <fstream>
+
+using namespace std;
+ 
+
+
+struct Node {
+  int id;
+  string cat;
+  vector<int> span;
+
+  MSGPACK_DEFINE(id, cat, span);
+};
+
+struct Vector {
+  double CountEF;
+  double EgivenFCoherent;
+  double Glue;
+  double IsSingletonF;
+  double IsSingletonFE;
+  double LanguageModel;
+  double LanguageModel_OOV;
+  double MaxLexFgivenE;
+  double MaxLexEgivenF;
+  double PassThrough;
+  double PassThrough_1;
+  double PassThrough_2;
+  double PassThrough_3;
+  double PassThrough_4;
+  double PassThrough_5;
+  double PassThrough_6;
+  double SampleCountF;
+  double WordPenalty;
+
+	MSGPACK_DEFINE(CountEF, EgivenFCoherent, Glue, IsSingletonF, IsSingletonFE, LanguageModel, LanguageModel_OOV, MaxLexEgivenF, MaxLexFgivenE, PassThrough, PassThrough_1, PassThrough_2, PassThrough_3, PassThrough_4, PassThrough_5, PassThrough_6, SampleCountF, WordPenalty);
+};
+
+struct Edge {
+  int head;
+  string rule;
+  vector<int> tails;
+  Vector f;
+  double weight;
+
+	MSGPACK_DEFINE(head, rule, tails, f, weight);
+};
+
+struct Hg {
+  Vector weights;
+  vector<Node> nodes;
+  vector<Edge> edges;
+
+  MSGPACK_DEFINE(weights, nodes, edges);
+};
+
+int
+main(int argc, char** argv) {
+  ifstream ifs(argv[1]);
+
+  size_t count = 0, n_, e_;
+  msgpack::unpacker pac;
+  while(true) {
+      pac.reserve_buffer(32*1024);
+      size_t bytes = ifs.readsome(pac.buffer(), pac.buffer_capacity());
+      pac.buffer_consumed(bytes);
+      msgpack::unpacked result;
+      while(pac.next(&result)) {
+          msgpack::object obj = result.get();
+          if (count ==  0) {
+            obj.convert(&n_);
+            n_ += 2;
+          } else  if (count == 1) {
+            obj.convert(&e_);
+            e_ += 2;
+          } else if (count == 2) {
+            Vector v; 
+            obj.convert(&v);
+          } else if (count > 2 && count <= n_) { 
+            Node n;
+            obj.convert(&n);
+          } else if (count > n_ && count <= n_+e_+1) {
+            Edge e;
+            obj.convert(&e);
+            string s = e.rule.substr(1, 4);
+            if (s == "Goal")
+              cout << s << endl;
+          }
+          count++;
+      }
+      if (!bytes) break;
+  }
+
+  return 0;
+}
+