From 9f83329836e45037bf471de4a78068fc7ab9b7e1 Mon Sep 17 00:00:00 2001 From: Patrick Simianer Date: Wed, 16 Jul 2014 11:13:29 +0200 Subject: msgpack streaming --- .gitignore | 4 +- Makefile | 3 ++ README.md | 60 ++++++++++++++--------- data/Makefile | 3 ++ data/make_paks.cc | 5 +- data/make_paks2.cc | 121 ++++++++++++++++++++++++++++++++++++++++++++++ run_msgpack.sh | 8 ++- test_msgpack.cc | 4 +- test_msgpack_streaming.cc | 99 +++++++++++++++++++++++++++++++++++++ 9 files changed, 276 insertions(+), 31 deletions(-) create mode 100644 data/make_paks2.cc create mode 100644 test_msgpack_streaming.cc diff --git a/.gitignore b/.gitignore index 80d3144..b740083 100644 --- a/.gitignore +++ b/.gitignore @@ -25,8 +25,10 @@ test_picojson test_rapidjson test_sajson test_msgpack +test_msgpack_streaming data/make_paks -data/*.pak +data/make_paks2 +data/*.pak* data/*.json data/*.gz data/*.in diff --git a/Makefile b/Makefile index bf290a4..3013d32 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,9 @@ test_cdec_json_parser: test_cdec_json_parser.cc test_msgpack: test_msgpack.cc $(COMPILER) $(CXXFLAGS) test_msgpack.cc -I./msgpack-c/include/ ./msgpack-c/lib/libmsgpack.a -o test_msgpack +test_msgpack_streaming: test_msgpack_streaming.cc + $(COMPILER) $(CXXFLAGS) test_msgpack_streaming.cc -I./msgpack-c/include/ ./msgpack-c/lib/libmsgpack.a -o test_msgpack_streaming + clean: rm -f test_gason test_json-cpp test_jsoncpp test_libjson rm -f test_picojson test_rapidjson test_sajson test_JsonBox diff --git a/README.md b/README.md index d1c1ea5..808bac8 100644 --- a/README.md +++ b/README.md @@ -217,29 +217,43 @@ MSGPACK parsing benchmark 
 [test_msgpack]
-data/1020.pak: 2.2 s
-data/1570.pak: 0.8 s
-data/1391.pak: 0.5 s
-data/429.pak:  0.15 s
-data/2002.pak: 0.09 s
-data/1889.pak: 0.02 s
-data/1495.pak: 0.0 s
-data/748.pak:  0.0 s
+data/1020.pak:  2.24 s
+data/1570.pak:  0.82 s
+data/1391.pak:  0.51 s
+data/429.pak:   0.15 s
+data/2002.pak:  0.08 s
+data/1889.pak:  0.02 s
+data/1495.pak:  0.0 s
+data/748.pak:   0.0 s
 ---
-overall: 0.47 s
- memory: 451 m
-
-[test_msgpack_ruby]
-data/1020.pak: 1.91 s
-data/1570.pak: 0.76 s
-data/1391.pak: 0.52 s
-data/429.pak:  0.23 s
-data/2002.pak: 0.19 s
-data/1889.pak: 0.14 s
-data/1495.pak: 0.13 s
-data/748.pak:  0.13 s
----
-overall: 0.5 s
- memory: 216 m
+overall:        0.47
+ memory:        446 m
+
+ [test_msgpack_streaming]
+ data/1020.pak2: 0.8 s
+ data/1570.pak2: 0.28 s
+ data/1391.pak2: 0.18 s
+ data/429.pak2:  0.06 s
+ data/2002.pak2: 0.04 s
+ data/1889.pak2: 0.01 s
+ data/1495.pak2: 0.0 s
+ data/748.pak2:  0.0 s
+ ---
+ overall:        0.17
+  memory:        175 m
+
+  [test_msgpack_ruby]
+  data/1020.pak:  1.94 s
+  data/1570.pak:  0.77 s
+  data/1391.pak:  0.52 s
+  data/429.pak:   0.24 s
+  data/2002.pak:  0.19 s
+  data/1889.pak:  0.14 s
+  data/1495.pak:  0.13 s
+  data/748.pak:   0.13 s
+  ---
+  overall:        0.5
+   memory:        224 m
+
diff --git a/data/Makefile b/data/Makefile index e56b229..24d85a3 100644 --- a/data/Makefile +++ b/data/Makefile @@ -1,6 +1,9 @@ make_paks: make_paks.cc g++ -std=c++11 make_paks.cc -I../msgpack-c/include/ ../msgpack-c/lib/libmsgpack.a -o make_paks +make_paks2: make_paks2.cc + g++ -std=c++11 make_paks2.cc -I../msgpack-c/include/ ../msgpack-c/lib/libmsgpack.a -o make_paks2 + clean: rm -f make_paks diff --git a/data/make_paks.cc b/data/make_paks.cc index 3477294..ca6c9b2 100644 --- a/data/make_paks.cc +++ b/data/make_paks.cc @@ -60,15 +60,14 @@ struct Hg { Vector weights; vector nodes; vector edges; - vector rules; - MSGPACK_DEFINE(weights, nodes, edges, rules); + MSGPACK_DEFINE(weights, nodes, edges); }; template inline void serialize(jsoncpp::Stream& stream, Hg& o) { - fields(o, stream, "weights", o.weights, "nodes", o.nodes, "edges", o.edges, "rules", o.rules); + fields(o, stream, "weights", o.weights, "nodes", o.nodes, "edges", o.edges); } template inline void diff --git a/data/make_paks2.cc b/data/make_paks2.cc new file mode 100644 index 0000000..1b5895b --- /dev/null +++ b/data/make_paks2.cc @@ -0,0 +1,121 @@ +#include +#include +#include +#include +#include +#include + + +/* + * https://github.com/ascheglov/json-cpp + * + */ +#include "../json-cpp.hpp" + +using namespace std; + + +struct Node { + int id; + string cat; + vector span; + + MSGPACK_DEFINE(id, cat, span); +}; + +struct Vector { + double CountEF; + double EgivenFCoherent; + double Glue; + double IsSingletonF; + double IsSingletonFE; + double LanguageModel; + double LanguageModel_OOV; + double MaxLexFgivenE; + double MaxLexEgivenF; + double PassThrough; + double PassThrough_1; + double PassThrough_2; + double PassThrough_3; + double PassThrough_4; + double PassThrough_5; + double PassThrough_6; + double SampleCountF; + double WordPenalty; + + MSGPACK_DEFINE(CountEF, EgivenFCoherent, Glue, IsSingletonF, IsSingletonFE, LanguageModel, LanguageModel_OOV, MaxLexEgivenF, MaxLexFgivenE, PassThrough, PassThrough_1, PassThrough_2, PassThrough_3, PassThrough_4, PassThrough_5, PassThrough_6, SampleCountF, WordPenalty); +}; + +struct Edge { + int head; + string rule; + vector tails; + Vector f; + double weight; + + MSGPACK_DEFINE(head, rule, tails, f, weight); +}; + +struct Hg { + Vector weights; + vector nodes; + vector edges; + + MSGPACK_DEFINE(weights, nodes, edges); +}; + +template inline void +serialize(jsoncpp::Stream& stream, Hg& o) +{ + fields(o, stream, "weights", o.weights, "nodes", o.nodes, "edges", o.edges); +} + +template inline void +serialize(jsoncpp::Stream& stream, Edge& o) +{ + fields(o, stream, "head", o.head, "rule", o.rule, "tails", o.tails, "f", o.f, "weight", o.weight); +} + +template inline void +serialize(jsoncpp::Stream& stream, Vector& o) +{ + fields(o, stream, "EgivenFCoherent", o.EgivenFCoherent, "SampleCountF", o.SampleCountF, "CountEF", o.CountEF, "MaxLexFgivenE", o.MaxLexFgivenE, "MaxLexEgivenF", o.MaxLexEgivenF, "IsSingletonF", o.IsSingletonF, "IsSingletonFE", o.IsSingletonFE, "LanguageModel", o.LanguageModel, "LanguageModel_OOV", o.LanguageModel_OOV, "PassThrough", o.PassThrough, "PassThrough_1", o.PassThrough_1, "PassThrough_2", o.PassThrough_2, "PassThrough_3", o.PassThrough_3, "PassThrough_4", o.PassThrough_4, "PassThrough_5", o.PassThrough_5, "PassThrough_6", o.PassThrough_6, "WordPenalty", o.WordPenalty, "Glue", o.Glue); +} + +template inline void +serialize(jsoncpp::Stream& stream, Node& o) +{ + fields(o, stream, "id", o.id, "cat", o.cat, "span", o.span); +} + +int +main(int argc, char** argv) +{ + ifstream ifs(argv[1]); + string json_str((istreambuf_iterator(ifs) ), + (istreambuf_iterator())); + + Hg hg; + Vector w; + hg.weights = w; + vector nodes; + hg.nodes = nodes; + vector edges; + hg.edges = edges; + jsoncpp::parse(hg, json_str); + + FILE* file = fopen(argv[2], "wb"); + msgpack::fbuffer fbuf(file); + msgpack::pack(fbuf, hg.nodes.size()); + msgpack::pack(fbuf, hg.edges.size()); + msgpack::pack(fbuf, hg.weights); + for (auto it = hg.nodes.begin(); it != hg.nodes.end(); it++) + msgpack::pack(fbuf, *it); + for (auto it = hg.edges.begin(); it != hg.edges.end(); it++) + msgpack::pack(fbuf, *it); + + fclose(file); + + return 0; +} + diff --git a/run_msgpack.sh b/run_msgpack.sh index 46c8127..b2f6eb5 100755 --- a/run_msgpack.sh +++ b/run_msgpack.sh @@ -15,12 +15,18 @@ echo for prg in \ test_msgpack \ + test_msgpack_streaming \ test_msgpack_ruby do echo "[$prg]" sync; echo 3 > /proc/sys/vm/drop_caches echo > .overall_msgpack - for file in `ls -S data/*.pak`; do + if [[ $prg == test_msgpack_streaming ]]; then + A="2" + else + A="" + fi + for file in `ls -S data/*.pak$A`; do echo "$file:\t$(./benchmark.rb $REPEAT ./$prg $file 2>/dev/null | tee -a .overall_msgpack | avg | round 2) s" done echo "---" diff --git a/test_msgpack.cc b/test_msgpack.cc index 1204b05..70cf1c8 100644 --- a/test_msgpack.cc +++ b/test_msgpack.cc @@ -7,7 +7,6 @@ * */ #include -#include #include using namespace std; @@ -58,9 +57,8 @@ struct Hg { Vector weights; vector nodes; vector edges; - vector rules; - MSGPACK_DEFINE(weights, nodes, edges, rules); + MSGPACK_DEFINE(weights, nodes, edges); }; int diff --git a/test_msgpack_streaming.cc b/test_msgpack_streaming.cc new file mode 100644 index 0000000..1a3cf55 --- /dev/null +++ b/test_msgpack_streaming.cc @@ -0,0 +1,99 @@ +#include +#include +#include +#include +#include + +using namespace std; + + + +struct Node { + int id; + string cat; + vector span; + + MSGPACK_DEFINE(id, cat, span); +}; + +struct Vector { + double CountEF; + double EgivenFCoherent; + double Glue; + double IsSingletonF; + double IsSingletonFE; + double LanguageModel; + double LanguageModel_OOV; + double MaxLexFgivenE; + double MaxLexEgivenF; + double PassThrough; + double PassThrough_1; + double PassThrough_2; + double PassThrough_3; + double PassThrough_4; + double PassThrough_5; + double PassThrough_6; + double SampleCountF; + double WordPenalty; + + MSGPACK_DEFINE(CountEF, EgivenFCoherent, Glue, IsSingletonF, IsSingletonFE, LanguageModel, LanguageModel_OOV, MaxLexEgivenF, MaxLexFgivenE, PassThrough, PassThrough_1, PassThrough_2, PassThrough_3, PassThrough_4, PassThrough_5, PassThrough_6, SampleCountF, WordPenalty); +}; + +struct Edge { + int head; + string rule; + vector tails; + Vector f; + double weight; + + MSGPACK_DEFINE(head, rule, tails, f, weight); +}; + +struct Hg { + Vector weights; + vector nodes; + vector edges; + + MSGPACK_DEFINE(weights, nodes, edges); +}; + +int +main(int argc, char** argv) { + ifstream ifs(argv[1]); + + size_t count = 0, n_, e_; + msgpack::unpacker pac; + while(true) { + pac.reserve_buffer(32*1024); + size_t bytes = ifs.readsome(pac.buffer(), pac.buffer_capacity()); + pac.buffer_consumed(bytes); + msgpack::unpacked result; + while(pac.next(&result)) { + msgpack::object obj = result.get(); + if (count == 0) { + obj.convert(&n_); + n_ += 2; + } else if (count == 1) { + obj.convert(&e_); + e_ += 2; + } else if (count == 2) { + Vector v; + obj.convert(&v); + } else if (count > 2 && count <= n_) { + Node n; + obj.convert(&n); + } else if (count > n_ && count <= n_+e_+1) { + Edge e; + obj.convert(&e); + string s = e.rule.substr(1, 4); + if (s == "Goal") + cout << s << endl; + } + count++; + } + if (!bytes) break; + } + + return 0; +} + -- cgit v1.2.3