summaryrefslogtreecommitdiff
path: root/data
diff options
context:
space:
mode:
authorPatrick Simianer <p@simianer.de>2014-07-13 14:04:45 +0200
committerPatrick Simianer <p@simianer.de>2014-07-13 14:04:45 +0200
commit625269764ebbe8d0b566e6ef5fc26a6bccd4181d (patch)
tree70c54cb1dae9afddaefd7f0824427e95dd688081 /data
init
Diffstat (limited to 'data')
-rw-r--r--data/Makefile6
-rw-r--r--data/cdec.ini4
-rwxr-xr-xdata/make.sh8
-rw-r--r--data/make_paks.cc126
-rwxr-xr-xdata/to_ascii.rb13
-rw-r--r--data/weights.init12
6 files changed, 169 insertions, 0 deletions
diff --git a/data/Makefile b/data/Makefile
new file mode 100644
index 0000000..e56b229
--- /dev/null
+++ b/data/Makefile
@@ -0,0 +1,6 @@
+make_paks: make_paks.cc
+ g++ -std=c++11 make_paks.cc -I../msgpack-c/include/ ../msgpack-c/lib/libmsgpack.a -o make_paks
+
+clean:
+ rm -f make_paks
+
diff --git a/data/cdec.ini b/data/cdec.ini
new file mode 100644
index 0000000..ddbe54c
--- /dev/null
+++ b/data/cdec.ini
@@ -0,0 +1,4 @@
+formalism=scfg
+intersection_strategy=full
+add_pass_through_rules=true
+
diff --git a/data/make.sh b/data/make.sh
new file mode 100755
index 0000000..5e0c31b
--- /dev/null
+++ b/data/make.sh
@@ -0,0 +1,8 @@
+#!/bin/zsh
+
+
+# wmt/14/newstest2008 data
+for i in 1020 1391 1495 1570 1889 2002 429 748; do
+ ~/src/weaver/util/cdec2json.py -c cdec.ini -w weights.init -g grammar.$i.gz < $i.in | ./to_ascii.rb > $i.json
+done
+
diff --git a/data/make_paks.cc b/data/make_paks.cc
new file mode 100644
index 0000000..3477294
--- /dev/null
+++ b/data/make_paks.cc
@@ -0,0 +1,126 @@
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <msgpack.hpp>
+#include <msgpack/fbuffer.h>
+#include <msgpack/fbuffer.hpp>
+
+
+/*
+ * https://github.com/ascheglov/json-cpp
+ *
+ */
+#include "../json-cpp.hpp"
+
+using namespace std;
+
+
+struct Node {
+ int id;
+ string cat;
+ vector<int> span;
+
+ MSGPACK_DEFINE(id, cat, span);
+};
+
+struct Vector {
+ double CountEF;
+ double EgivenFCoherent;
+ double Glue;
+ double IsSingletonF;
+ double IsSingletonFE;
+ double LanguageModel;
+ double LanguageModel_OOV;
+ double MaxLexFgivenE;
+ double MaxLexEgivenF;
+ double PassThrough;
+ double PassThrough_1;
+ double PassThrough_2;
+ double PassThrough_3;
+ double PassThrough_4;
+ double PassThrough_5;
+ double PassThrough_6;
+ double SampleCountF;
+ double WordPenalty;
+
+ MSGPACK_DEFINE(CountEF, EgivenFCoherent, Glue, IsSingletonF, IsSingletonFE, LanguageModel, LanguageModel_OOV, MaxLexEgivenF, MaxLexFgivenE, PassThrough, PassThrough_1, PassThrough_2, PassThrough_3, PassThrough_4, PassThrough_5, PassThrough_6, SampleCountF, WordPenalty);
+};
+
+struct Edge {
+ int head;
+ string rule;
+ vector<int> tails;
+ Vector f;
+ double weight;
+
+ MSGPACK_DEFINE(head, rule, tails, f, weight);
+};
+
+struct Hg {
+ Vector weights;
+ vector<Node> nodes;
+ vector<Edge> edges;
+ vector<string> rules;
+
+ MSGPACK_DEFINE(weights, nodes, edges, rules);
+};
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Hg& o)
+{
+ fields(o, stream, "weights", o.weights, "nodes", o.nodes, "edges", o.edges, "rules", o.rules);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Edge& o)
+{
+ fields(o, stream, "head", o.head, "rule", o.rule, "tails", o.tails, "f", o.f, "weight", o.weight);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Vector& o)
+{
+ fields(o, stream, "EgivenFCoherent", o.EgivenFCoherent, "SampleCountF", o.SampleCountF, "CountEF", o.CountEF, "MaxLexFgivenE", o.MaxLexFgivenE, "MaxLexEgivenF", o.MaxLexEgivenF, "IsSingletonF", o.IsSingletonF, "IsSingletonFE", o.IsSingletonFE, "LanguageModel", o.LanguageModel, "LanguageModel_OOV", o.LanguageModel_OOV, "PassThrough", o.PassThrough, "PassThrough_1", o.PassThrough_1, "PassThrough_2", o.PassThrough_2, "PassThrough_3", o.PassThrough_3, "PassThrough_4", o.PassThrough_4, "PassThrough_5", o.PassThrough_5, "PassThrough_6", o.PassThrough_6, "WordPenalty", o.WordPenalty, "Glue", o.Glue);
+}
+
+template<typename X> inline void
+serialize(jsoncpp::Stream<X>& stream, Node& o)
+{
+ fields(o, stream, "id", o.id, "cat", o.cat, "span", o.span);
+}
+
+int
+main(int argc, char** argv)
+{
+ ifstream ifs(argv[1]);
+ string json_str((istreambuf_iterator<char>(ifs) ),
+ (istreambuf_iterator<char>()));
+
+ Hg hg;
+ Vector w;
+ hg.weights = w;
+ vector<Node> nodes;
+ hg.nodes = nodes;
+ vector<Edge> edges;
+ hg.edges = edges;
+ jsoncpp::parse(hg, json_str);
+
+ FILE* file = fopen(argv[2], "wb");
+ msgpack::fbuffer fbuf(file);
+ msgpack::pack(fbuf, hg);
+ fclose(file);
+
+ /*ifstream ifs1(argv[2]);
+ string str1((istreambuf_iterator<char>(jfs1)),
+ (istreambuf_iterator<char>()));
+
+ msgpack::zone zone;
+ msgpack::object obj;
+ msgpack::unpack(str1.data(), str1.size(), NULL, &zone, &obj);
+
+ Hg hg;
+ obj.convert(&hg);*/
+
+ return 0;
+}
+
diff --git a/data/to_ascii.rb b/data/to_ascii.rb
new file mode 100755
index 0000000..6c1d23e
--- /dev/null
+++ b/data/to_ascii.rb
@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+
+while line = STDIN.gets
+ encoding_options = {
+ :invalid => :replace,
+ :undef => :replace,
+ :replace => '?',
+ :universal_newline => true
+ }
+ puts line.encode 'ASCII', encoding_options
+end
+
diff --git a/data/weights.init b/data/weights.init
new file mode 100644
index 0000000..0d09f9f
--- /dev/null
+++ b/data/weights.init
@@ -0,0 +1,12 @@
+CountEF 0.1
+EgivenFCoherent -0.1
+Glue 0.01
+IsSingletonF -0.01
+IsSingletonFE -0.01
+LanguageModel 0.1
+LanguageModel_OOV -1
+MaxLexFgivenE -0.1
+MaxLexEgivenF -0.1
+PassThrough -0.1
+SampleCountF -0.1
+WordPenalty -0.1