first implementation of dtrain_net and downpour

7 files changed, 70 insertions, 0 deletions

diff --git a/training/dtrain/examples/net/10.gz b/training/dtrain/examples/net/10.gz
new file mode 100644
index 00000000..44775573
--- /dev/null
+++ b/training/dtrain/examples/net/10.gz
diff --git a/training/dtrain/examples/net/README b/training/dtrain/examples/net/README
new file mode 100644
index 00000000..4acb721b
--- /dev/null
+++ b/training/dtrain/examples/net/README
@@ -0,0 +1,6 @@
+run
+ ../../downpour.rb -c dtrain.ini -p 3 -i 10.gz -l 0.00001 -e 3
+or
+ zcat 10.gz | head -6 | ../../feed.rb 60667
+ ../../dtrain_net -c dtrain.ini -a tcp://127.0.0.1:60667
+
diff --git a/training/dtrain/examples/net/cdec.ini b/training/dtrain/examples/net/cdec.ini
new file mode 100644
index 00000000..6c986d03
--- /dev/null
+++ b/training/dtrain/examples/net/cdec.ini
@@ -0,0 +1,27 @@
+formalism=scfg
+add_pass_through_rules=true
+scfg_max_span_limit=15
+intersection_strategy=cube_pruning
+cubepruning_pop_limit=200
+grammar=../standard/nc-wmt11.grammar.gz
+feature_function=WordPenalty
+feature_function=KLanguageModel ../standard/nc-wmt11.en.srilm.gz
+# all currently working feature functions for translation:
+# (with those features active that were used in the ACL paper)
+#feature_function=ArityPenalty
+#feature_function=CMR2008ReorderingFeatures
+#feature_function=Dwarf
+#feature_function=InputIndicator
+#feature_function=LexNullJump
+#feature_function=NewJump
+#feature_function=NgramFeatures
+#feature_function=NonLatinCount
+#feature_function=OutputIndicator
+#feature_function=RuleIdentityFeatures
+#feature_function=RuleSourceBigramFeatures
+#feature_function=RuleTargetBigramFeatures
+#feature_function=RuleShape
+#feature_function=LexicalFeatures 1 1 1
+#feature_function=SourceSpanSizeFeatures
+#feature_function=SourceWordPenalty
+#feature_function=SpanFeatures
diff --git a/training/dtrain/examples/net/dtrain.ini b/training/dtrain/examples/net/dtrain.ini
new file mode 100644
index 00000000..cfcd91bc
--- /dev/null
+++ b/training/dtrain/examples/net/dtrain.ini
@@ -0,0 +1,4 @@
+decoder_conf=./cdec.ini # config for cdec
+k=100                   # use 100best lists
+N=4                     # optimize (approx.) BLEU4
+margin=1.0              # perceptron's margin
diff --git a/training/dtrain/examples/net/work/out.0 b/training/dtrain/examples/net/work/out.0
new file mode 100644
index 00000000..37b0ea44
--- /dev/null
+++ b/training/dtrain/examples/net/work/out.0
@@ -0,0 +1,11 @@
+Loading the LM will be faster if you build a binary file.
+Reading ../standard/nc-wmt11.en.srilm.gz
+----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100
+****************************************************************************************************
+dtrain_net
+Parameters:
+                       k 100
+                       N 4
+                  margin 1
+            decoder conf './cdec.ini'
+ ........ 8
diff --git a/training/dtrain/examples/net/work/out.1 b/training/dtrain/examples/net/work/out.1
new file mode 100644
index 00000000..187b726e
--- /dev/null
+++ b/training/dtrain/examples/net/work/out.1
@@ -0,0 +1,11 @@
+Loading the LM will be faster if you build a binary file.
+Reading ../standard/nc-wmt11.en.srilm.gz
+----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100
+****************************************************************************************************
+dtrain_net
+Parameters:
+                       k 100
+                       N 4
+                  margin 1
+            decoder conf './cdec.ini'
+ ........... 11
diff --git a/training/dtrain/examples/net/work/out.2 b/training/dtrain/examples/net/work/out.2
new file mode 100644
index 00000000..187b726e
--- /dev/null
+++ b/training/dtrain/examples/net/work/out.2
@@ -0,0 +1,11 @@
+Loading the LM will be faster if you build a binary file.
+Reading ../standard/nc-wmt11.en.srilm.gz
+----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100
+****************************************************************************************************
+dtrain_net
+Parameters:
+                       k 100
+                       N 4
+                  margin 1
+            decoder conf './cdec.ini'
+ ........... 11