Merge pull request #36 from pks/master

RuleWordAlignmentFeatures, minor dtrain changes

3 files changed, 60 insertions, 59 deletions

diff --git a/training/dtrain/examples/standard/cdec.ini b/training/dtrain/examples/standard/cdec.ini
index e1edc68d..6cba9e1e 100644
--- a/training/dtrain/examples/standard/cdec.ini
+++ b/training/dtrain/examples/standard/cdec.ini
@@ -21,6 +21,7 @@ feature_function=RuleIdentityFeatures
 feature_function=RuleSourceBigramFeatures
 feature_function=RuleTargetBigramFeatures
 feature_function=RuleShape
+feature_function=RuleWordAlignmentFeatures
 #feature_function=SourceSpanSizeFeatures
 #feature_function=SourceWordPenalty
 #feature_function=SpanFeatures
diff --git a/training/dtrain/examples/standard/dtrain.ini b/training/dtrain/examples/standard/dtrain.ini
index fc83f08e..a515db02 100644
--- a/training/dtrain/examples/standard/dtrain.ini
+++ b/training/dtrain/examples/standard/dtrain.ini
@@ -2,7 +2,7 @@
 #refs=./nc-wmt11.en.gz
 bitext=./nc-wmt11.gz
 output=-                  # a weights file (add .gz for gzip compression) or STDOUT '-'
-select_weights=VOID       # output average (over epochs) weight vector
+select_weights=avg        # output average (over epochs) weight vector
 decoder_config=./cdec.ini # config for cdec
 # weights for these features will be printed on each iteration
 print_weights=Glue WordPenalty LanguageModel LanguageModel_OOV PhraseModel_0 PhraseModel_1 PhraseModel_2 PhraseModel_3 PhraseModel_4 PhraseModel_5 PhraseModel_6 PassThrough
diff --git a/training/dtrain/examples/standard/expected-output b/training/dtrain/examples/standard/expected-output
index 75f47337..fa831221 100644
--- a/training/dtrain/examples/standard/expected-output
+++ b/training/dtrain/examples/standard/expected-output
@@ -4,7 +4,7 @@ Reading ./nc-wmt11.en.srilm.gz
 ----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100
 ****************************************************************************************************
   Example feature: Shape_S00000_T00000
-Seeding random number sequence to 3751911392
+Seeding random number sequence to 4138446869
 
 dtrain
 Parameters:
@@ -22,7 +22,7 @@ Parameters:
                    pairs 'XYX'
                    hi lo 0.1
           pair threshold 0
-          select weights 'VOID'
+          select weights 'avg'
                   l1 reg 0 'none'
                     pclr no
                max pairs 4294967295
@@ -36,87 +36,87 @@ Iteration #1 of 3.
  . 10
 Stopping after 10 input sentences.
 WEIGHTS
-              Glue = -110
-       WordPenalty = -8.2082
-     LanguageModel = -319.91
- LanguageModel_OOV = -19.2
-     PhraseModel_0 = +312.82
-     PhraseModel_1 = -161.02
-     PhraseModel_2 = -433.65
-     PhraseModel_3 = +291.03
-     PhraseModel_4 = +252.32
-     PhraseModel_5 = +50.6
-     PhraseModel_6 = +146.7
-       PassThrough = -38.7
+              Glue = -80.3
+       WordPenalty = -51.247
+     LanguageModel = +282.46
+ LanguageModel_OOV = -85.8
+     PhraseModel_0 = -100.06
+     PhraseModel_1 = -98.692
+     PhraseModel_2 = -9.4958
+     PhraseModel_3 = +18.535
+     PhraseModel_4 = +62.35
+     PhraseModel_5 = +7
+     PhraseModel_6 = +31.4
+       PassThrough = -126.5
         ---
-       1best avg score: 0.16966 (+0.16966)
- 1best avg model score: 29874 (+29874)
-           avg # pairs: 906.3
+       1best avg score: 0.25631 (+0.25631)
+ 1best avg model score: -4843.6 (-4843.6)
+           avg # pairs: 744.4
         avg # rank err: 0 (meaningless)
      avg # margin viol: 0
        k-best loss imp: 100%
-    non0 feature count: 832
+    non0 feature count: 1274
            avg list sz: 91.3
-           avg f count: 139.77
-(time 0.35 min, 2.1 s/S)
+           avg f count: 143.72
+(time 0.4 min, 2.4 s/S)
 
 Iteration #2 of 3.
  . 10
 WEIGHTS
-              Glue = -122.1
-       WordPenalty = +83.689
-     LanguageModel = +233.23
- LanguageModel_OOV = -145.1
-     PhraseModel_0 = +150.72
-     PhraseModel_1 = -272.84
-     PhraseModel_2 = -418.36
-     PhraseModel_3 = +181.63
-     PhraseModel_4 = -289.47
-     PhraseModel_5 = +140.3
-     PhraseModel_6 = +3.5
-       PassThrough = -109.7
+              Glue = -117.4
+       WordPenalty = -99.584
+     LanguageModel = +395.05
+ LanguageModel_OOV = -136.8
+     PhraseModel_0 = +40.614
+     PhraseModel_1 = -123.29
+     PhraseModel_2 = -152
+     PhraseModel_3 = -161.13
+     PhraseModel_4 = -76.379
+     PhraseModel_5 = +39.1
+     PhraseModel_6 = +137.7
+       PassThrough = -162.1
         ---
-       1best avg score: 0.17399 (+0.004325)
- 1best avg model score: 4936.9 (-24937)
-           avg # pairs: 662.4
+       1best avg score: 0.26751 (+0.011198)
+ 1best avg model score: -10061 (-5216.9)
+           avg # pairs: 639.1
         avg # rank err: 0 (meaningless)
      avg # margin viol: 0
        k-best loss imp: 100%
-    non0 feature count: 1240
+    non0 feature count: 1845
            avg list sz: 91.3
-           avg f count: 125.11
-(time 0.27 min, 1.6 s/S)
+           avg f count: 139.88
+(time 0.35 min, 2.1 s/S)
 
 Iteration #3 of 3.
  . 10
 WEIGHTS
-              Glue = -157.4
-       WordPenalty = -1.7372
-     LanguageModel = +686.18
- LanguageModel_OOV = -399.7
-     PhraseModel_0 = -39.876
-     PhraseModel_1 = -341.96
-     PhraseModel_2 = -318.67
-     PhraseModel_3 = +105.08
-     PhraseModel_4 = -290.27
-     PhraseModel_5 = -48.6
-     PhraseModel_6 = -43.6
-       PassThrough = -298.5
+              Glue = -101.1
+       WordPenalty = -139.97
+     LanguageModel = +327.98
+ LanguageModel_OOV = -234.7
+     PhraseModel_0 = -144.49
+     PhraseModel_1 = -263.88
+     PhraseModel_2 = -149.25
+     PhraseModel_3 = -38.805
+     PhraseModel_4 = +50.575
+     PhraseModel_5 = -52.4
+     PhraseModel_6 = +41.6
+       PassThrough = -230.2
         ---
-       1best avg score: 0.30742 (+0.13343)
- 1best avg model score: -15393 (-20329)
-           avg # pairs: 623.8
+       1best avg score: 0.36222 (+0.094717)
+ 1best avg model score: -17416 (-7355.5)
+           avg # pairs: 661.2
         avg # rank err: 0 (meaningless)
      avg # margin viol: 0
        k-best loss imp: 100%
-    non0 feature count: 1776
+    non0 feature count: 2163
            avg list sz: 91.3
-           avg f count: 118.58
-(time 0.28 min, 1.7 s/S)
+           avg f count: 132.53
+(time 0.33 min, 2 s/S)
 
 Writing weights file to '-' ...
 done
 
 ---
-Best iteration: 3 [SCORE 'fixed_stupid_bleu'=0.30742].
-This took 0.9 min.
+Best iteration: 3 [SCORE 'fixed_stupid_bleu'=0.36222].
+This took 1.0833 min.