diff options
Diffstat (limited to 'dtrain/test/example')
| -rw-r--r-- | dtrain/test/example/README | 6 | ||||
| -rw-r--r-- | dtrain/test/example/cdec.ini | 24 | ||||
| -rw-r--r-- | dtrain/test/example/dtrain.ini | 21 | ||||
| -rw-r--r-- | dtrain/test/example/nc-wmt11.1k.gz | bin | 0 -> 21185883 bytes | |||
| -rw-r--r-- | dtrain/test/example/nc-wmt11.en.srilm.gz | bin | 0 -> 16017291 bytes |
5 files changed, 51 insertions, 0 deletions
diff --git a/dtrain/test/example/README b/dtrain/test/example/README new file mode 100644 index 00000000..e5a5de59 --- /dev/null +++ b/dtrain/test/example/README @@ -0,0 +1,6 @@ +Small example of input format for distributed training. +Call dtrain from cdec/dtrain/ with ./dtrain -c test/example/dtrain.ini . + +For this to work, disable '#define DTRAIN_LOCAL' from dtrain.h +and recompile. + diff --git a/dtrain/test/example/cdec.ini b/dtrain/test/example/cdec.ini new file mode 100644 index 00000000..6642107f --- /dev/null +++ b/dtrain/test/example/cdec.ini @@ -0,0 +1,24 @@ +formalism=scfg +add_pass_through_rules=true +scfg_max_span_limit=15 +intersection_strategy=cube_pruning +cubepruning_pop_limit=30 +feature_function=WordPenalty +feature_function=KLanguageModel test/example/nc-wmt11.en.srilm.gz +# all currently working feature functions for translation: +# (with those features active that were used in the ACL paper) +#feature_function=ArityPenalty +#feature_function=CMR2008ReorderingFeatures +#feature_function=Dwarf +#feature_function=InputIndicator +#feature_function=LexNullJump +#feature_function=NewJump +#feature_function=NgramFeatures +#feature_function=NonLatinCount +#feature_function=OutputIndicator +feature_function=RuleIdentityFeatures +feature_function=RuleNgramFeatures +feature_function=RuleShape +#feature_function=SourceSpanSizeFeatures +#feature_function=SourceWordPenalty +#feature_function=SpanFeatures diff --git a/dtrain/test/example/dtrain.ini b/dtrain/test/example/dtrain.ini new file mode 100644 index 00000000..2ad44688 --- /dev/null +++ b/dtrain/test/example/dtrain.ini @@ -0,0 +1,21 @@ +input=test/example/nc-wmt11.1k.gz # use '-' for STDIN +output=- # a weights file (add .gz for gzip compression) or STDOUT '-' +decoder_config=test/example/cdec.ini # config for cdec +# weights for these features will be printed on each iteration +print_weights=Glue WordPenalty LanguageModel LanguageModel_OOV PhraseModel_0 PhraseModel_1 PhraseModel_2 PhraseModel_3 PhraseModel_4 PhraseModel_5 PhraseModel_6 PassThrough +tmp=/tmp +stop_after=20 # stop epoch after 20 inputs + +# interesting stuff +epochs=3 # run over input 3 times +k=100 # use 100best lists +N=4 # optimize (approx) BLEU4 +scorer=stupid_bleu # use 'stupid' BLEU+1 +learning_rate=0.0001 # learning rate +gamma=0 # use SVM reg +sample_from=kbest # use kbest lists (as opposed to forest) +filter=uniq # only unique entries in kbest (surface form) +pair_sampling=XYX +hi_lo=0.1 # 10 vs 80 vs 10 and 80 vs 10 here +pair_threshold=0 # minimum distance in BLEU (this will still only use pairs with diff > 0) +select_weights=VOID # don't output weights diff --git a/dtrain/test/example/nc-wmt11.1k.gz b/dtrain/test/example/nc-wmt11.1k.gz Binary files differnew file mode 100644 index 00000000..45496cd8 --- /dev/null +++ b/dtrain/test/example/nc-wmt11.1k.gz diff --git a/dtrain/test/example/nc-wmt11.en.srilm.gz b/dtrain/test/example/nc-wmt11.en.srilm.gz Binary files differnew file mode 100644 index 00000000..7ce81057 --- /dev/null +++ b/dtrain/test/example/nc-wmt11.en.srilm.gz |