From c795be4ebdbdc37d3fd9ba63d70f0439407333e4 Mon Sep 17 00:00:00 2001 From: Patrick Simianer Date: Tue, 29 Jul 2014 15:02:22 +0200 Subject: make PassThrough1..N optional --- decoder/decoder.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'decoder/decoder.cc') diff --git a/decoder/decoder.cc b/decoder/decoder.cc index 6783cad0..2c044192 100644 --- a/decoder/decoder.cc +++ b/decoder/decoder.cc @@ -366,6 +366,7 @@ DecoderImpl::DecoderImpl(po::variables_map& conf, int argc, char** argv, istream ("beam_prune3", po::value(), "Optional pass 3") ("add_pass_through_rules,P","Add rules to translate OOV words as themselves") + ("add_extra_pass_through_features,Q",po::value()->default_value(6), "Add PassThrough{1..N} features, capped at N.") ("k_best,k",po::value(),"Extract the k best derivations") ("unique_k_best,r", "Unique k-best translation list") ("aligner,a", "Run as a word/phrase aligner (src & ref required)") -- cgit v1.2.3 From 70f75c7320929e9bd61c43b2664df85bee8dc103 Mon Sep 17 00:00:00 2001 From: Patrick Simianer Date: Sat, 2 Aug 2014 14:11:14 +0200 Subject: fix --- decoder/decoder.cc | 2 +- decoder/scfg_translator.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'decoder/decoder.cc') diff --git a/decoder/decoder.cc b/decoder/decoder.cc index 2c044192..081da8d6 100644 --- a/decoder/decoder.cc +++ b/decoder/decoder.cc @@ -366,7 +366,7 @@ DecoderImpl::DecoderImpl(po::variables_map& conf, int argc, char** argv, istream ("beam_prune3", po::value(), "Optional pass 3") ("add_pass_through_rules,P","Add rules to translate OOV words as themselves") - ("add_extra_pass_through_features,Q",po::value()->default_value(6), "Add PassThrough{1..N} features, capped at N.") + ("add_extra_pass_through_features,Q", po::value()->default_value(6), "Add PassThrough{1..N} features, capped at N.") ("k_best,k",po::value(),"Extract the k best derivations") ("unique_k_best,r", "Unique k-best translation list") ("aligner,a", "Run as a word/phrase aligner (src & ref required)") diff --git a/decoder/scfg_translator.cc b/decoder/scfg_translator.cc index 57ad85e2..0c3e4cf7 100644 --- a/decoder/scfg_translator.cc +++ b/decoder/scfg_translator.cc @@ -91,7 +91,7 @@ struct SCFGTranslatorImpl { SCFGTranslatorImpl(const boost::program_options::variables_map& conf) : max_span_limit(conf["scfg_max_span_limit"].as()), add_pass_through_rules(conf.count("add_pass_through_rules")), - num_pt_features(conf["add_pass_through_features"].as()), + num_pt_features(conf["add_extra_pass_through_features"].as()), goal(conf["goal"].as()), default_nt(conf["scfg_default_nt"].as()), use_ctf_(conf.count("coarse_to_fine_beam_prune")) -- cgit v1.2.3 From e6f2dd6892e277d0a868c22f726c4a83c86da016 Mon Sep 17 00:00:00 2001 From: Patrick Simianer Date: Sat, 2 Aug 2014 14:34:57 +0200 Subject: finally --- decoder/decoder.cc | 2 +- decoder/scfg_translator.cc | 8 +++++--- training/dtrain/examples/toy/cdec.ini | 1 + training/dtrain/examples/toy/dtrain.ini | 2 +- 4 files changed, 8 insertions(+), 5 deletions(-) (limited to 'decoder/decoder.cc') diff --git a/decoder/decoder.cc b/decoder/decoder.cc index 081da8d6..c384c33f 100644 --- a/decoder/decoder.cc +++ b/decoder/decoder.cc @@ -366,7 +366,7 @@ DecoderImpl::DecoderImpl(po::variables_map& conf, int argc, char** argv, istream ("beam_prune3", po::value(), "Optional pass 3") ("add_pass_through_rules,P","Add rules to translate OOV words as themselves") - ("add_extra_pass_through_features,Q", po::value()->default_value(6), "Add PassThrough{1..N} features, capped at N.") + ("add_extra_pass_through_features,Q", po::value()->default_value(0), "Add PassThrough{1..N} features, capped at N.") ("k_best,k",po::value(),"Extract the k best derivations") ("unique_k_best,r", "Unique k-best translation list") ("aligner,a", "Run as a word/phrase aligner (src & ref required)") diff --git a/decoder/scfg_translator.cc b/decoder/scfg_translator.cc index 0c3e4cf7..c3cfcaad 100644 --- a/decoder/scfg_translator.cc +++ b/decoder/scfg_translator.cc @@ -64,19 +64,21 @@ PassThroughGrammar::PassThroughGrammar(const Lattice& input, const string& cat, const int j = alts[k].dist2next + i; const string& src = TD::Convert(alts[k].label); if (ss.count(alts[k].label) == 0) { - TRulePtr pt; if (num_pt_features > 0) { int length = static_cast(log(UTF8StringLen(src)) / log(1.6)) + 1; if (length > num_pt_features) length = num_pt_features; string len_feat = "PassThrough_0=1"; len_feat[12] += length; TRulePtr pt(new TRule("[" + cat + "] ||| " + src + " ||| " + src + " ||| PassThrough=1 " + len_feat)); + pt->a_.push_back(AlignmentPoint(0,0)); + AddRule(pt); + RefineRule(pt, ctf_level); } else { TRulePtr pt(new TRule("[" + cat + "] ||| " + src + " ||| " + src + " ||| PassThrough=1 ")); pt->a_.push_back(AlignmentPoint(0,0)); + AddRule(pt); + RefineRule(pt, ctf_level); } - AddRule(pt); - RefineRule(pt, ctf_level); ss.insert(alts[k].label); } } diff --git a/training/dtrain/examples/toy/cdec.ini b/training/dtrain/examples/toy/cdec.ini index b14f4819..e6c19abe 100644 --- a/training/dtrain/examples/toy/cdec.ini +++ b/training/dtrain/examples/toy/cdec.ini @@ -1,3 +1,4 @@ formalism=scfg add_pass_through_rules=true grammar=grammar.gz +#add_extra_pass_through_features=6 diff --git a/training/dtrain/examples/toy/dtrain.ini b/training/dtrain/examples/toy/dtrain.ini index cd715f26..ef956df7 100644 --- a/training/dtrain/examples/toy/dtrain.ini +++ b/training/dtrain/examples/toy/dtrain.ini @@ -2,7 +2,7 @@ decoder_config=cdec.ini input=src refs=tgt output=- -print_weights=logp shell_rule house_rule small_rule little_rule PassThrough +print_weights=logp shell_rule house_rule small_rule little_rule PassThrough PassThrough_1 PassThrough_2 PassThrough_3 PassThrough_4 PassThrough_5 PassThrough_6 k=4 N=4 epochs=2 -- cgit v1.2.3