From 32a8d92affae91094f2348b73dd26be800e10abd Mon Sep 17 00:00:00 2001 From: Victor Chahuneau Date: Thu, 5 Jul 2012 14:33:14 -0400 Subject: Fix for pow(double, double) + [python] check bounds for candidate set --- dtrain/score.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'dtrain') diff --git a/dtrain/score.cc b/dtrain/score.cc index 4a7cac6e..34fc86a9 100644 --- a/dtrain/score.cc +++ b/dtrain/score.cc @@ -103,7 +103,7 @@ SmoothBleuScorer::Score(vector& hyp, vector& ref, i_bleu[j] += (1/((score_t)j+1)) * i_ng; } } - sum += exp(i_bleu[i])/(pow(2.0, N_-i)); + sum += exp(i_bleu[i])/pow(2.0, (double)(N_-i)); } return brevity_penalty(hyp_len, ref_len) * sum; } @@ -126,7 +126,7 @@ SumBleuScorer::Score(vector& hyp, vector& ref, unsigned j = 1; for (unsigned i = 0; i < M; i++) { if (counts.sum_[i] == 0 || counts.clipped_[i] == 0) break; - sum += ((score_t)counts.clipped_[i]/counts.sum_[i])/pow(2., N_-j+1); + sum += ((score_t)counts.clipped_[i]/counts.sum_[i])/pow(2.0, (double) (N_-j+1)); j++; } return brevity_penalty(hyp_len, ref_len) * sum; @@ -150,7 +150,7 @@ SumExpBleuScorer::Score(vector& hyp, vector& ref, unsigned j = 1; for (unsigned i = 0; i < M; i++) { if (counts.sum_[i] == 0 || counts.clipped_[i] == 0) break; - sum += exp(((score_t)counts.clipped_[i]/counts.sum_[i]))/pow(2., N_-j+1); + sum += exp(((score_t)counts.clipped_[i]/counts.sum_[i]))/pow(2.0, (double) (N_-j+1)); j++; } return brevity_penalty(hyp_len, ref_len) * sum; @@ -178,7 +178,7 @@ SumWhateverBleuScorer::Score(vector& hyp, vector& ref, unsigned j = 1; for (unsigned i = 0; i < M; i++) { if (counts.sum_[i] == 0 || counts.clipped_[i] == 0) break; - sum += exp(v[i] * log(((score_t)counts.clipped_[i]/counts.sum_[i])))/pow(2., N_-j+1); + sum += exp(v[i] * log(((score_t)counts.clipped_[i]/counts.sum_[i])))/pow(2.0, (double) (N_-j+1)); j++; } return brevity_penalty(hyp_len, ref_len) * sum; -- cgit v1.2.3 From 09d085e16f31b36b1177b313ee54414eb29d9082 Mon Sep 17 00:00:00 2001 From: Chris Dyer Date: Fri, 20 Jul 2012 00:46:10 -0400 Subject: enable programmatic setting of grammar objects, cleanup of grammar handling --- decoder/decoder.cc | 10 +++----- decoder/decoder.h | 6 +++-- decoder/scfg_translator.cc | 63 ++++++++++++++++++---------------------------- decoder/translator.h | 4 +-- dtrain/dtrain.cc | 4 +-- python/src/_cdec.pyx | 2 +- 6 files changed, 38 insertions(+), 51 deletions(-) (limited to 'dtrain') diff --git a/decoder/decoder.cc b/decoder/decoder.cc index 333f0fb6..ad4e9e07 100644 --- a/decoder/decoder.cc +++ b/decoder/decoder.cc @@ -743,16 +743,14 @@ bool Decoder::Decode(const string& input, DecoderObserver* o) { } vector& Decoder::CurrentWeightVector() { return pimpl_->CurrentWeightVector(); } const vector& Decoder::CurrentWeightVector() const { return pimpl_->CurrentWeightVector(); } -void Decoder::SetSupplementalGrammar(const std::string& grammar_string) { - assert(pimpl_->translator->GetDecoderType() == "SCFG"); - static_cast(*pimpl_->translator).SetSupplementalGrammar(grammar_string); +void Decoder::AddSupplementalGrammar(GrammarPtr gp) { + static_cast(*pimpl_->translator).AddSupplementalGrammar(gp); } -void Decoder::SetSentenceGrammarFromString(const std::string& grammar_str) { +void Decoder::AddSupplementalGrammarFromString(const std::string& grammar_string) { assert(pimpl_->translator->GetDecoderType() == "SCFG"); - static_cast(*pimpl_->translator).SetSentenceGrammarFromString(grammar_str); + static_cast(*pimpl_->translator).AddSupplementalGrammarFromString(grammar_string); } - bool DecoderImpl::Decode(const string& input, DecoderObserver* o) { string buf = input; NgramCache::Clear(); // clear ngram cache for remote LM (if used) diff --git a/decoder/decoder.h b/decoder/decoder.h index 6b2f7b16..bef2ff5e 100644 --- a/decoder/decoder.h +++ b/decoder/decoder.h @@ -37,6 +37,8 @@ struct DecoderObserver { virtual void NotifyDecodingComplete(const SentenceMetadata& smeta); }; +struct Grammar; // TODO once the decoder interface is cleaned up, + // this should be somewhere else struct Decoder { Decoder(int argc, char** argv); Decoder(std::istream* config_file); @@ -54,8 +56,8 @@ struct Decoder { // add grammar rules (currently only supported by SCFG decoders) // that will be used on subsequent calls to Decode. rules should be in standard // text format. This function does NOT read from a file. - void SetSupplementalGrammar(const std::string& grammar); - void SetSentenceGrammarFromString(const std::string& grammar_str); + void AddSupplementalGrammar(boost::shared_ptr gp); + void AddSupplementalGrammarFromString(const std::string& grammar_string); private: boost::program_options::variables_map conf; boost::shared_ptr pimpl_; diff --git a/decoder/scfg_translator.cc b/decoder/scfg_translator.cc index 185f979a..aaa6c40b 100644 --- a/decoder/scfg_translator.cc +++ b/decoder/scfg_translator.cc @@ -20,7 +20,6 @@ #define reverse_foreach BOOST_REVERSE_FOREACH using namespace std; -static bool usingSentenceGrammar = false; static bool printGrammarsUsed = false; struct SCFGTranslatorImpl { @@ -91,31 +90,31 @@ struct SCFGTranslatorImpl { bool show_tree_structure_; unsigned int ctf_iterations_; vector grammars; - GrammarPtr sup_grammar_; + set sup_grammars_; - struct Equals { Equals(const GrammarPtr& v) : v_(v) {} - bool operator()(const GrammarPtr& x) const { return x == v_; } const GrammarPtr& v_; }; + struct ContainedIn { + ContainedIn(const set& gs) : gs_(gs) {} + bool operator()(const GrammarPtr& x) const { return gs_.find(x) != gs_.end(); } + const set& gs_; + }; - void SetSupplementalGrammar(const std::string& grammar_string) { - grammars.erase(remove_if(grammars.begin(), grammars.end(), Equals(sup_grammar_)), grammars.end()); + void AddSupplementalGrammarFromString(const std::string& grammar_string) { + grammars.erase(remove_if(grammars.begin(), grammars.end(), ContainedIn(sup_grammars_)), grammars.end()); istringstream in(grammar_string); - sup_grammar_.reset(new TextGrammar(&in)); - grammars.push_back(sup_grammar_); + TextGrammar* sent_grammar = new TextGrammar(&in); + sent_grammar->SetMaxSpan(max_span_limit); + sent_grammar->SetGrammarName("SupFromString"); + AddSupplementalGrammar(GrammarPtr(sent_grammar)); } - struct NameEquals { NameEquals(const string name) : name_(name) {} - bool operator()(const GrammarPtr& x) const { return x->GetGrammarName() == name_; } const string name_; }; + void AddSupplementalGrammar(GrammarPtr gp) { + sup_grammars_.insert(gp); + grammars.push_back(gp); + } - void SetSentenceGrammarFromString(const std::string& grammar_str) { - assert(grammar_str != ""); - if (!SILENT) cerr << "Setting sentence grammar" << endl; - usingSentenceGrammar = true; - istringstream in(grammar_str); - TextGrammar* sent_grammar = new TextGrammar(&in); - sent_grammar->SetMaxSpan(max_span_limit); - sent_grammar->SetGrammarName("__psg"); - grammars.erase(remove_if(grammars.begin(), grammars.end(), NameEquals("__psg")), grammars.end()); - grammars.push_back(GrammarPtr(sent_grammar)); + void RemoveSupplementalGrammars() { + grammars.erase(remove_if(grammars.begin(), grammars.end(), ContainedIn(sup_grammars_)), grammars.end()); + sup_grammars_.clear(); } bool Translate(const string& input, @@ -301,34 +300,22 @@ Check for grammar pointer in the sentence markup, for use with sentence specific void SCFGTranslator::ProcessMarkupHintsImpl(const map& kv) { map::const_iterator it = kv.find("grammar"); - - if (it == kv.end()) { - usingSentenceGrammar= false; - return; - } - //Create sentence specific grammar from specified file name and load grammar into list of grammars - usingSentenceGrammar = true; TextGrammar* sentGrammar = new TextGrammar(it->second); sentGrammar->SetMaxSpan(pimpl_->max_span_limit); sentGrammar->SetGrammarName(it->second); - pimpl_->grammars.push_back(GrammarPtr(sentGrammar)); - + pimpl_->AddSupplementalGrammar(GrammarPtr(sentGrammar)); } -void SCFGTranslator::SetSupplementalGrammar(const std::string& grammar) { - pimpl_->SetSupplementalGrammar(grammar); +void SCFGTranslator::AddSupplementalGrammarFromString(const std::string& grammar) { + pimpl_->AddSupplementalGrammarFromString(grammar); } -void SCFGTranslator::SetSentenceGrammarFromString(const std::string& grammar_str) { - pimpl_->SetSentenceGrammarFromString(grammar_str); +void SCFGTranslator::AddSupplementalGrammar(GrammarPtr grammar) { + pimpl_->AddSupplementalGrammar(grammar); } void SCFGTranslator::SentenceCompleteImpl() { - - if(usingSentenceGrammar) // Drop the last sentence grammar from the list of grammars - { - pimpl_->grammars.pop_back(); - } + pimpl_->RemoveSupplementalGrammars(); } std::string SCFGTranslator::GetDecoderType() const { diff --git a/decoder/translator.h b/decoder/translator.h index cfd3b08a..fc2bb760 100644 --- a/decoder/translator.h +++ b/decoder/translator.h @@ -58,8 +58,8 @@ class SCFGTranslatorImpl; class SCFGTranslator : public Translator { public: SCFGTranslator(const boost::program_options::variables_map& conf); - void SetSupplementalGrammar(const std::string& grammar); - void SetSentenceGrammarFromString(const std::string& grammar); + void AddSupplementalGrammar(GrammarPtr gp); + void AddSupplementalGrammarFromString(const std::string& grammar); virtual std::string GetDecoderType() const; protected: bool TranslateImpl(const std::string& src, diff --git a/dtrain/dtrain.cc b/dtrain/dtrain.cc index b3e62914..b7a4bb6f 100644 --- a/dtrain/dtrain.cc +++ b/dtrain/dtrain.cc @@ -364,7 +364,7 @@ main(int argc, char** argv) boost::replace_all(in, "\t", "\n"); in += "\n"; grammar_buf_out << in << DTRAIN_GRAMMAR_DELIM << " " << in_split[0] << endl; - decoder.SetSentenceGrammarFromString(in); + decoder.AddSupplementalGrammarFromString(in); src_str_buf.push_back(in_split[1]); // decode observer->SetRef(ref_ids); @@ -378,7 +378,7 @@ main(int argc, char** argv) if (boost::starts_with(rule, DTRAIN_GRAMMAR_DELIM)) break; grammar_str += rule + "\n"; } - decoder.SetSentenceGrammarFromString(grammar_str); + decoder.AddSupplementalGrammarFromString(grammar_str); // decode observer->SetRef(ref_ids_buf[ii]); decoder.Decode(src_str_buf[ii], observer); diff --git a/python/src/_cdec.pyx b/python/src/_cdec.pyx index 879e8b72..164d6570 100644 --- a/python/src/_cdec.pyx +++ b/python/src/_cdec.pyx @@ -98,7 +98,7 @@ cdef class Decoder: else: raise TypeError('Cannot translate input type %s' % type(sentence)) if grammar: - self.dec.SetSentenceGrammarFromString(string( grammar)) + self.dec.AddSupplementalGrammarFromString(string( grammar)) cdef decoder.BasicObserver observer = decoder.BasicObserver() self.dec.Decode(string(inp), &observer) if observer.hypergraph == NULL: -- cgit v1.2.3