From 3418c41232756adb9bf29036980e55a3ce0140e0 Mon Sep 17 00:00:00 2001 From: "philblunsom@gmail.com" Date: Tue, 29 Jun 2010 04:28:03 +0000 Subject: Debugged hierarchical backoff model. git-svn-id: https://ws10smt.googlecode.com/svn/trunk@48 ec762483-ff6d-05da-a07a-a48fb63a330f --- gi/pyp-topics/src/contexts_corpus.cc | 69 ++++++++++++++++++++++++++++-------- gi/pyp-topics/src/contexts_corpus.hh | 35 ++++++++++++++++-- gi/pyp-topics/src/corpus.cc | 2 +- gi/pyp-topics/src/corpus.hh | 28 +++++++++++++-- gi/pyp-topics/src/pyp-topics.cc | 2 +- gi/pyp-topics/src/pyp-topics.hh | 2 ++ gi/pyp-topics/src/pyp.hh | 2 +- gi/pyp-topics/src/train.cc | 21 ++++++++++- 8 files changed, 138 insertions(+), 23 deletions(-) (limited to 'gi/pyp-topics') diff --git a/gi/pyp-topics/src/contexts_corpus.cc b/gi/pyp-topics/src/contexts_corpus.cc index 0b3ec644..afa1e19a 100644 --- a/gi/pyp-topics/src/contexts_corpus.cc +++ b/gi/pyp-topics/src/contexts_corpus.cc @@ -15,27 +15,59 @@ using namespace std; void read_callback(const ContextsLexer::PhraseContextsType& new_contexts, void* extra) { assert(new_contexts.contexts.size() == new_contexts.counts.size()); - ContextsCorpus* corpus_ptr = static_cast(extra); + std::pair* extra_pair + = static_cast< std::pair* >(extra); + + ContextsCorpus* corpus_ptr = extra_pair->first; + BackoffGenerator* backoff_gen = extra_pair->second; + Document* doc(new Document()); //std::cout << "READ: " << new_contexts.phrase << "\t"; - for (int i=0; i < new_contexts.contexts.size(); ++i) { - std::string context_str = ""; - for (ContextsLexer::Context::const_iterator it=new_contexts.contexts[i].begin(); - it != new_contexts.contexts[i].end(); ++it) { - //std::cout << *it << " "; - if (it != new_contexts.contexts[i].begin()) - context_str += "__"; - context_str += *it; + int cache_word_count = corpus_ptr->m_dict.max(); + WordID id = corpus_ptr->m_dict.Convert(new_contexts.contexts[i]); + if (cache_word_count != corpus_ptr->m_dict.max()) { + corpus_ptr->m_backoff->terms_at_level(0)++; + corpus_ptr->m_num_types++; } - WordID id = corpus_ptr->m_dict.Convert(context_str); int count = new_contexts.counts[i]; - for (int i=0; ipush_back(id); corpus_ptr->m_num_terms += count; + // generate the backoff map + if (backoff_gen) { + int order = 1; + WordID backoff_id = id; + ContextsLexer::Context backedoff_context = new_contexts.contexts[i]; + while (true) { + if (!corpus_ptr->m_backoff->has_backoff(backoff_id)) { + //std::cerr << "Backing off from " << corpus_ptr->m_dict.Convert(backoff_id) << " to "; + backedoff_context = (*backoff_gen)(backedoff_context); + + if (backedoff_context.empty()) { + //std::cerr << "Nothing." << std::endl; + (*corpus_ptr->m_backoff)[backoff_id] = -1; + break; + } + + if (++order > corpus_ptr->m_backoff->order()) + corpus_ptr->m_backoff->order(order); + + int cache_word_count = corpus_ptr->m_dict.max(); + int new_backoff_id = corpus_ptr->m_dict.Convert(backedoff_context); + if (cache_word_count != corpus_ptr->m_dict.max()) + corpus_ptr->m_backoff->terms_at_level(order-1)++; + + //std::cerr << corpus_ptr->m_dict.Convert(new_backoff_id) << " ." << std::endl; + + backoff_id = ((*corpus_ptr->m_backoff)[backoff_id] = new_backoff_id); + } + else break; + } + } //std::cout << context_str << " (" << id << ") ||| C=" << count << " ||| "; } //std::cout << std::endl; @@ -43,14 +75,23 @@ void read_callback(const ContextsLexer::PhraseContextsType& new_contexts, void* corpus_ptr->m_documents.push_back(doc); } -unsigned ContextsCorpus::read_contexts(const std::string &filename) { +unsigned ContextsCorpus::read_contexts(const std::string &filename, + BackoffGenerator* backoff_gen_ptr) { m_num_terms = 0; m_num_types = 0; igzstream in(filename.c_str()); - ContextsLexer::ReadContexts(&in, read_callback, this); + std::pair extra_pair(this,backoff_gen_ptr); + ContextsLexer::ReadContexts(&in, + read_callback, + &extra_pair); + + //m_num_types = m_dict.max(); - m_num_types = m_dict.max(); + std::cerr << "Read backoff with order " << m_backoff->order() << "\n"; + for (int o=0; oorder(); o++) + std::cerr << " Terms at " << o << " = " << m_backoff->terms_at_level(o) << std::endl; + std::cerr << std::endl; return m_documents.size(); } diff --git a/gi/pyp-topics/src/contexts_corpus.hh b/gi/pyp-topics/src/contexts_corpus.hh index e680cef5..bd0cd34c 100644 --- a/gi/pyp-topics/src/contexts_corpus.hh +++ b/gi/pyp-topics/src/contexts_corpus.hh @@ -11,6 +11,36 @@ #include "contexts_lexer.h" #include "../../../decoder/dict.h" + +class BackoffGenerator { +public: + virtual ContextsLexer::Context + operator()(const ContextsLexer::Context& c) = 0; + +protected: + ContextsLexer::Context strip_edges(const ContextsLexer::Context& c) { + if (c.size() <= 1) return ContextsLexer::Context(); + assert(c.size() % 2 == 1); + return ContextsLexer::Context(c.begin() + 1, c.end() - 1); + } +}; + +class NullBackoffGenerator : public BackoffGenerator { + virtual ContextsLexer::Context + operator()(const ContextsLexer::Context&) + { return ContextsLexer::Context(); } +}; + +class SimpleBackoffGenerator : public BackoffGenerator { + virtual ContextsLexer::Context + operator()(const ContextsLexer::Context& c) { + if (c.size() <= 3) + return ContextsLexer::Context(); + return strip_edges(c); + } +}; + + //////////////////////////////////////////////////////////////// // ContextsCorpus //////////////////////////////////////////////////////////////// @@ -22,10 +52,11 @@ public: typedef boost::ptr_vector::const_iterator const_iterator; public: - ContextsCorpus() {} + ContextsCorpus() : m_backoff(new TermBackoff) {} virtual ~ContextsCorpus() {} - unsigned read_contexts(const std::string &filename); + unsigned read_contexts(const std::string &filename, + BackoffGenerator* backoff_gen=0); TermBackoffPtr backoff_index() { return m_backoff; diff --git a/gi/pyp-topics/src/corpus.cc b/gi/pyp-topics/src/corpus.cc index 24b93a03..f182381f 100644 --- a/gi/pyp-topics/src/corpus.cc +++ b/gi/pyp-topics/src/corpus.cc @@ -11,7 +11,7 @@ using namespace std; // Corpus ////////////////////////////////////////////////// -Corpus::Corpus() {} +Corpus::Corpus() : m_num_terms(0), m_num_types(0) {} unsigned Corpus::read(const std::string &filename) { m_num_terms = 0; diff --git a/gi/pyp-topics/src/corpus.hh b/gi/pyp-topics/src/corpus.hh index 243f7e2c..c2f37130 100644 --- a/gi/pyp-topics/src/corpus.hh +++ b/gi/pyp-topics/src/corpus.hh @@ -22,7 +22,7 @@ public: public: Corpus(); - ~Corpus() {} + virtual ~Corpus() {} unsigned read(const std::string &filename); @@ -71,9 +71,10 @@ class TermBackoff { public: typedef std::vector dictionary_type; typedef dictionary_type::const_iterator const_iterator; + const static int NullBackoff=-1; public: - TermBackoff() : m_backoff_order(-1) {} + TermBackoff() { order(1); } ~TermBackoff() {} void read(const std::string &filename); @@ -86,12 +87,33 @@ public: return m_dict[t]; } + Term& operator[](const Term& t) { + if (t >= static_cast(m_dict.size())) + m_dict.resize(t+1, -1); + return m_dict[t]; + } + + bool has_backoff(const Term& t) { + return t >= 0 && t < static_cast(m_dict.size()) && m_dict[t] >= 0; + } + int order() const { return m_backoff_order; } + void order(int o) { + if (o >= (int)m_terms_at_order.size()) + m_terms_at_order.resize(o, 0); + m_backoff_order = o; + } + // int levels() const { return m_terms_at_order.size(); } bool is_null(const Term& term) const { return term < 0; } int terms_at_level(int level) const { assert (level < (int)m_terms_at_order.size()); - return m_terms_at_order[level]; + return m_terms_at_order.at(level); + } + + int& terms_at_level(int level) { + assert (level < (int)m_terms_at_order.size()); + return m_terms_at_order.at(level); } int size() const { return m_dict.size(); } diff --git a/gi/pyp-topics/src/pyp-topics.cc b/gi/pyp-topics/src/pyp-topics.cc index f3369f2e..c5fd728e 100644 --- a/gi/pyp-topics/src/pyp-topics.cc +++ b/gi/pyp-topics/src/pyp-topics.cc @@ -173,7 +173,7 @@ PYPTopics::F PYPTopics::word_pyps_p0(const Term& term, int topic, int level) con Term backoff_term = (*m_backoff)[term]; if (!m_backoff->is_null(backoff_term)) { assert (level < m_backoff->order()); - p0 = m_backoff->terms_at_level(level)*prob(backoff_term, topic, level+1); + p0 = (1.0/(double)m_backoff->terms_at_level(level))*prob(backoff_term, topic, level+1); } else p0 = m_term_p0; diff --git a/gi/pyp-topics/src/pyp-topics.hh b/gi/pyp-topics/src/pyp-topics.hh index 92d6f292..6b0b15f9 100644 --- a/gi/pyp-topics/src/pyp-topics.hh +++ b/gi/pyp-topics/src/pyp-topics.hh @@ -29,6 +29,8 @@ public: } void set_backoff(TermBackoffPtr backoff) { m_backoff = backoff; + m_word_pyps.clear(); + m_word_pyps.resize(m_backoff->order(), PYPs()); } F prob(const Term& term, int topic, int level=0) const; diff --git a/gi/pyp-topics/src/pyp.hh b/gi/pyp-topics/src/pyp.hh index b06c6021..874d84f5 100644 --- a/gi/pyp-topics/src/pyp.hh +++ b/gi/pyp-topics/src/pyp.hh @@ -121,7 +121,7 @@ private: }; template -PYP::PYP(long double a, long double b, Hash cmp) +PYP::PYP(long double a, long double b, Hash) : std::tr1::unordered_map(), _a(a), _b(b), _a_beta_a(1), _a_beta_b(1), _b_gamma_s(1), _b_gamma_c(1), //_a_beta_a(1), _a_beta_b(1), _b_gamma_s(10), _b_gamma_c(0.1), diff --git a/gi/pyp-topics/src/train.cc b/gi/pyp-topics/src/train.cc index 01ada182..a8fd994c 100644 --- a/gi/pyp-topics/src/train.cc +++ b/gi/pyp-topics/src/train.cc @@ -46,6 +46,7 @@ int main(int argc, char **argv) ("samples,s", value()->default_value(10), "number of sampling passes through the data") ("test-corpus", value(), "file containing the test data") ("backoff-paths", value(), "file containing the term backoff paths") + ("backoff-type", value(), "backoff type: none|simple") ; options_description config_options, cmdline_options; config_options.add(generic); @@ -91,10 +92,27 @@ int main(int argc, char **argv) } else { + BackoffGenerator* backoff_gen=0; + if (vm.count("backoff-type")) { + if (vm["backoff-type"].as() == "none") { + backoff_gen = 0; + } + else if (vm["backoff-type"].as() == "simple") { + backoff_gen = new SimpleBackoffGenerator(); + } + else { + std::cerr << "Backoff type (--backoff-type) must be one of none|simple." << std::endl; + return(1); + } + } + boost::shared_ptr contexts_corpus(new ContextsCorpus); - contexts_corpus->read_contexts(vm["contexts"].as()); + contexts_corpus->read_contexts(vm["contexts"].as(), backoff_gen); corpus = contexts_corpus; model.set_backoff(contexts_corpus->backoff_index()); + + if (backoff_gen) + delete backoff_gen; } // train the sampler @@ -146,6 +164,7 @@ int main(int argc, char **argv) } topics_out.close(); } + std::cout << std::endl; return 0; } -- cgit v1.2.3