more models

3 files changed, 37 insertions, 6 deletions

diff --git a/gi/pf/align-lexonly.cc b/gi/pf/align-lexonly.cc
index e9f1e7b6..76e2e009 100644
--- a/gi/pf/align-lexonly.cc
+++ b/gi/pf/align-lexonly.cc
@@ -122,10 +122,11 @@ struct BasicLexicalAlignment {
                                  vector<AlignedSentencePair>* corp) :
       letters(lets),
       corpus(*corp),
+      up0("fr-en.10k.translit-base.txt.gz"),
       //up0(words_e),
       //up0("en.chars.1gram", letters_e),
       //up0("en.words.1gram"),
-      up0(letters_e),
+      //up0(letters_e),
       //up0("en.chars.2gram"),
       tmodel(up0) {
   }
@@ -180,14 +181,18 @@ struct BasicLexicalAlignment {
   //PhraseConditionalUninformativeUnigramBase up0;
   //UnigramWordBase up0;
   //HierarchicalUnigramBase up0;
-  HierarchicalWordBase up0;
+  TableLookupBase up0;
+  //HierarchicalWordBase up0;
+  //PoissonUniformUninformativeBase up0;
   //CompletelyUniformBase up0;
   //FixedNgramBase up0;
   //ConditionalTranslationModel<PhraseConditionalUninformativeBase> tmodel;
   //ConditionalTranslationModel<PhraseConditionalUninformativeUnigramBase> tmodel;
   //ConditionalTranslationModel<UnigramWordBase> tmodel;
   //ConditionalTranslationModel<HierarchicalUnigramBase> tmodel;
-  ConditionalTranslationModel<HierarchicalWordBase> tmodel;
+  //ConditionalTranslationModel<HierarchicalWordBase> tmodel;
+  //ConditionalTranslationModel<PoissonUniformUninformativeBase> tmodel;
+  ConditionalTranslationModel<TableLookupBase> tmodel;
   //ConditionalTranslationModel<FixedNgramBase> tmodel;
   //ConditionalTranslationModel<CompletelyUniformBase> tmodel;
 };
@@ -222,6 +227,7 @@ void BasicLexicalAlignment::ResampleCorpus() {
 
 void ExtractLetters(const set<WordID>& v, vector<vector<WordID> >* l, set<WordID>* letset = NULL) {
   for (set<WordID>::const_iterator it = v.begin(); it != v.end(); ++it) {
+    if (*it >= l->size()) { l->resize(*it + 1); }
     vector<WordID>& letters = (*l)[*it];
     if (letters.size()) continue;   // if e and f have the same word
 
@@ -308,7 +314,7 @@ int main(int argc, char** argv) {
   x.InitializeRandom();
   const unsigned samples = conf["samples"].as<unsigned>();
   for (int i = 0; i < samples; ++i) {
-    for (int j = 4995; j < 4997; ++j) Debug(corpus[j]);
+    for (int j = 395; j < 397; ++j) Debug(corpus[j]);
     cerr << i << "\t" << x.tmodel.r.size() << "\t";
     if (i % 10 == 0) x.ResampleHyperparemeters();
     x.ResampleCorpus();
diff --git a/gi/pf/base_measures.cc b/gi/pf/base_measures.cc
index 7894d3e7..4b1863fa 100644
--- a/gi/pf/base_measures.cc
+++ b/gi/pf/base_measures.cc
@@ -37,7 +37,7 @@ TableLookupBase::TableLookupBase(const string& fname) {
       } else if (cc == 1) {
         x.e_.push_back(cur);
       } else if (cc == 2) {
-        table[x] = atof(TD::Convert(cur));
+        table[x].logeq(atof(TD::Convert(cur)));
         ++cc;
       } else {
         if (flag) cerr << endl;
diff --git a/gi/pf/base_measures.h b/gi/pf/base_measures.h
index 7214aa22..b0495bfd 100644
--- a/gi/pf/base_measures.h
+++ b/gi/pf/base_measures.h
@@ -51,6 +51,22 @@ struct Model1 {
   std::vector<std::map<WordID, prob_t> > ttable;
 };
 
+struct PoissonUniformUninformativeBase {
+  explicit PoissonUniformUninformativeBase(const unsigned ves) : kUNIFORM(1.0 / ves) {}
+  prob_t operator()(const TRule& r) const {
+    prob_t p; p.logeq(log_poisson(r.e_.size(), 1.0));
+    prob_t q = kUNIFORM; q.poweq(r.e_.size());
+    p *= q;
+    return p;
+  }
+  void Summary() const {}
+  void ResampleHyperparameters(MT19937*) {}
+  void Increment(const TRule&) {}
+  void Decrement(const TRule&) {}
+  prob_t Likelihood() const { return prob_t::One(); }
+  const prob_t kUNIFORM;
+};
+
 struct CompletelyUniformBase {
   explicit CompletelyUniformBase(const unsigned ves) : kUNIFORM(1.0 / ves) {}
   prob_t operator()(const TRule&) const {
@@ -83,10 +99,19 @@ struct TableLookupBase {
 
   prob_t operator()(const TRule& rule) const {
     const std::tr1::unordered_map<TRule,prob_t>::const_iterator it = table.find(rule);
-    assert(it != table.end());
+    if (it == table.end()) {
+      std::cerr << rule << " not found\n";
+      abort();
+    }
     return it->second;
   }
 
+  void ResampleHyperparameters(MT19937*) {}
+  void Increment(const TRule&) {}
+  void Decrement(const TRule&) {}
+  prob_t Likelihood() const { return prob_t::One(); }
+  void Summary() const {}
+
   std::tr1::unordered_map<TRule,prob_t,RuleHasher> table;
 };