1 files changed, 63 insertions, 13 deletions

diff --git a/extools/extractor_monolingual.cc b/extools/extractor_monolingual.cc
index 5db768e3..e26aa402 100644
--- a/extools/extractor_monolingual.cc
+++ b/extools/extractor_monolingual.cc
@@ -27,6 +27,7 @@ void InitCommandLine(int argc, char** argv, po::variables_map* conf) {
         ("input,i", po::value<string>()->default_value("-"), "Input file")
         ("phrases,p", po::value<string>(), "File contatining phrases of interest")
         ("phrase_context_size,S", po::value<int>()->default_value(2), "Use this many words of context on left and write when writing base phrase contexts")
+        ("combiner_size,c", po::value<size_t>()->default_value(800000), "Number of unique items to store in cache before writing rule counts. Set to 1 to disable cache. Set to 0 for no limit.")
         ("silent", "Write nothing to stderr except errors")
         ("help,h", "Print this help message and exit");
   po::options_description clo("Command line options");
@@ -90,20 +91,62 @@ struct TrieNode
   unordered_map<int, TrieNode*> next;
 };
 
-void WriteContext(const vector<int>& sentence, int start, int end, int ctx_size) 
-{
-  for (int i = start; i < end; ++i)
+struct CountCombiner {
+  CountCombiner(const size_t& csize) : combiner_size(csize) {
+    if (csize == 0) { cerr << "Using unlimited combiner cache.\n"; }
+  }
+  ~CountCombiner() {
+    if (!cache.empty()) WriteAndClearCache();
+  }
+
+  void Count(const vector<WordID>& key,
+             const vector<WordID>& val,
+             const int count_type)
   {
-    if (i != start) cout << " ";
-    cout << sentence[i];
+    if (combiner_size != 1) {
+      cache[key][val] += count_type;
+      if (combiner_size > 1 && cache.size() > combiner_size)
+        WriteAndClearCache();
+    } else {
+      cout << TD::GetString(key) << '\t' << TD::GetString(val) << " ||| C=" << count_type << "\n";
+    }
   }
-  cout << '\t';
+
+ private:
+  void WriteAndClearCache() {
+    for (unordered_map<vector<WordID>, Vec2PhraseCount, boost::hash<vector<WordID> > >::iterator it = cache.begin();
+         it != cache.end(); ++it) {
+      cout << TD::GetString(it->first) << '\t';
+      const Vec2PhraseCount& vals = it->second;
+      bool needdiv = false;
+      for (Vec2PhraseCount::const_iterator vi = vals.begin(); vi != vals.end(); ++vi) {
+        if (needdiv) cout << " ||| "; else needdiv = true;
+        cout << TD::GetString(vi->first) << " ||| C=" << vi->second;
+      }
+      cout << '\n';
+    }
+    cout << flush;
+    cache.clear();
+  }
+
+  const size_t combiner_size;
+  typedef unordered_map<vector<WordID>, int, boost::hash<vector<WordID> > > Vec2PhraseCount;
+  unordered_map<vector<WordID>, Vec2PhraseCount, boost::hash<vector<WordID> > > cache;
+};
+
+void WriteContext(const vector<int>& sentence, int start, int end, int ctx_size, CountCombiner &combiner) 
+{
+  vector<WordID> phrase, context;
+  for (int i = start; i < end; ++i)
+      phrase.push_back(sentence[i]);
+
   for (int i = ctx_size; i > 0; --i)
-    cout << TD::Convert(sentence[start-i]) << " ";
-  cout << " " << TD::Convert(kGAP);
+    context.push_back(sentence[start-i]);
+  context.push_back(kGAP);
   for (int i = 0; i < ctx_size; ++i)
-    cout << " " << TD::Convert(sentence[end+i]);
-  cout << "\n";
+    context.push_back(sentence[end+i]);
+
+  combiner.Count(phrase, context, 1);
 }
 
 inline bool IsWhitespace(char c) { 
@@ -151,6 +194,7 @@ int main(int argc, char** argv)
 
   bool silent = conf.count("silent") > 0;
   const int ctx_size = conf["phrase_context_size"].as<int>();
+  CountCombiner cc(conf["combiner_size"].as<size_t>());
 
   char buf[MAX_LINE_LENGTH];
   TrieNode phrase_trie(0);
@@ -167,6 +211,7 @@ int main(int argc, char** argv)
   while (iin) {
     ++line;
     iin.getline(buf, MAX_LINE_LENGTH);
+    //cout << "line: " << line << " '" << buf << "'" << endl;
     if (buf[0] == 0) continue;
     if (!silent) {
       if (line % 200 == 0) cerr << '.';
@@ -174,22 +219,27 @@ int main(int argc, char** argv)
     }
 
     vector<int> sentence = ReadSentence(buf, ctx_size);
-    vector<TrieNode*> tries(1, &phrase_trie);
+    //cout << "sentence: " << TD::GetString(sentence) << endl;
+    vector<TrieNode*> tries;
     for (int i = ctx_size; i < (int)sentence.size() - ctx_size; ++i)
     {
-      vector<TrieNode*> tries_prime(1, &phrase_trie);
+      //cout << "i: " << i << " token: " << TD::Convert(sentence[i]) << " tries: " << tries.size() << endl;
+      vector<TrieNode*> tries_prime;
+      tries.push_back(&phrase_trie);
       for (vector<TrieNode*>::iterator tit = tries.begin(); tit != tries.end(); ++tit)
       {
         TrieNode* next = (*tit)->follow(sentence[i]);
         if (next != 0)
         {
+          //cout << "\tfollowed edge: " << next->finish << endl;
           if (next->finish)
-            WriteContext(sentence, i - next->length, i, ctx_size);
+            WriteContext(sentence, i + 1 - next->length, i + 1, ctx_size, cc);
           tries_prime.push_back(next);
         }
       }
       swap(tries, tries_prime);
     }
+    //cout << "/sentence" << endl;
   }
   if (!silent) cerr << endl;
   return 0;