One extractor, multiple online contexts.

1 files changed, 51 insertions, 71 deletions

diff --git a/python/cdec/sa/rulefactory.pxi b/python/cdec/sa/rulefactory.pxi
index 10bb9737..a96b01e7 100644
--- a/python/cdec/sa/rulefactory.pxi
+++ b/python/cdec/sa/rulefactory.pxi
@@ -36,6 +36,31 @@ OnlineFeatureContext = namedtuple('OnlineFeatureContext',
      'bilex_fe'
     ])
 
+cdef class OnlineStats:
+    cdef public samples_f
+    cdef public phrases_f
+    cdef public phrases_e
+    cdef public phrases_fe
+    cdef public phrases_al
+    cdef public bilex_f
+    cdef public bilex_e
+    cdef public bilex_fe
+
+    def __cinit__(self):
+        # Keep track of everything that can be sampled:
+        self.samples_f = defaultdict(int)
+        
+        # Phrase counts
+        self.phrases_f = defaultdict(int)
+        self.phrases_e = defaultdict(int)
+        self.phrases_fe = defaultdict(lambda: defaultdict(int))
+        self.phrases_al = defaultdict(lambda: defaultdict(tuple))
+
+        # Bilexical counts
+        self.bilex_f = defaultdict(int)
+        self.bilex_e = defaultdict(int)
+        self.bilex_fe = defaultdict(lambda: defaultdict(int))
+
 cdef int PRECOMPUTE = 0
 cdef int MERGE = 1
 cdef int BAEZA_YATES = 2
@@ -276,14 +301,7 @@ cdef class HieroCachingRuleFactory:
     cdef IntList findexes1
 
     cdef bint online
-    cdef samples_f
-    cdef phrases_f
-    cdef phrases_e
-    cdef phrases_fe
-    cdef phrases_al
-    cdef bilex_f
-    cdef bilex_e
-    cdef bilex_fe
+    cdef online_stats
 
     def __cinit__(self,
             # compiled alignment object (REQUIRED)
@@ -396,21 +414,8 @@ cdef class HieroCachingRuleFactory:
         
         # True after data is added
         self.online = False
+        self.online_stats = defaultdict(OnlineStats)
         
-        # Keep track of everything that can be sampled:
-        self.samples_f = defaultdict(int)
-        
-        # Phrase counts
-        self.phrases_f = defaultdict(int)
-        self.phrases_e = defaultdict(int)
-        self.phrases_fe = defaultdict(lambda: defaultdict(int))
-        self.phrases_al = defaultdict(lambda: defaultdict(tuple))
-
-        # Bilexical counts
-        self.bilex_f = defaultdict(int)
-        self.bilex_e = defaultdict(int)
-        self.bilex_fe = defaultdict(lambda: defaultdict(int))
-
     def configure(self, SuffixArray fsarray, DataArray edarray,
             Sampler sampler, Scorer scorer):
         '''This gives the RuleFactory access to the Context object.
@@ -970,7 +975,7 @@ cdef class HieroCachingRuleFactory:
                     candidate.append([next_id,curr[1]+jump])
         return sorted(result);
 
-    def input(self, fwords, meta):
+    def input(self, fwords, meta, ctx_name=None):
         '''When this function is called on the RuleFactory,
         it looks up all of the rules that can be used to translate
         the input sentence'''
@@ -1154,7 +1159,7 @@ cdef class HieroCachingRuleFactory:
                                                fwords, self.fda, self.eda,
                                                meta,
                                                # Include online stats.  None if none.
-                                               self.online_ctx_lookup(f, e)))
+                                               self.online_ctx_lookup(f, e, ctx_name)))
                                     # Phrase pair processed
                                     if self.online:
                                         seen_phrases.add((f, e))
@@ -1184,6 +1189,7 @@ cdef class HieroCachingRuleFactory:
         
         # Online rule extraction and scoring
         if self.online:
+            stats = self.online_stats[ctx_name]
             f_syms = tuple(word[0][0] for word in fwords)
             for f, lex_i, lex_j in self.get_f_phrases(f_syms):
                 spanlen = (lex_j - lex_i) + 1
@@ -1191,7 +1197,7 @@ cdef class HieroCachingRuleFactory:
                     spanlen += 1
                 if not sym_isvar(f[1]):
                     spanlen += 1
-                for e in self.phrases_fe.get(f, ()):
+                for e in stats.phrases_fe.get(f, ()):
                     if (f, e) not in seen_phrases:
                         # Don't add multiple instances of the same phrase here
                         seen_phrases.add((f, e))
@@ -1200,8 +1206,8 @@ cdef class HieroCachingRuleFactory:
                                 spanlen, None, None, 
                                 fwords, self.fda, self.eda,
                                 meta,
-                                self.online_ctx_lookup(f, e)))
-                        alignment = self.phrases_al[f][e]
+                                self.online_ctx_lookup(f, e, ctx_name)))
+                        alignment = stats.phrases_al[f][e]
                         yield Rule(self.category, f, e, scores, alignment)
              
         stop_time = monitor_cpu()
@@ -1882,7 +1888,7 @@ cdef class HieroCachingRuleFactory:
     
     # Aggregate stats from a training instance
     # (Extract rules, update counts)
-    def add_instance(self, f_words, e_words, alignment):
+    def add_instance(self, f_words, e_words, alignment, ctx_name=None):
 
         self.online = True
         
@@ -2028,28 +2034,30 @@ cdef class HieroCachingRuleFactory:
                 continue
             extract(f_i, f_i, f_len + 1, -1, f_i, 0, [], [], False)
         
+        stats = self.online_stats[ctx_name]
+
         # Update possible phrases (samples)
         # This could be more efficiently integrated with extraction
         # at the cost of readability
         for f, lex_i, lex_j in self.get_f_phrases(f_words):
-            self.samples_f[f] += 1
+            stats.samples_f[f] += 1
             
         # Update phrase counts
         for rule in rules:
             (f_ph, e_ph, al) = rule[:3]
-            self.phrases_f[f_ph] += 1
-            self.phrases_e[e_ph] += 1
-            self.phrases_fe[f_ph][e_ph] += 1
-            if not self.phrases_al[f_ph][e_ph]:
-                self.phrases_al[f_ph][e_ph] = al
+            stats.phrases_f[f_ph] += 1
+            stats.phrases_e[e_ph] += 1
+            stats.phrases_fe[f_ph][e_ph] += 1
+            if not stats.phrases_al[f_ph][e_ph]:
+                stats.phrases_al[f_ph][e_ph] = al
             
         # Update Bilexical counts
         for e_w in e_words:
-            self.bilex_e[e_w] += 1
+            stats.bilex_e[e_w] += 1
         for f_w in f_words:
-            self.bilex_f[f_w] += 1
+            stats.bilex_f[f_w] += 1
             for e_w in e_words:
-                self.bilex_fe[f_w][e_w] += 1
+                stats.bilex_fe[f_w][e_w] += 1
 
     # Create a rule from source, target, non-terminals, and alignments
     def form_rule(self, f_i, e_i, f_span, e_span, nt, al):
@@ -2121,44 +2129,16 @@ cdef class HieroCachingRuleFactory:
         a_str = ' '.join('{0}-{1}'.format(*self.alignment.unlink(packed)) for packed in a)
         return '[X] ||| {0} ||| {1} ||| {2}'.format(f, e, a_str)
     
-    # Debugging
-    def dump_online_stats(self):
-        logger.info('------------------------------')
-        logger.info('         Online Stats         ')
-        logger.info('------------------------------')
-        logger.info('f')
-        for w in self.bilex_f:
-            logger.info(sym_tostring(w) + ' : ' + str(self.bilex_f[w]))
-        logger.info('e')
-        for w in self.bilex_e:
-            logger.info(sym_tostring(w) + ' : ' + str(self.bilex_e[w]))
-        logger.info('fe')
-        for w in self.bilex_fe:
-            for w2 in self.bilex_fe[w]:
-                logger.info(sym_tostring(w) + ' : ' + sym_tostring(w2) + ' : ' + str(self.bilex_fe[w][w2]))
-        logger.info('F')
-        for ph in self.phrases_f:
-            logger.info(str(ph) + ' ||| ' + str(self.phrases_f[ph]))
-        logger.info('E')
-        for ph in self.phrases_e:
-            logger.info(str(ph) + ' ||| ' + str(self.phrases_e[ph]))
-        logger.info('FE')
-        self.dump_online_rules()
-
-    def dump_online_rules(self):
-        for ph in self.phrases_fe:
-            for ph2 in self.phrases_fe[ph]:
-                logger.info(self.fmt_rule(str(ph), str(ph2), self.phrases_al[ph][ph2]) + ' ||| ' + str(self.phrases_fe[ph][ph2]))
-                    
     # Lookup online stats for phrase pair (f, e).  Return None if no match.
     # IMPORTANT: use get() to avoid adding items to defaultdict
-    def online_ctx_lookup(self, f, e):
+    def online_ctx_lookup(self, f, e, ctx_name=None):
         if self.online:
-            fcount = self.phrases_f.get(f, 0)
-            fsample_count = self.samples_f.get(f, 0)
-            d = self.phrases_fe.get(f, None)
+            stats = self.online_stats[ctx_name]
+            fcount = stats.phrases_f.get(f, 0)
+            fsample_count = stats.samples_f.get(f, 0)
+            d = stats.phrases_fe.get(f, None)
             paircount = d.get(e, 0) if d else 0
-            return OnlineFeatureContext(fcount, fsample_count, paircount, self.bilex_f, self.bilex_e, self.bilex_fe)
+            return OnlineFeatureContext(fcount, fsample_count, paircount, stats.bilex_f, stats.bilex_e, stats.bilex_fe)
         return None
     
     # Find all phrases that we might try to extract