Merge branch 'master' of https://github.com/redpony/cdec

1 files changed, 11 insertions, 0 deletions

diff --git a/python/cdec/sa/features.py b/python/cdec/sa/features.py
index c8fc1cca..dcc60401 100644
--- a/python/cdec/sa/features.py
+++ b/python/cdec/sa/features.py
@@ -140,3 +140,14 @@ def IsSupportedOnline(ctx): # Occurs in online data?
         return (ctx.online.paircount > 0.01)
     else:
         return False
+
+def CountExceptLM(vocab):
+    def CountExceptLM(ctx): # Word count in bitext (inc online data) but NOT mono text
+        return sum(1 for e in ctx.ephrase.words if e not in vocab)
+    return CountExceptLM
+
+def CountExceptLex(ttable):
+    def CountExceptLex(ctx): # Word count in online data but NOT aligned in original bitext
+        # TODO: Check that online data actually contains aligned word when rulefactory TODO is addressed.
+        return sum(1 for e in ctx.ephrase.words if not ttable.contains_e_word(e))
+    return CountExceptLex