From 6b0afcb161141cdb64e249adfb3708682febc089 Mon Sep 17 00:00:00 2001 From: Michael Denkowski Date: Mon, 24 Feb 2014 23:18:47 -0800 Subject: CountExceptLM and CountExceptLex features for online grammar extraction. --- python/cdec/sa/features.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'python/cdec/sa/features.py') diff --git a/python/cdec/sa/features.py b/python/cdec/sa/features.py index c8fc1cca..fe3fb2bd 100644 --- a/python/cdec/sa/features.py +++ b/python/cdec/sa/features.py @@ -140,3 +140,19 @@ def IsSupportedOnline(ctx): # Occurs in online data? return (ctx.online.paircount > 0.01) else: return False + +def CountExceptLM(vocab): + def CountExceptLM(ctx): # In bitext (inc online data) but NOT mono text + return sum(1 for e in ctx.ephrase.words if e not in vocab) + return CountExceptLM + +def CountExceptLex(ttable): + def CountExceptLex(ctx): + count = 0.0 + for e in ctx.ephrase.words: + if not ttable.contains_e_word(e): + count += 1 + return count + return CountExceptLex + + -- cgit v1.2.3 From 3cb43f4e3980457cbb7b749cee51a5bb777e18f8 Mon Sep 17 00:00:00 2001 From: Michael Denkowski Date: Mon, 24 Feb 2014 23:25:27 -0800 Subject: refactoring --- python/cdec/sa/features.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'python/cdec/sa/features.py') diff --git a/python/cdec/sa/features.py b/python/cdec/sa/features.py index fe3fb2bd..f5cbdb8d 100644 --- a/python/cdec/sa/features.py +++ b/python/cdec/sa/features.py @@ -142,17 +142,11 @@ def IsSupportedOnline(ctx): # Occurs in online data? return False def CountExceptLM(vocab): - def CountExceptLM(ctx): # In bitext (inc online data) but NOT mono text + def CountExceptLM(ctx): # Word count in bitext (inc online data) but NOT mono text return sum(1 for e in ctx.ephrase.words if e not in vocab) return CountExceptLM def CountExceptLex(ttable): - def CountExceptLex(ctx): - count = 0.0 - for e in ctx.ephrase.words: - if not ttable.contains_e_word(e): - count += 1 - return count + def CountExceptLex(ctx): # Word count in online data but NOT original bitext + return sum(1 for e in ctx.ephrase.words if not ttable.contains_e_word(e)) return CountExceptLex - - -- cgit v1.2.3 From 691ea055abe803b430e9dbb3a78ec211722945d3 Mon Sep 17 00:00:00 2001 From: Michael Denkowski Date: Wed, 26 Feb 2014 12:15:02 -0800 Subject: Comments --- python/cdec/sa/features.py | 3 ++- python/cdec/sa/rulefactory.pxi | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'python/cdec/sa/features.py') diff --git a/python/cdec/sa/features.py b/python/cdec/sa/features.py index f5cbdb8d..dcc60401 100644 --- a/python/cdec/sa/features.py +++ b/python/cdec/sa/features.py @@ -147,6 +147,7 @@ def CountExceptLM(vocab): return CountExceptLM def CountExceptLex(ttable): - def CountExceptLex(ctx): # Word count in online data but NOT original bitext + def CountExceptLex(ctx): # Word count in online data but NOT aligned in original bitext + # TODO: Check that online data actually contains aligned word when rulefactory TODO is addressed. return sum(1 for e in ctx.ephrase.words if not ttable.contains_e_word(e)) return CountExceptLex diff --git a/python/cdec/sa/rulefactory.pxi b/python/cdec/sa/rulefactory.pxi index 2be5cad6..78a23196 100644 --- a/python/cdec/sa/rulefactory.pxi +++ b/python/cdec/sa/rulefactory.pxi @@ -2052,6 +2052,7 @@ cdef class HieroCachingRuleFactory: stats.phrases_al[f_ph][e_ph] = al # Update Bilexical counts + # TODO: use alignments instead of cooc for e_w in e_words: stats.bilex_e[e_w] += 1 for f_w in f_words: -- cgit v1.2.3