summaryrefslogtreecommitdiff
path: root/python/cdec/sa/extractor.py
diff options
context:
space:
mode:
authorMichael Denkowski <mdenkows@cs.cmu.edu>2014-02-24 23:18:47 -0800
committerMichael Denkowski <mdenkows@cs.cmu.edu>2014-02-24 23:18:47 -0800
commit6b0afcb161141cdb64e249adfb3708682febc089 (patch)
tree40c27903869fcd7bb7f2ff0230719b0cc9734593 /python/cdec/sa/extractor.py
parentd843587027d815f3a1c9b8dd5394f3fe04ac85fa (diff)
CountExceptLM and CountExceptLex features for online grammar extraction.
Diffstat (limited to 'python/cdec/sa/extractor.py')
-rw-r--r--python/cdec/sa/extractor.py9
1 files changed, 7 insertions, 2 deletions
diff --git a/python/cdec/sa/extractor.py b/python/cdec/sa/extractor.py
index 5a95ded8..244f84be 100644
--- a/python/cdec/sa/extractor.py
+++ b/python/cdec/sa/extractor.py
@@ -1,16 +1,17 @@
from itertools import chain
import os, sys
import cdec.configobj
+from cdec.sa._sa import gzip_or_text
from cdec.sa.features import EgivenFCoherent, SampleCountF, CountEF,\
MaxLexEgivenF, MaxLexFgivenE, IsSingletonF, IsSingletonFE,\
- IsSupportedOnline
+ IsSupportedOnline, CountExceptLM, CountExceptLex
import cdec.sa
# maximum span of a grammar rule in TEST DATA
MAX_INITIAL_SIZE = 15
class GrammarExtractor:
- def __init__(self, config, online=False, features=None):
+ def __init__(self, config, online=False, vocab=None, features=None):
if isinstance(config, basestring):
if not os.path.exists(config):
raise IOError('cannot read configuration from {0}'.format(config))
@@ -62,6 +63,10 @@ class GrammarExtractor:
extended_features = []
if online:
extended_features.append(IsSupportedOnline)
+ if vocab:
+ vcb_set = set(line.strip() for line in gzip_or_text(vocab))
+ extended_features.append(CountExceptLM(vcb_set))
+ extended_features.append(CountExceptLex(tt))
# TODO: use @cdec.sa.features decorator for standard features too
# + add a mask to disable features