From a49f3a5b19547e7e46a652b22fab601da8fc210f Mon Sep 17 00:00:00 2001 From: Michael Denkowski Date: Thu, 6 Mar 2014 15:35:10 -0800 Subject: Compile count-based bilex table for online grammar extraction. --- python/cdec/sa/extractor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'python/cdec/sa/extractor.py') diff --git a/python/cdec/sa/extractor.py b/python/cdec/sa/extractor.py index 244f84be..25313cc3 100644 --- a/python/cdec/sa/extractor.py +++ b/python/cdec/sa/extractor.py @@ -4,14 +4,14 @@ import cdec.configobj from cdec.sa._sa import gzip_or_text from cdec.sa.features import EgivenFCoherent, SampleCountF, CountEF,\ MaxLexEgivenF, MaxLexFgivenE, IsSingletonF, IsSingletonFE,\ - IsSupportedOnline, CountExceptLM, CountExceptLex + IsSupportedOnline, CountExceptLM import cdec.sa # maximum span of a grammar rule in TEST DATA MAX_INITIAL_SIZE = 15 class GrammarExtractor: - def __init__(self, config, online=False, vocab=None, features=None): + def __init__(self, config, online=False, features=None): if isinstance(config, basestring): if not os.path.exists(config): raise IOError('cannot read configuration from {0}'.format(config)) @@ -60,13 +60,13 @@ class GrammarExtractor: tt = cdec.sa.BiLex(from_binary=config['lex_file']) # TODO: clean this up + # Load data and add features for online grammar extraction extended_features = [] if online: extended_features.append(IsSupportedOnline) - if vocab: + vocab_file = config['vocab_file'] vcb_set = set(line.strip() for line in gzip_or_text(vocab)) extended_features.append(CountExceptLM(vcb_set)) - extended_features.append(CountExceptLex(tt)) # TODO: use @cdec.sa.features decorator for standard features too # + add a mask to disable features -- cgit v1.2.3