diff options
Diffstat (limited to 'python/cdec/sa/extractor.py')
| -rw-r--r-- | python/cdec/sa/extractor.py | 8 |
1 files changed, 4 insertions, 4 deletions
diff --git a/python/cdec/sa/extractor.py b/python/cdec/sa/extractor.py index 244f84be..25313cc3 100644 --- a/python/cdec/sa/extractor.py +++ b/python/cdec/sa/extractor.py @@ -4,14 +4,14 @@ import cdec.configobj from cdec.sa._sa import gzip_or_text from cdec.sa.features import EgivenFCoherent, SampleCountF, CountEF,\ MaxLexEgivenF, MaxLexFgivenE, IsSingletonF, IsSingletonFE,\ - IsSupportedOnline, CountExceptLM, CountExceptLex + IsSupportedOnline, CountExceptLM import cdec.sa # maximum span of a grammar rule in TEST DATA MAX_INITIAL_SIZE = 15 class GrammarExtractor: - def __init__(self, config, online=False, vocab=None, features=None): + def __init__(self, config, online=False, features=None): if isinstance(config, basestring): if not os.path.exists(config): raise IOError('cannot read configuration from {0}'.format(config)) @@ -60,13 +60,13 @@ class GrammarExtractor: tt = cdec.sa.BiLex(from_binary=config['lex_file']) # TODO: clean this up + # Load data and add features for online grammar extraction extended_features = [] if online: extended_features.append(IsSupportedOnline) - if vocab: + vocab_file = config['vocab_file'] vcb_set = set(line.strip() for line in gzip_or_text(vocab)) extended_features.append(CountExceptLM(vcb_set)) - extended_features.append(CountExceptLex(tt)) # TODO: use @cdec.sa.features decorator for standard features too # + add a mask to disable features |