diff options
author | Patrick Simianer <p@simianer.de> | 2014-02-28 14:09:55 +0100 |
---|---|---|
committer | Patrick Simianer <p@simianer.de> | 2014-02-28 14:09:55 +0100 |
commit | 739a8cd9a92ee10411e352e1677235a8c39ba8b3 (patch) | |
tree | 6ea4e6994d0a8c9fb9750f25f2694f5388a4dfe8 /python/cdec/sa/extractor.py | |
parent | ab71c44e61d00c788e84b44156d0be16191e267d (diff) | |
parent | 5675965782e2c9201a7a2fe54b542f5b06d660ef (diff) |
Merge remote-tracking branch 'upstream/master'
Diffstat (limited to 'python/cdec/sa/extractor.py')
-rw-r--r-- | python/cdec/sa/extractor.py | 9 |
1 files changed, 7 insertions, 2 deletions
diff --git a/python/cdec/sa/extractor.py b/python/cdec/sa/extractor.py index 5a95ded8..244f84be 100644 --- a/python/cdec/sa/extractor.py +++ b/python/cdec/sa/extractor.py @@ -1,16 +1,17 @@ from itertools import chain import os, sys import cdec.configobj +from cdec.sa._sa import gzip_or_text from cdec.sa.features import EgivenFCoherent, SampleCountF, CountEF,\ MaxLexEgivenF, MaxLexFgivenE, IsSingletonF, IsSingletonFE,\ - IsSupportedOnline + IsSupportedOnline, CountExceptLM, CountExceptLex import cdec.sa # maximum span of a grammar rule in TEST DATA MAX_INITIAL_SIZE = 15 class GrammarExtractor: - def __init__(self, config, online=False, features=None): + def __init__(self, config, online=False, vocab=None, features=None): if isinstance(config, basestring): if not os.path.exists(config): raise IOError('cannot read configuration from {0}'.format(config)) @@ -62,6 +63,10 @@ class GrammarExtractor: extended_features = [] if online: extended_features.append(IsSupportedOnline) + if vocab: + vcb_set = set(line.strip() for line in gzip_or_text(vocab)) + extended_features.append(CountExceptLM(vcb_set)) + extended_features.append(CountExceptLex(tt)) # TODO: use @cdec.sa.features decorator for standard features too # + add a mask to disable features |