From 9cf584746a55fd724de6af39f2fbe7bff1b79643 Mon Sep 17 00:00:00 2001 From: Michael Denkowski Date: Mon, 24 Feb 2014 23:18:47 -0800 Subject: CountExceptLM and CountExceptLex features for online grammar extraction. --- python/cdec/sa/extractor.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'python/cdec/sa/extractor.py') diff --git a/python/cdec/sa/extractor.py b/python/cdec/sa/extractor.py index 5a95ded8..244f84be 100644 --- a/python/cdec/sa/extractor.py +++ b/python/cdec/sa/extractor.py @@ -1,16 +1,17 @@ from itertools import chain import os, sys import cdec.configobj +from cdec.sa._sa import gzip_or_text from cdec.sa.features import EgivenFCoherent, SampleCountF, CountEF,\ MaxLexEgivenF, MaxLexFgivenE, IsSingletonF, IsSingletonFE,\ - IsSupportedOnline + IsSupportedOnline, CountExceptLM, CountExceptLex import cdec.sa # maximum span of a grammar rule in TEST DATA MAX_INITIAL_SIZE = 15 class GrammarExtractor: - def __init__(self, config, online=False, features=None): + def __init__(self, config, online=False, vocab=None, features=None): if isinstance(config, basestring): if not os.path.exists(config): raise IOError('cannot read configuration from {0}'.format(config)) @@ -62,6 +63,10 @@ class GrammarExtractor: extended_features = [] if online: extended_features.append(IsSupportedOnline) + if vocab: + vcb_set = set(line.strip() for line in gzip_or_text(vocab)) + extended_features.append(CountExceptLM(vcb_set)) + extended_features.append(CountExceptLex(tt)) # TODO: use @cdec.sa.features decorator for standard features too # + add a mask to disable features -- cgit v1.2.3