From b939bff222736e87fa234c2835511cc29fce644f Mon Sep 17 00:00:00 2001 From: Victor Chahuneau Date: Wed, 5 Sep 2012 19:17:29 +0100 Subject: Revert to the "old style" pair count... + API naming fixes + Multiple feature definition files can be passed to the extractor --- python/pkg/cdec/sa/extract.py | 23 +++++++++++++---------- python/pkg/cdec/sa/extractor.py | 2 ++ 2 files changed, 15 insertions(+), 10 deletions(-) (limited to 'python/pkg/cdec/sa') diff --git a/python/pkg/cdec/sa/extract.py b/python/pkg/cdec/sa/extract.py index 472f128b..3136c5a7 100644 --- a/python/pkg/cdec/sa/extract.py +++ b/python/pkg/cdec/sa/extract.py @@ -11,16 +11,17 @@ extractor, prefix = None, None def make_extractor(config, grammars, features): global extractor, prefix signal.signal(signal.SIGINT, signal.SIG_IGN) # Let parent process catch Ctrl+C - if features: load_features(features) + load_features(features) extractor = cdec.sa.GrammarExtractor(config) prefix = grammars def load_features(features): - logging.info('Loading additional feature definitions from %s', features) - prefix = os.path.dirname(features) - sys.path.append(prefix) - __import__(os.path.basename(features).replace('.py', '')) - sys.path.remove(prefix) + for featdef in features: + logging.info('Loading additional feature definitions from %s', featdef) + prefix = os.path.dirname(featdef) + sys.path.append(prefix) + __import__(os.path.basename(featdef).replace('.py', '')) + sys.path.remove(prefix) def extract(inp): global extractor, prefix @@ -44,15 +45,17 @@ def main(): help='number of parallel extractors') parser.add_argument('-s', '--chunksize', type=int, default=10, help='number of sentences / chunk') - parser.add_argument('-f', '--features', type=str, default=None, + parser.add_argument('-f', '--features', action='append', help='additional feature definitions') args = parser.parse_args() if not os.path.exists(args.grammars): os.mkdir(args.grammars) - if not (args.features is None or args.features.endswith('.py')): - sys.stderr.write('Error: feature definition file should be a python module\n') - sys.exit(1) + for featdef in args.features: + if not featdef.endswith('.py'): + sys.stderr.write('Error: feature definition file <{0}>' + ' should be a python module\n'.format(featdef)) + sys.exit(1) if args.jobs > 1: logging.info('Starting %d workers; chunk size: %d', args.jobs, args.chunksize) diff --git a/python/pkg/cdec/sa/extractor.py b/python/pkg/cdec/sa/extractor.py index 89e35bf8..940544fb 100644 --- a/python/pkg/cdec/sa/extractor.py +++ b/python/pkg/cdec/sa/extractor.py @@ -57,6 +57,8 @@ class GrammarExtractor: # lexical weighting tables tt = cdec.sa.BiLex(from_binary=config['lex_file']) + # TODO: use @cdec.sa.features decorator for standard features too + # + add a mask to disable features scorer = cdec.sa.Scorer(EgivenFCoherent, SampleCountF, CountEF, MaxLexFgivenE(tt), MaxLexEgivenF(tt), IsSingletonF, IsSingletonFE, *cdec.sa._SA_FEATURES) -- cgit v1.2.3 From dba6e87082501a66d8c444ebcd7ed6bd756aaf23 Mon Sep 17 00:00:00 2001 From: Victor Chahuneau Date: Wed, 5 Sep 2012 19:47:07 +0100 Subject: [pycdec.sa] Fix the -f option specification --- python/pkg/cdec/sa/extract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'python/pkg/cdec/sa') diff --git a/python/pkg/cdec/sa/extract.py b/python/pkg/cdec/sa/extract.py index 3136c5a7..10a81556 100644 --- a/python/pkg/cdec/sa/extract.py +++ b/python/pkg/cdec/sa/extract.py @@ -45,7 +45,7 @@ def main(): help='number of parallel extractors') parser.add_argument('-s', '--chunksize', type=int, default=10, help='number of sentences / chunk') - parser.add_argument('-f', '--features', action='append', + parser.add_argument('-f', '--features', nargs='*', default=[], help='additional feature definitions') args = parser.parse_args() -- cgit v1.2.3