diff options
author | Victor Chahuneau <vchahune@cs.cmu.edu> | 2012-07-27 23:33:45 -0400 |
---|---|---|
committer | Victor Chahuneau <vchahune@cs.cmu.edu> | 2012-07-27 23:33:45 -0400 |
commit | ee5e376e263d9aeabdeee6968b4457f53d3fc772 (patch) | |
tree | 34966090241c5dbd798b324a9f0f806c49aef778 /python/cdec/sa/compile.py | |
parent | 1d481414a2fa8505a2591c88e2b7b8f86a682ca2 (diff) |
[python] Move python files to avoid pythonpath conflicts
Diffstat (limited to 'python/cdec/sa/compile.py')
-rw-r--r-- | python/cdec/sa/compile.py | 94 |
1 files changed, 0 insertions, 94 deletions
diff --git a/python/cdec/sa/compile.py b/python/cdec/sa/compile.py deleted file mode 100644 index 30e605a6..00000000 --- a/python/cdec/sa/compile.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python -import argparse -import os -import logging -import cdec.configobj -import cdec.sa - -MAX_PHRASE_LENGTH = 4 -def precompute(f_sa, max_len, max_nt, max_size, min_gap, rank1, rank2): - lcp = cdec.sa.LCP(f_sa) - stats = sorted(lcp.compute_stats(MAX_PHRASE_LENGTH), reverse=True) - precomp = cdec.sa.Precomputation(from_stats=stats, - fsarray=f_sa, - precompute_rank=rank1, - precompute_secondary_rank=rank2, - max_length=max_len, - max_nonterminals=max_nt, - train_max_initial_size=max_size, - train_min_gap_size=min_gap) - return precomp - -def main(): - logging.basicConfig(level=logging.INFO) - logger = logging.getLogger('cdec.sa.compile') - parser = argparse.ArgumentParser(description='Compile a corpus into a suffix array.') - parser.add_argument('--maxnt', '-n', type=int, default=2, - help='Maximum number of non-terminal symbols') - parser.add_argument('--maxlen', '-l', type=int, default=5, - help='Maximum number of terminals') - parser.add_argument('--maxsize', '-s', type=int, default=15, - help='Maximum rule span') - parser.add_argument('--mingap', '-g', type=int, default=1, - help='Minimum gap size') - parser.add_argument('--rank1', '-r1', type=int, default=100, - help='Number of pre-computed frequent patterns') - parser.add_argument('--rank2', '-r2', type=int, default=10, - help='Number of pre-computed super-frequent patterns)') - parser.add_argument('-c', '--config', default='/dev/stdout', - help='Output configuration') - parser.add_argument('-o', '--output', required=True, - help='Output path') - parser.add_argument('-f', '--source', required=True, - help='Source language corpus') - parser.add_argument('-e', '--target', required=True, - help='Target language corpus') - parser.add_argument('-a', '--alignment', required=True, - help='Bitext word alignment') - args = parser.parse_args() - - param_names = ("max_len", "max_nt", "max_size", "min_gap", "rank1", "rank2") - params = (args.maxlen, args.maxnt, args.maxsize, args.mingap, args.rank1, args.rank2) - - if not os.path.exists(args.output): - os.mkdir(args.output) - - f_sa_bin = os.path.join(args.output, 'f.sa.bin') - e_bin = os.path.join(args.output, 'e.bin') - precomp_file = 'precomp.{0}.{1}.{2}.{3}.{4}.{5}.bin'.format(*params) - precomp_bin = os.path.join(args.output, precomp_file) - a_bin = os.path.join(args.output, 'a.bin') - lex_bin = os.path.join(args.output, 'lex.bin') - - logger.info('Compiling source suffix array') - f_sa = cdec.sa.SuffixArray(from_text=args.source) - f_sa.write_binary(f_sa_bin) - - logger.info('Compiling target data array') - e = cdec.sa.DataArray(from_text=args.target) - e.write_binary(e_bin) - - logger.info('Precomputing frequent phrases') - precompute(f_sa, *params).write_binary(precomp_bin) - - logger.info('Compiling alignment') - a = cdec.sa.Alignment(from_text=args.alignment) - a.write_binary(a_bin) - - logger.info('Compiling bilexical dictionary') - lex = cdec.sa.BiLex(from_data=True, alignment=a, earray=e, fsarray=f_sa) - lex.write_binary(lex_bin) - - # Write configuration - config = cdec.configobj.ConfigObj(args.config, unrepr=True) - config['f_sa_file'] = f_sa_bin - config['e_file'] = e_bin - config['a_file'] = a_bin - config['lex_file'] = lex_bin - config['precompute_file'] = precomp_bin - for name, value in zip(param_names, params): - config[name] = value - config.write() - -if __name__ == '__main__': - main() |