summaryrefslogtreecommitdiff
path: root/python/cdec/sa/compile.py
diff options
context:
space:
mode:
authorVictor Chahuneau <vchahune@cs.cmu.edu>2012-07-27 01:16:03 -0400
committerVictor Chahuneau <vchahune@cs.cmu.edu>2012-07-27 01:16:03 -0400
commitb2a8bccb2bd713d9ec081cf3dad0162c2cb492d8 (patch)
treec661044fd2a3943cf2ad12109b916fd7b56a519e /python/cdec/sa/compile.py
parent148b1168c2b07abf0c7757a31141377c28ec3d91 (diff)
[python] Fork of the suffix-array extractor with surface improvements
Available as the cdec.sa module, with commande-line helpers: python -m cdec.sa.compile -f ... -e ... -a ... -o sa-out/ -c extract.ini python -m cdec.sa.extract -c extract.ini -g grammars-out/ < input.txt > input.sgml + renamed cdec.scfg -> cdec.sa + Python README
Diffstat (limited to 'python/cdec/sa/compile.py')
-rw-r--r--python/cdec/sa/compile.py94
1 files changed, 94 insertions, 0 deletions
diff --git a/python/cdec/sa/compile.py b/python/cdec/sa/compile.py
new file mode 100644
index 00000000..061cdab2
--- /dev/null
+++ b/python/cdec/sa/compile.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+import argparse
+import os
+import logging
+import configobj
+import cdec.sa
+
+MAX_PHRASE_LENGTH = 4
+def precompute(f_sa, max_len, max_nt, max_size, min_gap, rank1, rank2):
+ lcp = cdec.sa.LCP(f_sa)
+ stats = sorted(lcp.compute_stats(MAX_PHRASE_LENGTH), reverse=True)
+ precomp = cdec.sa.Precomputation(from_stats=stats,
+ fsarray=f_sa,
+ precompute_rank=rank1,
+ precompute_secondary_rank=rank2,
+ max_length=max_len,
+ max_nonterminals=max_nt,
+ train_max_initial_size=max_size,
+ train_min_gap_size=min_gap)
+ return precomp
+
+def main():
+ logging.basicConfig(level=logging.INFO)
+ logger = logging.getLogger('cdec.sa.compile')
+ parser = argparse.ArgumentParser(description='Compile a corpus into a suffix array.')
+ parser.add_argument('--maxnt', '-n', type=int, default=2,
+ help='Maximum number of non-terminal symbols')
+ parser.add_argument('--maxlen', '-l', type=int, default=5,
+ help='Maximum number of terminals')
+ parser.add_argument('--maxsize', '-s', type=int, default=15,
+ help='Maximum rule span')
+ parser.add_argument('--mingap', '-g', type=int, default=1,
+ help='Minimum gap size')
+ parser.add_argument('--rank1', '-r1', type=int, default=100,
+ help='Number of pre-computed frequent patterns')
+ parser.add_argument('--rank2', '-r2', type=int, default=10,
+ help='Number of pre-computed super-frequent patterns)')
+ parser.add_argument('-c', '--config', default='/dev/stdout',
+ help='Output configuration')
+ parser.add_argument('-o', '--output', required=True,
+ help='Output path')
+ parser.add_argument('-f', '--source', required=True,
+ help='Source language corpus')
+ parser.add_argument('-e', '--target', required=True,
+ help='Target language corpus')
+ parser.add_argument('-a', '--alignment', required=True,
+ help='Bitext word alignment')
+ args = parser.parse_args()
+
+ param_names = ("max_len", "max_nt", "max_size", "min_gap", "rank1", "rank2")
+ params = (args.maxlen, args.maxnt, args.maxsize, args.mingap, args.rank1, args.rank2)
+
+ if not os.path.exists(args.output):
+ os.mkdir(args.output)
+
+ f_sa_bin = os.path.join(args.output, 'f.sa.bin')
+ e_bin = os.path.join(args.output, 'e.bin')
+ precomp_file = 'precomp.{0}.{1}.{2}.{3}.{4}.{5}.bin'.format(*params)
+ precomp_bin = os.path.join(args.output, precomp_file)
+ a_bin = os.path.join(args.output, 'a.bin')
+ lex_bin = os.path.join(args.output, 'lex.bin')
+
+ logger.info('Compiling source suffix array')
+ f_sa = cdec.sa.SuffixArray(from_text=args.source)
+ f_sa.write_binary(f_sa_bin)
+
+ logger.info('Compiling target data array')
+ e = cdec.sa.DataArray(from_text=args.target)
+ e.write_binary(e_bin)
+
+ logger.info('Precomputing frequent phrases')
+ precompute(f_sa, *params).write_binary(precomp_bin)
+
+ logger.info('Compiling alignment')
+ a = cdec.sa.Alignment(from_text=args.alignment)
+ a.write_binary(a_bin)
+
+ logger.info('Compiling bilexical dictionary')
+ lex = cdec.sa.BiLex(from_data=True, alignment=a, earray=e, fsarray=f_sa)
+ lex.write_binary(lex_bin)
+
+ # Write configuration
+ config = configobj.ConfigObj(args.config, unrepr=True)
+ config['f_sa_file'] = f_sa_bin
+ config['e_file'] = e_bin
+ config['a_file'] = a_bin
+ config['lex_file'] = lex_bin
+ config['precompute_file'] = precomp_bin
+ for name, value in zip(param_names, params):
+ config[name] = value
+ config.write()
+
+if __name__ == '__main__':
+ main()