[python] Fork of the suffix-array extractor with surface improvements

Available as the cdec.sa module, with commande-line helpers:
python -m cdec.sa.compile -f ... -e ... -a ... -o sa-out/ -c extract.ini
python -m cdec.sa.extract -c extract.ini -g grammars-out/ < input.txt > input.sgml

+ renamed cdec.scfg -> cdec.sa
+ Python README

1 files changed, 32 insertions, 0 deletions

diff --git a/python/cdec/sa/extract.py b/python/cdec/sa/extract.py
new file mode 100644
index 00000000..c6da5e9d
--- /dev/null
+++ b/python/cdec/sa/extract.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+import sys
+import os
+import argparse
+import logging
+import configobj
+import cdec.sa
+
+def main():
+    logging.basicConfig(level=logging.INFO)
+    parser = argparse.ArgumentParser(description='Extract grammars from a compiled corpus.')
+    parser.add_argument('-c', '--config', required=True,
+                        help='Extractor configuration')
+    parser.add_argument('-g', '--grammars', required=True,
+                        help='Grammar output path')
+    args = parser.parse_args()
+
+    if not os.path.exists(args.grammars):
+        os.mkdir(args.grammars)
+
+    extractor = cdec.sa.GrammarExtractor(configobj.ConfigObj(args.config, unrepr=True))
+    for i, sentence in enumerate(sys.stdin):
+        sentence = sentence[:-1]
+        grammar_file = os.path.join(args.grammars, 'grammar.{0}'.format(i))
+        with open(grammar_file, 'w') as output:
+            for rule in extractor.grammar(sentence):
+                output.write(str(rule)+'\n')
+        grammar_file = os.path.abspath(grammar_file)
+        print('<seg grammar="{0}">{1}</seg>'.format(grammar_file, sentence))
+
+if __name__ == '__main__':
+    main()