summaryrefslogtreecommitdiff
path: root/python/examples
diff options
context:
space:
mode:
authorVictor Chahuneau <vchahune@cs.cmu.edu>2012-08-10 19:12:29 -0400
committerVictor Chahuneau <vchahune@cs.cmu.edu>2012-08-10 19:12:29 -0400
commit4a0a5980cf81ea0764911845c016bf314f535848 (patch)
treef55db7c99024438859152e388a5673d34cda8f20 /python/examples
parent3f2cc751d1f2655aa0ff14ca735da648899edc40 (diff)
[python] Parallel decoding
Diffstat (limited to 'python/examples')
-rw-r--r--python/examples/cdec-mt.py41
1 files changed, 41 insertions, 0 deletions
diff --git a/python/examples/cdec-mt.py b/python/examples/cdec-mt.py
new file mode 100644
index 00000000..9621df80
--- /dev/null
+++ b/python/examples/cdec-mt.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+import sys
+import argparse
+import logging
+import multiprocessing as mp
+import cdec
+
+decoder = None
+def make_decoder(config, weights):
+ global decoder
+ decoder = cdec.Decoder(config)
+ decoder.read_weights(weights)
+
+def translate(sentence):
+ global decoder
+ return decoder.translate(sentence).viterbi()
+
+def main():
+ logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+ parser = argparse.ArgumentParser(description='Run multiple decoders concurrentely')
+ parser.add_argument('-c', '--config', required=True,
+ help='decoder configuration')
+ parser.add_argument('-w', '--weights', required=True,
+ help='feature weights')
+ parser.add_argument('-j', '--jobs', type=int, default=mp.cpu_count(),
+ help='number of decoder instances')
+ parser.add_argument('-s', '--chunksize', type=int, default=10,
+ help='number of sentences / chunk')
+ args = parser.parse_args()
+
+ with open(args.config) as config:
+ config = config.read()
+ logging.info('Starting %d workers; chunk size: %d', args.jobs, args.chunksize)
+ pool = mp.Pool(args.jobs, make_decoder, (config, args.weights))
+ for output in pool.imap(translate, sys.stdin, args.chunksize):
+ print(output.encode('utf8'))
+ logging.info('Shutting down workers...')
+
+if __name__ == '__main__':
+ main()