From 3e5590e041563095833753f3fed24373a7461d3e Mon Sep 17 00:00:00 2001
From: Victor Chahuneau <vchahune@cs.cmu.edu>
Date: Fri, 10 Aug 2012 19:12:29 -0400
Subject: [python] Parallel decoding

---
 python/examples/cdec-mt.py | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 python/examples/cdec-mt.py

(limited to 'python/examples')

diff --git a/python/examples/cdec-mt.py b/python/examples/cdec-mt.py
new file mode 100644
index 00000000..9621df80
--- /dev/null
+++ b/python/examples/cdec-mt.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+import sys
+import argparse
+import logging
+import multiprocessing as mp
+import cdec
+
+decoder = None
+def make_decoder(config, weights):
+    global decoder
+    decoder = cdec.Decoder(config)
+    decoder.read_weights(weights)
+
+def translate(sentence):
+    global decoder
+    return decoder.translate(sentence).viterbi()
+
+def main():
+    logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+    parser = argparse.ArgumentParser(description='Run multiple decoders concurrentely')
+    parser.add_argument('-c', '--config', required=True,
+                        help='decoder configuration')
+    parser.add_argument('-w', '--weights', required=True,
+                        help='feature weights')
+    parser.add_argument('-j', '--jobs', type=int, default=mp.cpu_count(),
+                        help='number of decoder instances')
+    parser.add_argument('-s', '--chunksize', type=int, default=10,
+                        help='number of sentences / chunk')
+    args = parser.parse_args()
+
+    with open(args.config) as config:
+        config = config.read()
+    logging.info('Starting %d workers; chunk size: %d', args.jobs, args.chunksize)
+    pool = mp.Pool(args.jobs, make_decoder, (config, args.weights))
+    for output in pool.imap(translate, sys.stdin, args.chunksize):
+        print(output.encode('utf8'))
+    logging.info('Shutting down workers...')
+
+if __name__ == '__main__':
+    main()
-- 
cgit v1.2.3