Tool to pull out separate language data from context.txt.gz

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@326 ec762483-ff6d-05da-a07a-a48fb63a330f

1 files changed, 23 insertions, 0 deletions

diff --git a/gi/posterior-regularisation/split-languages.py b/gi/posterior-regularisation/split-languages.py
new file mode 100755
index 00000000..206da661
--- /dev/null
+++ b/gi/posterior-regularisation/split-languages.py
@@ -0,0 +1,23 @@
+#!/usr/bin/python
+
+import sys
+
+sout = open(sys.argv[1], 'w')
+tout = open(sys.argv[2], 'w')
+for line in sys.stdin:
+	phrase, contexts = line.rstrip().split('\t')
+	sp, tp = phrase.split(' <SPLIT> ')
+	sout.write('%s\t' % sp)
+	tout.write('%s\t' % tp)
+	parts = contexts.split(' ||| ')
+	for i in range(0, len(parts), 2):
+		sc, tc = parts[i].split(' <SPLIT> ')
+		if i != 0:
+			sout.write(' ||| ')
+			tout.write(' ||| ')
+		sout.write('%s ||| %s' % (sc, parts[i+1]))
+		tout.write('%s ||| %s' % (tc, parts[i+1]))
+	sout.write('\n')
+	tout.write('\n')
+sout.close()
+tout.close()