summaryrefslogtreecommitdiff
path: root/corpus/xml-tok.py
diff options
context:
space:
mode:
Diffstat (limited to 'corpus/xml-tok.py')
-rwxr-xr-xcorpus/xml-tok.py47
1 files changed, 47 insertions, 0 deletions
diff --git a/corpus/xml-tok.py b/corpus/xml-tok.py
new file mode 100755
index 00000000..4357ced6
--- /dev/null
+++ b/corpus/xml-tok.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+import os
+import re
+import subprocess
+import sys
+
+# Tokenize XML files with tokenize-anything.sh
+# in: <seg id="963"> The earnings on its 10-year bonds are 28.45%. </seg>
+# out: <seg id="963"> The earnings on its 10 - year bonds are 28.45 % . </seg>
+
+def escape(s):
+ return s.replace('&', '&amp;').replace('>', '&gt;').replace('<', '&lt;').replace('"', '&quot;').replace('\'', '&apos;')
+
+def unescape(s):
+ return s.replace('&gt;', '>').replace('&lt;', '<').replace('&quot;', '"').replace('&apos;', '\'').replace('&amp;', '&')
+
+def main():
+ tok = subprocess.Popen([os.path.join(os.path.dirname(__file__), 'tokenize-anything.sh'), '-u'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ while True:
+ line = sys.stdin.readline()
+ if not line:
+ break
+ line = line.strip()
+ pieces = []
+ eol = len(line)
+ pos = 0
+ while pos < eol:
+ next = line.find('<', pos)
+ if next == -1:
+ next = eol
+ tok.stdin.write('{}\n'.format(unescape(line[pos:next])))
+ pieces.append(escape(tok.stdout.readline().strip()))
+ if next == eol:
+ break
+ pos = line.find('>', next + 1)
+ if pos == -1:
+ pos = eol
+ else:
+ pos += 1
+ pieces.append(line[next:pos])
+ sys.stdout.write('{}\n'.format(' '.join(pieces).strip()))
+ tok.stdin.close()
+ tok.wait()
+
+if __name__ == '__main__':
+ main()