summaryrefslogtreecommitdiff
path: root/python/pkg/cdec
diff options
context:
space:
mode:
authorChris Dyer <cdyer@allegro.clab.cs.cmu.edu>2012-11-14 23:14:34 -0500
committerChris Dyer <cdyer@allegro.clab.cs.cmu.edu>2012-11-14 23:14:34 -0500
commit3a62298f86e51aaaa819d01450ec9f8f47edf247 (patch)
tree83061d17dfd0399148ef808d988412132d7c1aee /python/pkg/cdec
parent7928695272b000de7142b91e05959a8fab6b1d2a (diff)
deal with references
Diffstat (limited to 'python/pkg/cdec')
-rw-r--r--python/pkg/cdec/sa/extract.py8
1 files changed, 7 insertions, 1 deletions
diff --git a/python/pkg/cdec/sa/extract.py b/python/pkg/cdec/sa/extract.py
index 10a81556..b7d2fe6e 100644
--- a/python/pkg/cdec/sa/extract.py
+++ b/python/pkg/cdec/sa/extract.py
@@ -3,6 +3,7 @@ import sys
import os
import argparse
import logging
+import re
import multiprocessing as mp
import signal
import cdec.sa
@@ -27,12 +28,17 @@ def extract(inp):
global extractor, prefix
i, sentence = inp
sentence = sentence[:-1]
+ fields = re.split('\s*\|\|\|\s*', sentence)
+ suffix = ''
+ if len(fields) > 1:
+ sentence = fields[0]
+ suffix = ' ||| ' + ' ||| '.join(fields[1:])
grammar_file = os.path.join(prefix, 'grammar.{0}'.format(i))
with open(grammar_file, 'w') as output:
for rule in extractor.grammar(sentence):
output.write(str(rule)+'\n')
grammar_file = os.path.abspath(grammar_file)
- return '<seg grammar="{0}" id="{1}">{2}</seg>'.format(grammar_file, i, sentence)
+ return '<seg grammar="{0}" id="{1}"> {2} </seg>{3}'.format(grammar_file, i, sentence, suffix)
def main():
logging.basicConfig(level=logging.INFO)