summaryrefslogtreecommitdiff
path: root/data
diff options
context:
space:
mode:
Diffstat (limited to 'data')
-rw-r--r--data/geoquery/README3
-rw-r--r--data/geoquery/smt-semparse/decode_sentence.py36
-rw-r--r--data/geoquery/smt-semparse/functionalizer.py143
-rw-r--r--data/geoquery/smt-semparse/moses.py173
-rw-r--r--data/geoquery/wasp-1.0/eval.pl (renamed from data/geoquery/wasp/eval.pl)0
-rw-r--r--data/geoquery/wasp-1.0/geoquery.pl (renamed from data/geoquery/wasp/geoquery.pl)0
6 files changed, 1 insertions, 354 deletions
diff --git a/data/geoquery/README b/data/geoquery/README
index 8e147fa..05067ee 100644
--- a/data/geoquery/README
+++ b/data/geoquery/README
@@ -13,6 +13,5 @@ split880.train.ids : 880 train/test split train ids
../stopwords.en : English stopwords file
../weights.init : initial weights
-smt-semparse/ : slightly adapted code for Andreas' smt-semparse
-wasp/ : adapted stuff from wasp-1.0
+wasp-1.0/ : modified stuff from wasp-1.0
diff --git a/data/geoquery/smt-semparse/decode_sentence.py b/data/geoquery/smt-semparse/decode_sentence.py
deleted file mode 100644
index 1914734..0000000
--- a/data/geoquery/smt-semparse/decode_sentence.py
+++ /dev/null
@@ -1,36 +0,0 @@
-import sys
-import os
-import tempfile, shutil
-from src.extractor import Extractor
-from src.smt_semparse_config import SMTSemparseConfig
-from src.moses import Moses
-from src.functionalizer import Functionalizer
-
-#input: English sentence
-if __name__ == '__main__':
- sentence = ''
- if len(sys.argv) == 3:
- experiment_dir = sys.argv[1]
- sentence = sys.argv[2]
- else:
- assert False
-
- # load config
- config = SMTSemparseConfig('/workspace/grounded/smt-semparse-cp/settings.yaml', '/workspace/grounded/smt-semparse-cp/dependencies.yaml')
-
- #stem
- sentence = Extractor(config).preprocess_nl(sentence)
-
- # we need a temp dir!
- temp_dir = tempfile.mkdtemp()
-
- #decode
- moses = Moses(config)
- moses.decode_sentence(experiment_dir, sentence, temp_dir)
-
- #convert to bracket structure
- print Functionalizer(config).run_sentence(experiment_dir, temp_dir)
-
- #delete tmp files
- shutil.rmtree(temp_dir)
-
diff --git a/data/geoquery/smt-semparse/functionalizer.py b/data/geoquery/smt-semparse/functionalizer.py
deleted file mode 100644
index 782b4e5..0000000
--- a/data/geoquery/smt-semparse/functionalizer.py
+++ /dev/null
@@ -1,143 +0,0 @@
-import logging
-import util
-import sys
-
-class Functionalizer:
-
- def __init__(self, config):
- self.config = config
-
- def run(self):
- hyp_file = open('%s/hyp.mrl.nbest' % self.config.experiment_dir)
- fun_file = open('%s/hyp.fun' % self.config.experiment_dir, 'w')
-
- hypsets = []
- hypset = []
- last_eid = 0
- for line in hyp_file:
- parts = line.split('|||')
- eid = int(parts[0])
- if eid != last_eid:
- hypsets.append(hypset)
- hypset = []
- last_eid = eid
- score = parts[2] + ' ||| ' + parts[3].strip()
- hyp = parts[1].strip()
- hypset.append((hyp,score))
- hypsets.append(hypset)
-
- counter = 0
- for hypset in hypsets:
- hypset = list(reversed(hypset))
- while hypset:
- hyp, score = hypset.pop()
- fun = self.functionalize(hyp)
- if fun:
- print >>fun_file, counter, '|||', fun, '|||', score
- break
- counter += 1
-
- def run_sentence(self, experiment_dir, temp_dir):
- hyp_file = open('%s/nbest.tmp' % temp_dir, 'r')
-
- hypsets = []
- hypset = []
- last_eid = 0
- for line in hyp_file:
- parts = line.split('|||')
- eid = int(parts[0])
- if eid != last_eid:
- hypsets.append(hypset)
- hypset = []
- last_eid = eid
- score = parts[2] + ' ||| ' + parts[3].strip()
- hyp = parts[1].strip()
- hypset.append((hyp,score))
- hypsets.append(hypset)
- hyp_file.close()
-
- counter = 0
- for hypset in hypsets:
- hypset = list(reversed(hypset))
- while hypset:
- hyp, score = hypset.pop()
- fun = self.functionalize(hyp)
- if fun:
- return fun
- break
- counter += 1
- return ""
-
- #xc = 0
- def functionalize(self, mrl):
-
- #if '_@0' in mrl and 'cityid@2' in mrl:
- # #print '==='
- # #print mrl
- # self.xc += 1
- # if self.xc > 5:
- # exit()
-
- stack = []
- r = []
- tokens = list(reversed(mrl.split()))
-
- #print tokens
-
- while tokens:
- it = tokens.pop()
- #print it
- if util.ARITY_SEP not in it:
- token = it
- arity = util.ARITY_STR
- logging.warn('unrecognized token: %s', it)
- else:
- token, arity = it.rsplit(util.ARITY_SEP)
- if arity == util.ARITY_STR:
- arity = 0
- arity_str = True
- elif not (arity == util.ARITY_ANY):
- arity = int(arity)
- arity_str = False
-
- if arity == util.ARITY_ANY or arity > 0:
- r.append(token)
- r.append('(')
- stack.append(arity)
- else:
- assert arity == 0
- if arity_str:
- r.append("'%s'" % token.replace('_', ' '))
- else:
- r.append(token)
- #print r
- while stack:
- top = stack.pop()
- if top == util.ARITY_ANY and tokens:
- r.append(',')
- stack.append(util.ARITY_ANY)
- break
- elif top != util.ARITY_ANY and top > 1:
- r.append(',')
- stack.append(top - 1)
- break
- else:
- r.append(')')
-
- if not stack and tokens:
- return None
-
- if stack:
- return None
-
- r = ''.join(r)
-
- # nasty hacks to fix misplaced _
- if '(_' in r:
- return None
- if ',_' in r and not ('cityid' in r):
- return None
- if '_),_)' in r:
- return None
-
- return r
diff --git a/data/geoquery/smt-semparse/moses.py b/data/geoquery/smt-semparse/moses.py
deleted file mode 100644
index 9a159c3..0000000
--- a/data/geoquery/smt-semparse/moses.py
+++ /dev/null
@@ -1,173 +0,0 @@
-import logging
-import os
-import subprocess
-import gzip
-
-from subprocess import Popen, PIPE, STDOUT
-
-class Moses:
-
- def __init__(self, config):
- self.config = config
-
- def run_train(self):
- args = [self.config.moses_train,
- '--root-dir', self.config.experiment_dir,
- '--corpus', '%s/%s' % (self.config.experiment_dir,
- self.config.train_name),
- '--f', self.config.src,
- '--e', self.config.tgt,
- '--lm', '0:3:%s/%s.arpa' % (self.config.experiment_dir, self.config.tgt),
- #'-score-options', "'--OnlyDirect --NoPhraseCount'"
- '--alignment', self.config.symm,
- '-external-bin-dir', self.config.giza]
- if self.config.model == 'hier':
- args += ['-hierarchical', '-glue-grammar']
-
- logging.info(' '.join(args))
-
- log = open('%s/train.log' % self.config.experiment_dir, 'w')
- p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log)
- p.wait()
- log.close()
-
- def run_retrain(self):
- old_train_nl = '%s/%s.nl' % (self.config.experiment_dir,
- self.config.train_name)
- old_train_mrl = '%s/%s.mrl' % (self.config.experiment_dir,
- self.config.train_name)
- moved_train_nl = '%s.notune' % old_train_nl
- moved_train_mrl = '%s.notune' % old_train_mrl
- tune_nl = '%s/tune.nl' % self.config.experiment_dir
- tune_mrl = '%s/tune.mrl' % self.config.experiment_dir
- os.rename(old_train_nl, moved_train_nl)
- os.rename(old_train_mrl, moved_train_mrl)
- with open(old_train_nl, 'w') as rt_train_nl:
- subprocess.call(['cat', moved_train_nl, tune_nl], stdout=rt_train_nl)
- with open(old_train_mrl, 'w') as rt_train_mrl:
- subprocess.call(['cat', moved_train_mrl, tune_mrl], stdout=rt_train_mrl)
-
- os.remove('%s/model/extract.inv.gz' % self.config.experiment_dir)
- os.remove('%s/model/extract.gz' % self.config.experiment_dir)
- if self.config.model == 'hier':
- os.remove('%s/model/rule-table.gz' % self.config.experiment_dir)
- else:
- os.remove('%s/model/phrase-table.gz' % self.config.experiment_dir)
-
- self.run_train()
-
- def parens_ok(self, line):
- mrl_part = line.split(' ||| ')[1]
- tokens = [t[-1] for t in mrl_part.split() if t[-2] == '@']
- tokens.reverse()
- stack = []
- while tokens:
- t = tokens.pop()
- assert t != '*'
- if t == 's':
- t = 0
- t = int(t)
- if t > 0:
- stack.append(t)
- else:
- while stack:
- top = stack.pop()
- if top > 1:
- stack.append(top - 1)
- break
- if tokens and not stack:
- return False
- return True
-
- def filter_phrase_table(self):
- table_name = 'phrase' if self.config.model == 'phrase' else 'rule'
- oldname = '%s/model/%s-table.gz' % (self.config.experiment_dir, table_name)
- newname = '%s/model/%s-table.old.gz' % (self.config.experiment_dir, table_name)
- os.rename(oldname, newname)
-
- with gzip.open(oldname, 'w') as filtered_table_f:
- with gzip.open(newname, 'r') as old_table_f:
- for line in old_table_f:
- if self.parens_ok(line):
- print >>filtered_table_f, line,
-
- def run_tune(self):
- wd = os.getcwd()
- os.chdir(self.config.experiment_dir)
- args = [self.config.moses_tune,
- '%s/tune.%s' % (self.config.experiment_dir, self.config.src),
- '%s/tune.%s' % (self.config.experiment_dir, self.config.tgt)]
- if self.config.model == 'hier':
- args += [self.config.moses_decode_hier]
- else:
- args += [self.config.moses_decode_phrase]
- args += ['%s/model/moses.ini' % self.config.experiment_dir,
- '--mertdir', '%s/bin' % self.config.moses]
- if self.config.model == 'hier':
- args += ['--filtercmd',
- '%s/scripts/training/filter-model-given-input.pl --Hierarchical'\
- % self.config.moses]
-
- log = open('%s/tune.log' % self.config.experiment_dir, 'w')
- p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log)
- p.wait()
- log.close()
- os.chdir(wd)
-
- def run_decode(self):
- if self.config.model == 'phrase':
- args = [self.config.moses_decode_phrase]
- elif self.config.model == 'hier':
- args = [self.config.moses_decode_hier]
- else:
- assert False
-
- if self.config.run == 'test' or self.config.run == 'all':
- args += ['-f', '%s/mert-work/moses.ini' % self.config.experiment_dir]
- else:
- args += ['-f', '%s/model/moses.ini' % self.config.experiment_dir]
- #args += ['-f', '%s/model/moses.ini' % self.config.experiment_dir]
-
- args += ['-drop-unknown',
- '-n-best-list', '%s/hyp.%s.nbest' % (self.config.experiment_dir, self.config.tgt),
- str(self.config.nbest), 'distinct',
- '-threads', '3']
-
- #nullfile = open(os.devnull, 'w')
- infile = open('%s/test.%s' % (self.config.experiment_dir, self.config.src))
- outfile = open('%s/hyp.%s' % (self.config.experiment_dir, self.config.tgt), 'w')
- log = open('%s/decode.log' % self.config.experiment_dir, 'w')
- p = subprocess.Popen(args, stdin=infile, stdout=outfile, stderr=log)
- p.wait()
- infile.close()
- log.close()
- outfile.close()
-
- def decode_sentence(self, experiment_dir, sentence, temp_dir):
- if self.config.model == 'phrase':
- args = [self.config.moses_decode_phrase]
- elif self.config.model == 'hier':
- args = [self.config.moses_decode_hier]
- else:
- assert False
-
- if self.config.run == 'test' or self.config.run == 'all':
- args += ['-f', '%s/mert-work/moses.ini' % experiment_dir]
- else:
- args += ['-f', '%s/model/moses.ini' % experiment_dir]
-
- args += ['-drop-unknown',
- '-n-best-list', '%s/nbest.tmp' % temp_dir,
- str(self.config.nbest), 'distinct',
- '-threads', '1']
-
- infile = open('%s/sent.tmp' % temp_dir, 'w')
- print >>infile, sentence
- infile.close
- infile = open('%s/sent.tmp' % temp_dir, 'r')
- nullfile = open(os.devnull, 'w')
- p = subprocess.Popen(args, stdin=infile, stdout=nullfile, stderr=nullfile)
- p.wait()
- infile.close()
- return
-
diff --git a/data/geoquery/wasp/eval.pl b/data/geoquery/wasp-1.0/eval.pl
index e00a067..e00a067 100644
--- a/data/geoquery/wasp/eval.pl
+++ b/data/geoquery/wasp-1.0/eval.pl
diff --git a/data/geoquery/wasp/geoquery.pl b/data/geoquery/wasp-1.0/geoquery.pl
index 5d5d9bc..5d5d9bc 100644
--- a/data/geoquery/wasp/geoquery.pl
+++ b/data/geoquery/wasp-1.0/geoquery.pl
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-22 04:31:04 +0000