summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xrampion_with_feedback.rb285
1 files changed, 211 insertions, 74 deletions
diff --git a/rampion_with_feedback.rb b/rampion_with_feedback.rb
index a69a922..ca99272 100755
--- a/rampion_with_feedback.rb
+++ b/rampion_with_feedback.rb
@@ -3,21 +3,44 @@
require 'trollop'
require 'tempfile'
require 'open3'
+require 'memcached'
SMT_SEMPARSE = 'python /workspace/grounded/smt-semparse-cp/decode_sentence.py /workspace/grounded/smt-semparse-cp/working/full_dataset'
EVAL_PL = '/workspace/grounded/wasp-1.0/data/geo-funql/eval/eval.pl'
CDEC = "/toolbox/cdec-dtrain/bin/cdec"
+$cache = Memcached.new("localhost:11211")
+
# execute
def exec natural_language_string, reference_output, no_output=false
- func = `#{SMT_SEMPARSE} "#{natural_language_string}"`.strip
- output = `echo "execute_funql_query(#{func}, X)." | swipl -s #{EVAL_PL} 2>&1 | grep "X ="`.strip.split('X = ')[1]
+ func = nil
+ output = nil
+ feedback = nil
+ key_prefix = natural_language_string.encode("ASCII", :invalid => :replace, :undef => :replace, :replace => "?").gsub(/ /,'_')
+ begin
+ func = $cache.get key_prefix+"__FUNC"
+ output = $cache.get key_prefix+"__OUTPUT"
+ feedback = $cache.get key_prefix+"__FEEDBACK"
+ rescue Memcached::NotFound
+ func = `#{SMT_SEMPARSE} "#{natural_language_string}"`.strip
+ output = `echo "execute_funql_query(#{func}, X)." | swipl -s #{EVAL_PL} 2>&1 | grep "X ="`.strip.split('X = ')[1].strip
+ feedback = output==reference_output
+ begin
+ $cache.set key_prefix+"__FUNC", func
+ $cache.set key_prefix+"__OUTPUT", output
+ $cache.set key_prefix+"__FEEDBACK", feedback
+ rescue SystemExit, Interrupt
+ $cache.delete key_prefix+"__FUNC"
+ $cache.delete key_prefix+"__OUTPUT"
+ $cache.delete key_prefix+"__FEEDBACK"
+ end
+ end
puts " nrl: #{natural_language_string}" if !no_output
puts " mrl: #{func}" if !no_output
puts " output: #{output}" if !no_output
- puts " correct?: #{output==reference_output}" if !no_output
- return output==reference_output, func, output
+ puts " correct?: #{feedback}" if !no_output
+ return feedback, func, output
end
# decoder interaction/translations
@@ -217,17 +240,18 @@ class Stats
end
def update feedback, func, output
- @with_parse +=1 if func!="None"
- @with_output +=1 if output!="null"
+ @with_parse +=1 if func!="None"&&func!=''
+ @with_output +=1 if output!="null"&&output!=''
@correct_output += 1 if feedback==true
end
def print total
+ without_parse = total-@with_parse
<<-eos
[#{@name}]
- with parse #{((@with_parse/total)*100).round 2} abs:#{@with_parse}
- with output #{((@with_output/total)*100).round 2} abs:#{@with_output}
-with correct output #{((@correct_output/total)*100).round 2} abs:#{@correct_output}
+ #{@name} with parse #{((@with_parse/total)*100).round 2} adj:#{((@with_parse/(total-without_parse))*100).round 2} abs:#{@with_parse}
+ #{@name} with output #{((@with_output/total)*100).round 2} adj:#{((@with_output/(total-without_parse))*100).round 2} abs:#{@with_output}
+#{@name} with correct output #{((@correct_output/total)*100).round 2} adj:#{((@correct_output/(total-without_parse))*100).round 2} abs:#{@correct_output}
eos
end
end
@@ -240,8 +264,31 @@ def bag_of_words s, stopwords=[]
s.split.uniq.sort.reject{|v| stopwords.include? v}
end
+def get_hope_fear_standard kbest, feedback
+ hope = nil; fear = nil
+ if feedback == true
+ hope = kbest[0]
+ else
+ hope = hope_and_fear(kbest, 'hope')
+ end
+ fear = hope_and_fear(kbest, 'fear')
+ return hope, fear
+end
+
+def get_hope_fear_standard kbest, feedback
+ hope = nil; fear = nil
+ if feedback == true
+ hope = kbest[0]
+ else
+ hope = hope_and_fear(kbest, 'hope')
+ end
+ fear = hope_and_fear(kbest, 'fear')
+ return hope, fear
+end
+
def main
opts = Trollop::options do
+ # data
opt :k, "k", :type => :int, :required => true
opt :input, "'foreign' input", :type => :string, :required => true
opt :references, "(parseable) references", :type => :string, :required => true
@@ -249,48 +296,66 @@ def main
opt :gold_mrl, "gold parse", :type => :string, :short => '-h', :require => true
opt :init_weights, "initial weights", :type => :string, :required => true, :short => '-w'
opt :cdec_ini, "cdec config file", :type => :string, :default => './cdec.ini'
- opt :eta, "learning rate", :type => :float, :default => 0.01
+ # output
+ opt :debug, "debug output", :type => :bool, :default => false
opt :no_update, "don't update weights", :type => :bool, :default => false
- opt :stop_after, "stop after x examples", :type => :int, :default => -1
opt :output_weights, "output file for final weights", :type => :string, :required => true
+ opt :stop_after, "stop after x examples", :type => :int, :default => -1
+ opt :print_kbests, "print full kbest lists", :type => :bool, :default => false, :short => '-j'
+ # misc parameters
+ opt :eta, "learning rate", :type => :float, :default => 0.01
opt :scale_model, "scale model score by this factor", :type => :float, :default => 1.0, :short => '-m'
opt :normalize, "normalize weights after each update", :type => :bool, :default => false, :short => '-l'
- opt :print_kbests, "print full kbest lists", :type => :bool, :default => false, :short => '-j'
+ # learning parameters
+ opt :iterate, "iteration X epochs", :type => :int, :default => 1, :short => '-u'
+ opt :real, "'real' rampion updates", :type => :bool, :default => false, :short => '-q'
+ opt :only_exec, "update only when top1 executes!", :default => false, :short => '-d'
opt :hope2, "select hope from the first X items in kbest that executes", :type => :int, :default => 0, :short => '-x'
+ opt :hope3, "skip example if hope doesn't execute", :type => :bool, :default => false, :short => '-b'
+ opt :variant, "use top1 as fear if it does not execute", :type => :bool, :default => false
opt :fear2, "skip example if fear executes", :type => :bool, :default => false
+ opt :skip_on_no_proper_gold, "skip if the reference didn't produce a proper gold output", :default => false, :short => '-n'
end
+ # output configuration
puts "cfg"
- opts.each_pair {|k,v| puts "#{k}\t#{v}"}
+ opts.each_pair {|k,v| puts "#{k}=#{v}"}
puts
- input = File.new(opts[:input], 'r').readlines.map{|i|i.strip}
- references = File.new(opts[:references], 'r').readlines.map{|i|i.strip}
- gold = File.new(opts[:gold], 'r').readlines.map{|i|i.strip}
- gold_mrl = File.new(opts[:gold_mrl], 'r').readlines.map{|i|i.strip}
-
- stopwords = File.new('stopwords.en', 'r').readlines.map{|i|i.strip}
+ # read files
+ input = File.readlines(opts[:input], :encoding=>'utf-8').map{|i|i.strip}
+ references = File.readlines(opts[:references], :encoding=>'utf-8').map{|i|[i.strip, nil]}
+ references_own = references.map{|i|false}
+ gold = File.readlines(opts[:gold], :encoding=>'utf-8').map{|i|i.strip}
+ gold_mrl = File.readlines(opts[:gold_mrl], :encoding=>'utf-8').map{|i|i.strip}
+ stopwords = File.readlines('d/stopwords.en', :encoding=>'utf-8').map{|i|i.strip}
# init weights
w = NamedSparseVector.new
w.from_file opts[:init_weights]
+ last_wf = ''
- without_translations = 0
- count = 0
+# iterate
+opts[:iterate].times { |iter|
+ # numerous counters
+ without_translations = 0
+ no_proper_gold_output = 0
+ count = 0
top1_stats = Stats.new 'top1'
hope_stats = Stats.new 'hope'
fear_stats = Stats.new 'fear'
refs_stats = Stats.new 'refs'
- type1_updates = 0
- type2_updates = 0
- top1_hit = 0
- top1_variant = 0
+ type1_updates = 0
+ type2_updates = 0
+ top1_hit = 0
+ top1_variant = 0
top1_real_variant = 0
- hope_hit = 0
- hope_variant = 0
+ hope_hit = 0
+ hope_variant = 0
hope_real_variant = 0
- kbest_sz = 0
- last_wf = ''
+ kbest_sz = 0
+
+ # for each example
input.each_with_index { |i,j|
count += 1
# write current weights to file
@@ -302,83 +367,140 @@ def main
# get kbest list for current input
kbest = predict_translation i, opts[:k], opts[:cdec_ini], tmp_file_path
kbest_sz += kbest.size
- if kbest.size==0
+ # output
+ puts "EXAMPLE #{j}"
+ puts "GOLD MRL: #{gold_mrl[j]}"
+ puts "GOLD OUTPUT #{gold[j]}"
+ # skip if no translation could be produced
+ if kbest.size == 0
without_translations += 1
+ puts "NO MT OUTPUT, skipping example\n\n"
next
end
- score_translations kbest, references[j]
+ # no
+ if gold[j] == '[]' || gold[j] == '[...]'
+ no_proper_gold_output += 1
+ if opts[:skip_on_no_proper_gold]
+ puts "NO PROPER GOLD OUTPUT, skipping example\n\n"
+ next
+ end
+ end
+ # score kbest list
+ score_translations kbest, references[j][0]
+ # print kbest list
if opts[:print_kbests]
- puts "KBEST"
+ puts "<<<KBEST"
kbest.each_with_index { |k,l|
_print l, k.s, k.model, k.score
}
+ puts ">>>"
end
+ # adjust model scores to fit in [0,1]
adj_model kbest, opts[:scale_model]
- # get feedback
- puts "EXAMPLE #{j}"
- puts "GOLD MRL: #{gold_mrl[j]}"
- puts "GOLD OUTPUT #{gold[j]}"
- # fear
- fear = hope_and_fear kbest, 'fear'
- if opts[:fear2]
- f, g, o = exec fear.s, gold[j], true
- if f
- puts "FEAR EXECUTED, skipping example\n\n"
- next
- end
- end
# top1
puts "---top1"
+ puts "TOP1 TRANSLATION: #{kbest[0].s}" if iter+1==opts[:iterate]
_print 0, kbest[0].s, kbest[0].model, kbest[0].score
feedback, func, output = exec kbest[0].s, gold[j]
+ top1_stats.update feedback, func, output
+ # reference as bag of words
+ ref_words = bag_of_words references[j][0], stopwords
+
+
+
+
# hope2
- parses = []
- if opts[:hope2]>0
- already_seen = {}
- puts "<<KBEST EXEC"
+ hope_idx = nil
+ if opts[:hope2] > 0
(1).upto([opts[:hope2]-1, kbest.size-1].min) { |l|
- f, g, o = exec kbest[l].s, gold[j], true
- words = bag_of_words kbest[l].s, stopwords
- parses << f
- puts "#{f} | #{l} | #{kbest[l].s} #{words.to_s}" if !already_seen.has_key? words
- already_seen[words] = true
+ f = exec kbest[l].s, gold[j], true
+ if f[0]
+ hope_idx = l
+ next
+ end
}
- puts ">>>"
end
- top1_stats.update feedback, func, output
- # hope & update
- ref_words = bag_of_words references[j], stopwords
- hope = nil
- if feedback==true
- if kbest[0].s == references[j]
+ hope = nil; fear = nil
+ if opts[:real]
+ if kbest[0].s != references[j][0]
+ hope = hope_and_fear(kbest, 'hope')
+ fear = kbest[0]
+ else
+ hope = kbest[0]
+ fear = hope_and_fear(kbest, 'fear')
+ end
+ elsif feedback==true
+ type1_updates += 1
+ if kbest[0].s == references[j][0]
top1_hit +=1
else
top1_variant += 1
- top1_real_variant += 1 if bag_of_words(kbest[0].s,stopwords)!=ref_words
+ if bag_of_words(kbest[0].s,stopwords) != ref_words
+ top1_real_variant += 1
+ if opts[:debug]
+ puts "<<<DEBUG top1 variant"
+ puts kbest[0].s
+ puts bag_of_words(kbest[0].s,stopwords).to_s
+ puts "ref: #{ref_words.to_s}"
+ puts ">>>"
+ end
+ end
+ end
+ if opts[:only_exec]
+ references[j] = [kbest[0].s, kbest[0]]
+ references_own[j] = true
end
- #references[j] = kbest[0].s
hope = kbest[0]
- type1_updates += 1
+ elsif opts[:only_exec]
+ if references_own[j]
+ hope = references[j][1]
+ else
+ puts "CANNOT FIND HOPE BC NO TOP1 DOESN'T EXEC, skipping example\n\n"
+ next
+ end
else
- if opts[:hope2]>0
- c=-1; found = parses.detect{|b| c+=1; b }
- hope = kbest[c] if found
- if !found
+ type2_updates += 1
+ if opts[:variant]
+ fear = kbest[0]
+ end
+ if opts[:hope2] > 0
+ if hope_idx
+ hope = kbest[hope_idx]
+ else
puts "NO GOOD HOPE, skipping example\n\n"
next
end
else
hope = hope_and_fear kbest, 'hope'
+ if opts[:hope3]
+ f = exec hope.s, gold[j], true
+ if !f[0]
+ puts "HOPE NO +FEEDBACK, skipping example\n\n"
+ next
+ end
+ end
end
- if hope.s == references[j]
+ if hope.s == references[j][0]
hope_hit += 1
else
hope_variant += 1
hope_real_variant += 1 if bag_of_words(hope.s,stopwords)!=ref_words
end
- type2_updates += 1
+ end
+ fear = hope_and_fear(kbest, 'fear') if !fear
+ if opts[:fear2]
+ f = exec fear.s, gold[j], true
+ f = f[0]
+ if f
+ puts "FEAR EXECUTED, skipping example\n\n"
+ next
+ end
end
+
+
+
+ # output info for current example
puts "---hope"
_print hope.rank, hope.s, hope.model, hope.score
feedback, func, output = exec hope.s, gold[j]
@@ -388,17 +510,30 @@ def main
feedback, func, output = exec fear.s, gold[j]
fear_stats.update feedback, func, output
puts "---reference"
- _print 'x', references[j], 'x', 1.0
- feedback, func, output = exec references[j], gold[j]
+ _print 'x', references[j][0], 'x', 1.0
+ feedback, func, output = exec references[j][0], gold[j]
refs_stats.update feedback, func, output
-
puts
+ # update
w = update w, hope, fear, opts[:eta] if !opts[:no_update]
+
+ # normalize weight vector to length 1
w.normalize! if opts[:normalize]
+
+ # stopx after x examples
break if opts[:stop_after]>0 && (j+1)==opts[:stop_after]
}
- FileUtils::cp(last_wf, opts[:output_weights])
+
+ # keep weight files for each iteration
+ if opts[:iterate] > 1
+ FileUtils::cp(last_wf, "#{opts[:output_weights]}.#{iter}")
+ else
+ FileUtils::cp(last_wf, opts[:output_weights])
+ end
+
+ # output stats
+ puts "iteration ##{iter}/#{opts[:iterate]}"
puts "#{count} examples"
puts " type1 updates: #{type1_updates}"
puts " type2 updates: #{type2_updates}"
@@ -410,10 +545,12 @@ def main
puts "hope real variant: #{hope_real_variant}"
puts " kbest size: #{(kbest_sz/count).round 2}"
puts "#{((without_translations.to_f/count)*100).round 2}% without translations (abs: #{without_translations})"
+ puts "#{((no_proper_gold_output.to_f/count)*100).round 2}% no good gold output (abs: #{no_proper_gold_output})"
puts top1_stats.print count
puts hope_stats.print count
puts fear_stats.print count
puts refs_stats.print count
+}
end