too much to say

1 files changed, 244 insertions, 145 deletions

diff --git a/rampion_with_feedback.rb b/rampfion.rb
index ca99272..ce40917 100755
--- a/rampion_with_feedback.rb
+++ b/rampfion.rb
@@ -4,14 +4,31 @@ require 'trollop'
 require 'tempfile'
 require 'open3'
 require 'memcached'
+require 'timeout'
 
 
-SMT_SEMPARSE = 'python /workspace/grounded/smt-semparse-cp/decode_sentence.py /workspace/grounded/smt-semparse-cp/working/full_dataset'
+SMT_SEMPARSE = 'python /workspace/grounded/smt-semparse-cp/decode_sentence.py /workspace/grounded/smt-semparse-cp/working/full_dataset 2>/dev/null'
 EVAL_PL = '/workspace/grounded/wasp-1.0/data/geo-funql/eval/eval.pl'
 CDEC = "/toolbox/cdec-dtrain/bin/cdec"
 
 $cache = Memcached.new("localhost:11211")
 
+# the semantic parser hangs sometimes
+def spawn_with_timeout cmd, t=4, debug=false
+  puts cmd if debug
+  pipe_in, pipe_out = IO.pipe
+  pid = Process.spawn(cmd, :out => pipe_out)
+  begin
+    Timeout.timeout(t) { Process.wait pid }
+  rescue Timeout::Error
+    return ""
+    # accept the zombies
+    #Process.kill('TERM', pid)
+  end
+  pipe_out.close
+  return pipe_in.read
+end
+
 # execute
 def exec natural_language_string, reference_output, no_output=false
   func = nil
@@ -23,8 +40,8 @@ def exec natural_language_string, reference_output, no_output=false
     output = $cache.get key_prefix+"__OUTPUT"
     feedback = $cache.get key_prefix+"__FEEDBACK"
   rescue Memcached::NotFound
-    func   = `#{SMT_SEMPARSE} "#{natural_language_string}"`.strip
-    output = `echo "execute_funql_query(#{func}, X)." | swipl -s #{EVAL_PL} 2>&1  | grep "X ="`.strip.split('X = ')[1].strip
+    func   = spawn_with_timeout("#{SMT_SEMPARSE} \"#{natural_language_string}\"").strip
+    output = spawn_with_timeout("echo \"execute_funql_query(#{func}, X).\" | swipl -s #{EVAL_PL} 2>&1  | grep \"X =\"").strip.split('X = ')[1]
     feedback = output==reference_output
     begin
       $cache.set key_prefix+"__FUNC", func
@@ -249,8 +266,8 @@ class Stats
     without_parse = total-@with_parse
 <<-eos
   [#{@name}]
-         #{@name} with parse #{((@with_parse/total)*100).round 2} adj:#{((@with_parse/(total-without_parse))*100).round 2} abs:#{@with_parse}
-        #{@name} with output #{((@with_output/total)*100).round 2} adj:#{((@with_output/(total-without_parse))*100).round 2} abs:#{@with_output}
+         #{@name} with parse #{((@with_parse/total)*100).round 2}  abs:#{@with_parse}
+        #{@name} with output #{((@with_output/total)*100).round 2} abs:#{@with_output}
 #{@name} with correct output #{((@correct_output/total)*100).round 2} adj:#{((@correct_output/(total-without_parse))*100).round 2} abs:#{@correct_output}
 eos
   end
@@ -264,32 +281,174 @@ def bag_of_words s, stopwords=[]
   s.split.uniq.sort.reject{|v| stopwords.include? v}
 end
 
-def get_hope_fear_standard kbest, feedback
-  hope = nil; fear = nil
+def gethopefear_standard kbest, feedback
+  hope = fear = nil
+  type1 = type2 = false
   if feedback == true
     hope = kbest[0]
+    type1 = true
   else
     hope = hope_and_fear(kbest, 'hope')
+    type2 = true
   end
   fear = hope_and_fear(kbest, 'fear')
-  return hope, fear
+  return hope, fear, false, type1, type2
+end
+
+def gethopefear_fear_no_exec kbest, feedback, gold, max
+  hope = fear = nil
+  type1 = type2 = false
+  if feedback == true
+    hope = kbest[0]
+    type1 = true
+  else
+    hope = hope_and_fear(kbest, 'hope')
+    type2 = true
+  end
+  kbest.sort{|x,y|(y.model+y.score)<=>(x.model+x.score)}.each_with_index { |k,i|
+    break if i==max
+    if !exec(k.s, gold, true)[0]
+       fear = k
+       break
+    end
+  }
+  skip=true if !fear
+  return hope, fear, skip, type1, type2
 end
 
-def get_hope_fear_standard kbest, feedback
-  hope = nil; fear = nil
+def gethopefear_fear_no_exec_skip kbest, feedback, gold
+  hope = fear = nil
+  type1 = type2 = false
   if feedback == true
     hope = kbest[0]
+    type1 = true
   else
     hope = hope_and_fear(kbest, 'hope')
+    type2 = true
   end
   fear = hope_and_fear(kbest, 'fear')
-  return hope, fear
+  skip = exec(fear.s, gold, true)[0]
+  return hope, fear, skip, type1, type2
+end
+
+def gethopefear_fear_no_exec_hope_exec kbest, feedback, gold, max
+  hope = fear = nil; hope_idx = 0
+  type1 = type2 = false
+  sorted_kbest = kbest.sort{|x,y|(y.model+y.score)<=>(x.model+x.score)}
+  if feedback == true
+    hope = kbest[0]
+    type1 = true
+  else
+    sorted_kbest.each_with_index { |k,i|
+      next if i==0
+      break if i==max
+      if exec(k.s, gold, true)[0]
+        hope_idx = i
+        hope = k
+        break
+      end
+    }
+    type2 = true
+  end
+  sorted_kbest.each_with_index { |k,i|
+    break if i>(kbest.size-(hope_idx+1))||i==max
+    if !exec(k.s, gold, true)[0]
+      fear = k
+      break
+    end
+  }
+  skip = true if !hope||!fear
+  return hope, fear, skip, type1, type2
+end
+
+def gethopefear_only_exec kbest, feedback, gold, max, own_reference=nil
+  hope = fear = nil; hope_idx = 0; new_reference = nil
+  type1 = type2 = false
+  if feedback == true
+    hope = kbest[0]
+    new_reference = hope
+    type1 = true
+  elsif own_reference
+    hope = own_reference
+    type1 = true
+  else
+    kbest.each_with_index { |k,i|
+      next if i==0
+      break if i==max
+      if exec(k.s, gold, true)[0]
+        hope_idx = i
+        hope = k
+        break
+      end
+    }
+    type2 = true
+  end
+  kbest.each_with_index { |k,i|
+    next if i==0||i==hope_idx
+    break if i==max
+    if !exec(k.s, gold, true)[0]
+      fear = k
+      break
+    end
+  }
+  skip = true if !hope||!fear
+  return hope, fear, skip, type1, type2, new_reference
+end
+
+def gethopefear_only_exec_simple kbest, feedback, gold, max, own_reference=nil
+  hope = fear = nil; hope_idx = 0; new_reference = nil
+  type1 = type2 = false
+  if feedback == true
+    hope = kbest[0]
+    new_reference = hope
+    type1 = true
+  elsif own_reference
+    hope = own_reference
+    type1 = true
+  else
+    kbest.each_with_index { |k,i|
+      next if i==0
+      break if i==max
+      if exec(k.s, gold, true)[0]
+        hope_idx = i
+        hope = k
+        break
+      end
+    }
+    type2 = true
+  end
+  kbest.each_with_index { |k,i|
+    next if i==0||i==hope_idx
+    break if i==max
+    if !exec(k.s, gold, true)[0]
+      fear = k
+      break
+    end
+  }
+  skip = true if !hope||!fear
+  return hope, fear, skip, type1, type2, new_reference
+end
+
+def gethopefear_rampion kbest, reference
+  hope = fear = nil
+  type1 = type2 = false
+  if kbest[0].s == reference
+    hope = kbest[0]
+    fear = hope_and_fear(kbest, 'fear')
+    type1 = true
+  else
+    hope = hope_and_fear(kbest, 'hope')
+    fear = kbest[0]
+    type2 = true
+  end
+  return hope, fear, false, type1, type2
 end
 
 def main
   opts = Trollop::options do
     # data
-    opt :k, "k", :type => :int, :required => true
+    opt :k, "k", :type => :int, :default => 10000
+    opt :hope_fear_max, "asdf",  :type => :int, :default => 32, :short => '-q'
     opt :input, "'foreign' input", :type => :string, :required => true
     opt :references, "(parseable) references", :type => :string, :required => true
     opt :gold, "gold output", :type => :string, :require => true
@@ -298,43 +457,36 @@ def main
     opt :cdec_ini, "cdec config file", :type => :string, :default => './cdec.ini'
     # output
     opt :debug, "debug output", :type => :bool, :default => false
-    opt :no_update, "don't update weights", :type => :bool, :default => false
     opt :output_weights, "output file for final weights", :type => :string, :required => true
     opt :stop_after, "stop after x examples", :type => :int, :default => -1
-    opt :print_kbests, "print full kbest lists", :type => :bool, :default => false, :short => '-j'
-    # misc parameters
+    opt :print_kbests, "print full kbest lists", :type => :bool, :default => false, :short => '-l'
+    # important parameters
     opt :eta, "learning rate", :type => :float, :default => 0.01
+    opt :iterate, "iteration X epochs", :type => :int, :default => 1, :short => '-j'
+    opt :variant, "standard, rampion, fear_no_exec, fear_no_exec_skip, fear_no_exec_hope_exec, only_exec", :default => 'standard'
+    # misc parameters
     opt :scale_model, "scale model score by this factor", :type => :float, :default => 1.0, :short => '-m'
-    opt :normalize, "normalize weights after each update", :type => :bool, :default => false, :short => '-l'
-    # learning parameters
-    opt :iterate, "iteration X epochs", :type => :int, :default => 1, :short => '-u'
-    opt :real, "'real' rampion updates", :type => :bool, :default => false, :short => '-q'
-    opt :only_exec, "update only when top1 executes!", :default => false, :short => '-d'
-    opt :hope2, "select hope from the first X items in kbest that executes", :type => :int, :default => 0, :short => '-x'
-    opt :hope3, "skip example if hope doesn't execute", :type => :bool, :default => false, :short => '-b'
-    opt :variant, "use top1 as fear if it does not execute", :type => :bool, :default => false
-    opt :fear2, "skip example if fear executes", :type => :bool, :default => false
-    opt :skip_on_no_proper_gold, "skip if the reference didn't produce a proper gold output", :default => false, :short => '-n'
+    opt :normalize, "normalize weights after each update", :type => :bool, :default => false, :short => '-n'
+    opt :skip_on_no_proper_gold, "skip if the reference didn't produce a proper gold output", :default => false, :short => '-x'
+    opt :no_update, "don't update weights", :type => :bool, :default => false, :short => '-y'
   end
-
   # output configuration
   puts "cfg"
   opts.each_pair {|k,v| puts "#{k}=#{v}"}
   puts
-
   # read files
-  input          = File.readlines(opts[:input], :encoding=>'utf-8').map{|i|i.strip}
-  references     = File.readlines(opts[:references], :encoding=>'utf-8').map{|i|[i.strip, nil]}
-  references_own = references.map{|i|false}
-  gold           = File.readlines(opts[:gold], :encoding=>'utf-8').map{|i|i.strip}
-  gold_mrl       = File.readlines(opts[:gold_mrl], :encoding=>'utf-8').map{|i|i.strip}
-  stopwords      = File.readlines('d/stopwords.en', :encoding=>'utf-8').map{|i|i.strip}
-
+  input      = File.readlines(opts[:input], :encoding=>'utf-8').map{|i|i.strip}
+  references = File.readlines(opts[:references], :encoding=>'utf-8').map{|i|i.strip}
+  gold       = File.readlines(opts[:gold], :encoding=>'utf-8').map{|i|i.strip}
+  gold_mrl   = File.readlines(opts[:gold_mrl], :encoding=>'utf-8').map{|i|i.strip}
+  stopwords  = File.readlines('d/stopwords.en', :encoding=>'utf-8').map{|i|i.strip}
+  # only_exec: new refs
+  own_references = nil
+  own_references = references.map{|i|nil} if opts[:variant]== 'only_exec'
   # init weights
   w = NamedSparseVector.new
   w.from_file opts[:init_weights]
   last_wf = ''
-
 # iterate
 opts[:iterate].times { |iter|
   # numerous counters
@@ -354,7 +506,6 @@ opts[:iterate].times { |iter|
   hope_variant      = 0
   hope_real_variant = 0
   kbest_sz          = 0
-
   # for each example
   input.each_with_index { |i,j|
     count += 1
@@ -377,8 +528,8 @@ opts[:iterate].times { |iter|
       puts "NO MT OUTPUT, skipping example\n\n"
       next
     end
-    # no 
-    if gold[j] == '[]' || gold[j] == '[...]'
+    # no  proper gold
+    if gold[j] == '[]' || gold[j] == '[...]' || gold[j] == '[].'
       no_proper_gold_output += 1
       if opts[:skip_on_no_proper_gold]
         puts "NO PROPER GOLD OUTPUT, skipping example\n\n"
@@ -386,7 +537,7 @@ opts[:iterate].times { |iter|
       end
     end
     # score kbest list
-    score_translations kbest, references[j][0]
+    score_translations kbest, references[j]
     # print kbest list
     if opts[:print_kbests]
       puts "<<<KBEST"
@@ -404,136 +555,84 @@ opts[:iterate].times { |iter|
     feedback, func, output = exec kbest[0].s, gold[j]
     top1_stats.update feedback, func, output
     # reference as bag of words
-    ref_words = bag_of_words references[j][0], stopwords
-
-
-
-
-    # hope2
-    hope_idx = nil
-    if opts[:hope2] > 0
-      (1).upto([opts[:hope2]-1, kbest.size-1].min) { |l|
-        f = exec kbest[l].s, gold[j], true
-        if f[0]
-          hope_idx = l
-          next
-        end
-      }
+    ref_words = bag_of_words references[j], stopwords
+    # hope and fear
+    hope = fear = new_reference = nil
+    type1 = type2 = skip = false
+    if    opts[:variant] == 'standard'
+      hope, fear, skip, type1, type2 = gethopefear_standard kbest, feedback
+    elsif opts[:variant] == 'rampion'
+      hope, fear, skip, type1, type2 = gethopefear_rampion kbest, references[j]
+    elsif opts[:variant] == 'fear_no_exec_skip'
+      hope, fear, skip, type1, type2 = gethopefear_fear_no_exec_skip kbest, feedback, gold[j]
+    elsif opts[:variant] == 'fear_no_exec'
+      hope, fear, skip, type1, type2 = gethopefear_fear_no_exec kbest, feedback, gold[j], opts[:hope_fear_max]
+    elsif opts[:variant] == 'fear_no_exec_hope_exec'
+      hope, fear, skip, type1, type2 = gethopefear_fear_no_exec_hope_exec kbest, feedback, gold[j], opts[:hope_fear_max]
+    elsif opts[:variant] == 'only_exec'
+      hope, fear, skip, type1, type2, new_reference = gethopefear_only_exec kbest, feedback, gold[j], opts[:hope_fear_max], own_references[j]
+    else
+      puts "no such hope/fear variant"
+      exit 1
     end
-    hope = nil; fear = nil
-    if opts[:real]
-      if kbest[0].s != references[j][0]
-        hope = hope_and_fear(kbest, 'hope')
-        fear = kbest[0]
-      else
-        hope = kbest[0]
-        fear = hope_and_fear(kbest, 'fear')
-      end
-    elsif feedback==true
-      type1_updates += 1
-      if kbest[0].s == references[j][0]
-        top1_hit +=1
-      else
-        top1_variant += 1
-        if bag_of_words(kbest[0].s,stopwords) != ref_words
-          top1_real_variant += 1
-          if opts[:debug]
-            puts "<<<DEBUG top1 variant"
-            puts kbest[0].s
-            puts bag_of_words(kbest[0].s,stopwords).to_s
-            puts "ref: #{ref_words.to_s}"
-            puts ">>>"
-          end
-        end
-      end
-      if opts[:only_exec]
-        references[j] = [kbest[0].s, kbest[0]]
-        references_own[j] = true
-      end
-      hope = kbest[0]
-    elsif opts[:only_exec]
-      if references_own[j]
-        hope = references[j][1]
-      else
-        puts "CANNOT FIND HOPE BC NO TOP1 DOESN'T EXEC, skipping example\n\n"
-        next
-      end
+    # new reference (only_exec)
+    if new_reference
+      own_references[j] = new_reference
+    end
+    # type1/type2
+    type1_updates+=1 if type1
+    type2_updates+=1 if type2
+    # top1/hope hit
+    if kbest[0].s == references[j]
+      top1_hit += 1
     else
-      type2_updates += 1
-      if opts[:variant]
-        fear = kbest[0]
-      end
-      if opts[:hope2] > 0
-        if hope_idx
-          hope = kbest[hope_idx]
-        else
-          puts "NO GOOD HOPE, skipping example\n\n"
-          next
-        end
-      else
-        hope = hope_and_fear kbest, 'hope'
-        if opts[:hope3]
-          f = exec hope.s, gold[j], true
-          if !f[0]
-            puts "HOPE NO +FEEDBACK, skipping example\n\n"
-            next
-          end
-        end
-      end
-      if hope.s == references[j][0]
-        hope_hit += 1
-      else
-        hope_variant += 1
-        hope_real_variant += 1 if bag_of_words(hope.s,stopwords)!=ref_words
-      end
+      top1_variant += 1
+      top1_real_variant += 1 if bag_of_words(kbest[0].s,stopwords)!=ref_words
     end
-    fear = hope_and_fear(kbest, 'fear') if !fear
-    if opts[:fear2]
-      f = exec fear.s, gold[j], true
-      f = f[0]
-      if f
-        puts "FEAR EXECUTED, skipping example\n\n"
-        next
-      end
+    if hope&&hope.s == references[j]
+      hope_hit += 1
+    elsif hope
+      hope_variant += 1
+      hope_real_variant += 1 if bag_of_words(hope.s,stopwords)!=ref_words
     end
-
-
-
-
     # output info for current example
     puts "---hope"
-    _print hope.rank, hope.s, hope.model, hope.score
-    feedback, func, output =  exec hope.s, gold[j]
-    hope_stats.update feedback, func, output
+    if hope
+      _print hope.rank, hope.s, hope.model, hope.score
+      feedback, func, output =  exec hope.s, gold[j]
+      hope_stats.update feedback, func, output
+    end
     puts "---fear"
-    _print fear.rank, fear.s, fear.model, fear.score
-    feedback, func, output = exec fear.s, gold[j]
-    fear_stats.update  feedback, func, output
+    if fear
+      _print fear.rank, fear.s, fear.model, fear.score
+      feedback, func, output = exec fear.s, gold[j]
+      fear_stats.update  feedback, func, output
+    end
     puts "---reference"
-    _print 'x', references[j][0], 'x', 1.0
-    feedback, func, output = exec references[j][0], gold[j]
+    _print 'x', references[j], 'x', 1.0
+    feedback, func, output = exec references[j], gold[j]
     refs_stats.update feedback, func, output
+    # skip example?
+    if skip||!hope||!fear
+      puts "NO GOOD FEAR/HOPE, skipping example\n\n"
+      next
+    end
     puts
-
     # update
     w = update w, hope, fear, opts[:eta] if !opts[:no_update]
-
     # normalize weight vector to length 1
     w.normalize! if opts[:normalize]
-
     # stopx after x examples
     break if opts[:stop_after]>0 && (j+1)==opts[:stop_after]
   }
-
   # keep weight files for each iteration
   if opts[:iterate] > 1
     FileUtils::cp(last_wf, "#{opts[:output_weights]}.#{iter}")
   else
     FileUtils::cp(last_wf, opts[:output_weights])
   end
-
   # output stats
-  puts "iteration ##{iter}/#{opts[:iterate]}"
+  puts "iteration ##{iter+1}/#{opts[:iterate]}"
   puts "#{count} examples"
   puts "    type1 updates: #{type1_updates}"
   puts "    type2 updates: #{type2_updates}"