better debug output, actual semantic parsing

1 files changed, 60 insertions, 17 deletions

diff --git a/rampion_with_feedback.rb b/rampion_with_feedback.rb
index 720d55c..70bf463 100755
--- a/rampion_with_feedback.rb
+++ b/rampion_with_feedback.rb
@@ -1,16 +1,22 @@
 #!/usr/bin/env ruby
 
 require 'trollop'
+require 'tempfile'
 require 'open3'
 
 
 # execute
-SMT_SEMPARSE = '/workspace/grounded/mosesdecoder/moses-chart-cmd/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/moses_chart -f /workspace/grounded/smt-semparse/working/2013-12-10T21.33.38/mert-work/moses.ini 2>/dev/null'
+SMT_SEMPARSE = '/workspace/grounded/mosesdecoder/moses-chart-cmd/bin/gcc-4.7/release/debug-symbols-on/link-static/threading-multi/moses_chart -f /workspace/grounded/smt-semparse/latest/model/moses.ini 2>/dev/null'
 EVAL_PL = '/workspace/grounded/wasp-1.0/data/geo-funql/eval/eval.pl'
 def exec natural_language_string, reference_output
-  res = SMT_SEMPARSE
-  r = `echo "execute_funql_query(#{natural_language_string}, X)." | swipl -s #{EVAL_PL} 2>&1  | grep "X ="`.strip
-  return r==reference_output
+  flat_mrl = `echo "#{natural_language_string}" | ./stem.py | #{SMT_SEMPARSE}`.strip
+  func = `echo "#{flat_mrl}" | ./functionalize.py 2>/dev/null`.strip
+  res = `echo "execute_funql_query(#{func}, X)." | swipl -s #{EVAL_PL} 2>&1  | grep "X ="`.strip.split('X = ')[1]
+  puts "     nrl: #{natural_language_string}"
+  puts "flat mrl: #{flat_mrl}"
+  puts "    func: #{func}"
+  puts "  output: #{res}"
+  return res==reference_output, func, res
 end
 
 
@@ -151,7 +157,7 @@ class NamedSparseVector
   def to_file
     s = []
     @h.each_pair { |k,v| s << "#{k} #{v}" }
-    s.join "\n"
+    s.join("\n")+"\n"
   end
 
   def - other
@@ -236,6 +242,7 @@ def main
     opt :input, "'foreign' input", :type => :string, :required => true
     opt :references, "(parseable) references", :type => :string, :required => true
     opt :gold, "gold standard parser output", :type => :string, :require => true
+    opt :gold_mrl, "gold standard mrl", :type => :string, :short => '-h', :require => true
     opt :init_weights, "initial weights", :type => :string, :required => true, :short => '-w'
     opt :cdec_ini, "cdec config file", :type => :string, :default => './cdec.ini'
   end
@@ -243,38 +250,74 @@ def main
   input = File.new(opts[:input], 'r').readlines.map{|i|i.strip}
   references = File.new(opts[:references], 'r').readlines.map{|i|i.strip}
   gold = File.new(opts[:gold], 'r').readlines.map{|i|i.strip}
+  gold_mrl = File.new(opts[:gold_mrl], 'r').readlines.map{|i|i.strip}
 
   # init weights
   w = NamedSparseVector.new
   w.from_file opts[:init_weights]
 
+
+  positive_feedback = 0
+  without_translations = 0
+  with_proper_parse = 0
+  with_output = 0
+  count = 0
   input.each_with_index { |i,j|
+    count += 1
     # write current weights to file
-    f = File.new('weights.tmp', 'w+')
-    f.write w.to_file
-    f.close
+    tmp_file = Tempfile.new('rampion')
+    tmp_file_path = tmp_file.path
+    tmp_file.write w.to_file
+    tmp_file.close
     # get kbest list for current input
-    kbest = predict_translation i, opts[:k], opts[:cdec_ini], 'weights.tmp'
-    next if kbest.size==0 # FIXME
+    kbest = predict_translation i, opts[:k], opts[:cdec_ini], tmp_file_path
+    if kbest.size==0 # FIXME: shouldnt happen
+      without_translations += 1
+      next
+    end
     score_translations kbest, references[j]
     adj_model kbest
     # get feedback
-    feedback = exec kbest[0].s, gold[j]
+ 
+    puts "----top1"
+    puts "0 #{kbest[0].s} #{kbest[0].model} #{kbest[0].score}"
+    feedback, func, output = exec kbest[0].s, gold[j]
+    with_proper_parse +=1 if func!="None"
+    with_output +=1 if output!="null"
+    positive_feedback  += 1 if feedback==true
     hope = ''; fear = ''
-    if feedback == true
-      references[i] = kbest[0].s
-      hope = kbest[0].s
+    if feedback==true
+      puts "'#{kbest[0].s}'"
+      references[j] = kbest[0].s
+      hope = kbest[0]
     else
       hope = hope_and_fear kbest, 'hope'
     end
     fear = hope_and_fear kbest, 'fear'
     
-    puts "top1: 0 #{kbest[0].s} #{kbest[0].model} #{kbest[0].score}"
-    puts "hope: #{hope.rank} #{hope.s} #{hope.model} #{hope.score}"
-    puts "fear: #{fear.rank} #{fear.s} #{fear.model} #{fear.score}"
+    puts "----hope"
+    puts "#{hope.rank} #{hope.s} #{hope.model} #{hope.score}"
+    exec hope.s, gold[j]
+
+    puts "----fear"
+    puts "#{fear.rank} #{fear.s} #{fear.model} #{fear.score}"
+    exec fear.s, gold[j]
+
+    puts "----reference"
+    puts "// #{references[j]} // 1.0"
+    exec references[j], gold[j]
+    puts "GOLD MRL: #{gold_mrl[j]}"
+    puts "GOLD OUTPUT #{gold[j]}"
+
     puts
+
     w = update w, hope, fear
   }
+  puts "#{count} examples"
+  puts "#{((positive_feedback.to_f/count)*100).round 2}% with positive feedback (abs: #{positive_feedback})"
+  puts "#{((with_proper_parse.to_f/count)*100).round 2}% with proper parse (abs: #{with_proper_parse})"
+  puts "#{((with_output.to_f/count)*100).round 2}% with output (abs: #{with_output})"
+  puts "#{((without_translations.to_f/count)*100).round 2}% without translations (abs: #{without_translations})"
 end