merge,cleanup,rename

8 files changed, 59 insertions, 126 deletions

diff --git a/README.md b/README.md
index 9293207..f43f61d 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,8 @@
-lampion
-=======
+rebol
+=====
 
 code for grounded SMT
+
+This has nothing to do with the programming language REBOL
+http://www.rebol.com/
+
diff --git a/example/example.sh b/example/example.sh
index 1c67ed0..b359dfd 100755
--- a/example/example.sh
+++ b/example/example.sh
@@ -2,8 +2,8 @@
 
 # memcached has to be running! `memcached -p 31337`
 
-# run lampion with rampion variant for 1 epoch over 10 examples (data.*)
-../lampion.rb \
+# run rebol with rampion variant for 1 epoch over 10 examples (data.*)
+../rebol.rb \
   -k 100 \
   -i $(pwd)/data.in \
   -r $(pwd)/data.en \
diff --git a/hopefear.rb b/hopefear.rb
index 37782a4..aed0c9c 100644
--- a/hopefear.rb
+++ b/hopefear.rb
@@ -9,114 +9,60 @@ def hope_and_fear kbest, action
       max_idx = i; max = k.scores[:decoder] - k.scores[:psb]
     end
   }
-  return kbest[max_idx]
+  return max_idx
 end
 
-def gethopefear_standard kbest, feedback
-  hope = fear = nil
-  type1 = type2 = false
-  if feedback == true
-    hope = kbest[0]
-    type1 = true
-  else
-    hope = hope_and_fear kbest, 'hope'
-    type2 = true
-  end
-  fear = hope_and_fear kbest, 'fear'
-  return hope, fear, false, type1, type2
-end
-
-def gethopefear_fear_no_exec kbest, feedback, gold, max
-  hope = fear = nil
-  type1 = type2 = false
-  if feedback == true
-    hope = kbest[0]
-    type1 = true
-  else
-    hope = hope_and_fear kbest, 'hope'
-    type2 = true
-  end
-  # sorted in descending order by max(decoder, psb), best ('hope') first
-  # select the 'best' translation that does not deliver the correct answer
-  kbest.sort{ |x,y| (y.scores[:decoder]+y.scores[:psb])<=>(x.scores[:decoder]+x.scores[:psb]) }.each_with_index { |k,i|
-    break if i==max
-    if !exec(k.s, gold, true)[0]
-       fear = k
-       break
-    end
-  }
-  skip=true if !fear
-  return hope, fear, skip, type1, type2
-end
-
-def gethopefear_fear_no_exec_skip kbest, feedback, gold
-  hope = fear = nil
-  type1 = type2 = false
-  if feedback == true
-    hope = kbest[0]
-    type1 = true
-  else
-    hope = hope_and_fear kbest, 'hope'
-    type2 = true
-  end
-  fear = hope_and_fear(kbest, 'fear')
-  # skip example if fear gives the right answer
-  skip = exec(fear.s, gold, true)[0]
-  return hope, fear, skip, type1, type2
-end
-
-def gethopefear_fear_no_exec_hope_exec kbest, feedback, gold, max
-  hope = fear = nil; hope_idx = 0
+def gethopefear_rebol kbest, feedback, gold, max, own_reference=nil
+  hope = fear = nil; new_reference = nil
   type1 = type2 = false
-  # sorted in descending order by max(decoder, psb), best ('hope') first
-  sorted_kbest = kbest.sort{ |x,y| (y.scores[:decoder]+y.scores[:psb])<=>(x.scores[:decoder]+x.scores[:psb]) }
   if feedback == true
+    # hope
     hope = kbest[0]
+    new_reference = hope
+    kbest.each { |k| k.scores[:psb] = BLEU::per_sentence_bleu k.s, new_reference }
+    # fear
+    kbest.sort_by { |k| -(k.scores[:model]-k.score[:psb]) }.each_with_index { |k,i|
+      break if i==max
+      if !exec(k.s, gold, true)[0]
+        fear = k
+        break
+      end
+    }
     type1 = true
   else
-    # select 'best' translation that correctly executes
-    sorted_kbest.each_with_index { |k,i|
-      next if i==0
+    # fear
+    fear = kbest[0]
+    # hope
+    kbest.sort_by { |k| -(k.scores[:model]+k.score[:psb]) }.each_with_index { |k,i|
       break if i==max
       if exec(k.s, gold, true)[0]
-        hope_idx = i
         hope = k
         break
       end
     }
     type2 = true
   end
-  # select 'best' translation that does not correctly execute
-  sorted_kbest.each_with_index { |k,i|
-    break if i>(kbest.size-(hope_idx+1))||i==max
-    if !exec(k.s, gold, true)[0]
-      fear = k
-      break
-    end
-  }
-  # skip if hope or fear could no be found
   skip = true if !hope||!fear
-  return hope, fear, skip, type1, type2
+  return hope, fear, skip, type1, type2, new_reference
 end
 
-def gethopefear_fear_no_exec_hope_exec_skip kbest, feedback, gold, max
+def gethopefear_rebol_light kbest, feedback, gold
   hope = fear = nil
   type1 = type2 = false
   if feedback == true
     hope = kbest[0]
     type1 = true
   else
-    hope = hope_and_fear kbest, 'hope'
+    hope = kbest[hope_and_fear kbest, 'hope']
     type2 = true
   end
-  fear = hope_and_fear kbest, 'fear'
-  # skip if fear executes correctly or hope doesn't
-  skip = exec(fear.s, gold, true)[0]||!exec(hope.s, gold, true)[0]
+  fear = kbest[hope_and_fear kbest, 'fear']
+  # skip example if fear gives the right answer
+  skip = exec(fear.s, gold, true)[0]
   return hope, fear, skip, type1, type2
 end
 
-# new variant w/ "real" reference
-def gethopefear_only_exec kbest, feedback, gold, max, own_reference=nil
+def gethopefear_exec kbest, feedback, gold, max, own_reference=nil
   hope = fear = nil; hope_idx = 0; new_reference = nil
   type1 = type2 = false
   if feedback == true
@@ -158,10 +104,10 @@ def gethopefear_rampion kbest, reference
   # 1best is automatically hope if it matches reference
   if kbest[0].s == reference
     hope = kbest[0]
-    fear = hope_and_fear kbest, 'fear'
+    fear = kbest[hope_and_fear kbest, 'fear']
     type1 = true
   else
-    hope = hope_and_fear kbest, 'hope'
+    hope = kbest[hope_and_fear kbest, 'hope']
     # 1best is automatically fear if it doesn't match reference
     fear = kbest[0]
     type2 = true
diff --git a/lampion.rb b/rebol.rb
index 90db3c9..3c54a3c 100755
--- a/lampion.rb
+++ b/rebol.rb
@@ -91,18 +91,18 @@ def main
     opt :debug,          "debug output",                  :type => :bool,   :default => false, :short => '-d'
     opt :print_kbest,    "print full kbest lists",        :type => :bool,   :default => false, :short => '-l'
     # [learning parameters]
-    opt :eta,                    "learning rate",                                              :type => :float, :default => 0.01,   :short => '-e'
-    opt :iterate,                "iteration X epochs",                                         :type => :int,   :default => 1,      :short => '-j'
-    opt :stop_after,             "stop after x examples",                                      :type => :int,   :default => -1,     :short => '-s'
-    opt :scale_model,            "scale model scores by this factor",                          :type => :float, :default => 1.0,    :short => '-m'
-    opt :normalize,              "normalize weights after each update",                        :type => :bool,  :default => false,  :short => '-n'
+    opt :eta,                    "learning rate",                                              :type => :float,  :default => 0.01,      :short => '-e'
+    opt :iterate,                "iteration X epochs",                                         :type => :int,    :default => 1,         :short => '-j'
+    opt :stop_after,             "stop after x examples",                                      :type => :int,    :default => -1,        :short => '-s'
+    opt :scale_model,            "scale model scores by this factor",                          :type => :float,  :default => 1.0,       :short => '-m'
+    opt :normalize,              "normalize weights after each update",                        :type => :bool,   :default => false,     :short => '-n'
     # don't use when 'bad' examples are filtered:
-    opt :skip_on_no_proper_gold, "skip, if the reference didn't produce a proper gold output", :type => :bool,  :default => false,  :short => '-x'
-    opt :no_update,              "don't update weights",                                       :type => :bool,  :default => false,  :short => '-y'
+    opt :skip_on_no_proper_gold, "skip, if the reference didn't produce a proper gold output", :type => :bool,   :default => false,     :short => '-x'
+    opt :no_update,              "don't update weights",                                       :type => :bool,   :default => false,     :short => '-y'
     # don't use:
-    opt :hope_fear_max,          "# entries to consider when searching good hope/fear",        :type => :int,   :default => 10**10, :short => '-q'
+    opt :hope_fear_max,          "# entries to consider when searching good hope/fear",        :type => :int,    :default => 10**10,    :short => '-q'
     # see hopefear.rb:
-    opt :variant, "standard, rampion, fear_no_exec, fear_no_exec_skip, fear_no_exec_hope_exec, fear_no_exec_hope_exec_skip, only_exec", :default => 'standard', :short => '-v'
+    opt :variant, "rampion, rebol, rebol_light, exec",                                         :type => :string, :default => 'rampion', :short => '-v'
   end
 
   require_relative cfg[:global_vars]
@@ -119,9 +119,8 @@ def main
   gold_mrl   = ReadFile.readlines_strip cfg[:gold_mrl]
   stopwords  = ReadFile.readlines_strip cfg[:stopwords_file]
 
-  # only for 'only_exec' variant
   own_references = nil
-  own_references = references.map{ |i| nil } if cfg[:variant]=='only_exec'
+  own_references = references.map{ |i| nil }
 
   # initialize model
   w = SparseVector.from_file cfg[:init_weights], ' '
@@ -211,28 +210,21 @@ def main
       hope = fear = new_reference = nil
       type1 = type2 = skip = false
       case cfg[:variant]
-      when 'standard'
-        hope, fear, skip, type1, type2 = gethopefear_standard kbest, feedback
       when 'rampion'
         hope, fear, skip, type1, type2 = gethopefear_rampion kbest, references[j]
-      when 'fear_no_exec_skip'
-        hope, fear, skip, type1, type2 = gethopefear_fear_no_exec_skip kbest, feedback, gold[j]
-      when 'fear_no_exec'
-        hope, fear, skip, type1, type2 = gethopefear_fear_no_exec kbest, feedback, gold[j], cfg[:hope_fear_max]
-      when 'fear_no_exec_hope_exec'
-        hope, fear, skip, type1, type2 = gethopefear_fear_no_exec_hope_exec kbest, feedback, gold[j], cfg[:hope_fear_max]
-      when 'fear_no_exec_hope_exec_skip'
-        hope, fear, skip, type1, type2 = gethopefear_fear_no_exec_hope_exec_skip kbest, feedback, gold[j], cfg[:hope_fear_max]
+      when 'rebol'
+        hope, fear, skip, type1, type2, new_reference = gethopefear_rebol kbest, feedback, gold[j], cfg[:hope_fear_max], own_references[j]
+      when 'rebol_light'
+        hope, fear, skip, type1, type2 = gethopefear_rebol_light kbest, feedback, gold[j]
       when 'only_exec'
-        hope, fear, skip, type1, type2, new_reference = gethopefear_only_exec kbest, feedback, gold[j], cfg[:hope_fear_max], own_references[j]
+        hope, fear, skip, type1, type2, new_reference = gethopefear_exec kbest, feedback, gold[j], cfg[:hope_fear_max], own_references[j]
       else
         STDERR.write "NO SUCH VARIANT, exiting.\n"
         exit 1
       end
 
-      # for 'only_exec' variant
       if new_reference
-        own_references[j] = new_reference
+        own_references[j] = new_reference.s
       end
 
       type1_updates+=1 if type1
@@ -312,6 +304,12 @@ def main
 
 eos
 
+    STDERR.write "<<< #{own_references.size} OWN REFERENCES"
+    own_references.each_with_index { |i,j|
+      STDERR.write "#{j} '#{i}'" if i 
+    }
+    STDERR.write ">>>"
+
   }
 end
 
diff --git a/scripts/geoquery/test-nof-old-crawl.sh b/scripts/geoquery/test-nof-old-crawl.sh
deleted file mode 100755
index 79c35a8..0000000
--- a/scripts/geoquery/test-nof-old-crawl.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-/workspace/grounded/lampion/scripts/geoquery/translate.sh $1 $2 < /workspace/grounded/lampion/proper/d/split880.test-nof-old-crawl.in | tee $2.transl | /workspace/grounded/lampion/scripts/geoquery/semparse.rb $3 | tee $2.parsed | /workspace/grounded/lampion/scripts/geoquery/query.rb $3 > $2.output
-/workspace/grounded/lampion/scripts/geoquery/eval.rb /workspace/grounded/lampion/proper/d/split880.test-nof.gold < $2.output > $2.result
-
diff --git a/scripts/geoquery/test-nof-old.sh b/scripts/geoquery/test-nof-old.sh
deleted file mode 100755
index 99a8241..0000000
--- a/scripts/geoquery/test-nof-old.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-/workspace/grounded/lampion/scripts/geoquery/translate.sh $1 $2 < /workspace/grounded/lampion/proper/d/split880.test-nof-old.in | tee $2.transl | /workspace/grounded/lampion/scripts/geoquery/semparse.rb $3 | tee $2.parsed | /workspace/grounded/lampion/scripts/geoquery/query.rb $3 > $2.output
-/workspace/grounded/lampion/scripts/geoquery/eval.rb /workspace/grounded/lampion/proper/d/split880.test-nof.gold < $2.output > $2.result
-
diff --git a/scripts/geoquery/test-nof.sh b/scripts/geoquery/test-nof.sh
deleted file mode 100755
index 786afc2..0000000
--- a/scripts/geoquery/test-nof.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-
-/workspace/grounded/lampion/scripts/geoquery/translate.sh $1 $2 < /workspace/grounded/lampion/proper/d/split880.test-nof.in | tee $2.transl | /workspace/grounded/lampion/scripts/geoquery/semparse.rb $3 | tee $2.parsed | /workspace/grounded/lampion/scripts/geoquery/query.rb $3 > $2.output
-/workspace/grounded/lampion/scripts/geoquery/eval.rb /workspace/grounded/lampion/proper/d/split880.test-nof.gold < $2.output > $2.result
-
diff --git a/scripts/geoquery/test.sh b/scripts/geoquery/test.sh
index 3ac8b2d..3dea047 100755
--- a/scripts/geoquery/test.sh
+++ b/scripts/geoquery/test.sh
@@ -1,5 +1,5 @@
 #!/bin/sh
 
-/workspace/grounded/lampion/scripts/geoquery/translate.sh $1 $2 < /workspace/grounded/lampion/proper/d/split880.test.in | tee $2.transl | /workspace/grounded/lampion/scripts/geoquery/semparse.rb $3 | tee $2.parsed | /workspace/grounded/lampion/scripts/geoquery/query.rb $3 > $2.output
-/workspace/grounded/lampion/scripts/geoquery/eval.rb /workspace/grounded/lampion/proper/d/split880.test.gold < $2.output > $2.result
+/workspace/grounded/rebol/scripts/geoquery/translate.sh $1 $2 < /workspace/grounded/rebol/proper/d/split880.test.in | tee $2.transl | /workspace/grounded/rebol/scripts/geoquery/semparse.rb $3 | tee $2.parsed | /workspace/grounded/rebol/scripts/geoquery/query.rb $3 > $2.output
+/workspace/grounded/rebol/scripts/geoquery/eval.rb /workspace/grounded/rebol/proper/d/split880.test.gold < $2.output > $2.result