summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xlib/nlp_ruby.rb1
-rw-r--r--lib/nlp_ruby/SparseVector.rb15
-rw-r--r--lib/nlp_ruby/bleu.rb25
-rw-r--r--lib/nlp_ruby/misc.rb26
-rw-r--r--lib/nlp_ruby/stringutil.rb5
-rw-r--r--lib/nlp_ruby/ttable.rb17
6 files changed, 80 insertions, 9 deletions
diff --git a/lib/nlp_ruby.rb b/lib/nlp_ruby.rb
index 212c367..f0242af 100755
--- a/lib/nlp_ruby.rb
+++ b/lib/nlp_ruby.rb
@@ -11,6 +11,7 @@ require 'nlp_ruby/dags'
require 'nlp_ruby/semirings'
require 'nlp_ruby/bleu'
require 'nlp_ruby/misc'
+require 'nlp_ruby/cdec'
STDIN.set_encoding 'utf-8'
STDOUT.set_encoding 'utf-8'
diff --git a/lib/nlp_ruby/SparseVector.rb b/lib/nlp_ruby/SparseVector.rb
index 9919a65..964ef4e 100644
--- a/lib/nlp_ruby/SparseVector.rb
+++ b/lib/nlp_ruby/SparseVector.rb
@@ -53,10 +53,21 @@ class SparseVector < Hash
return Math.sqrt(sum)
end
- def to_kv
+ # FIXME
+ def from_kv_file fn, sep=' '
+ f = ReadFile.new(fn)
+ while line = f.gets
+ key, value = line.strip.split sep
+ value = value.to_f
+ self[key] = value
+ end
+ end
+
+ # FIXME
+ def to_kv sep='='
a = []
self.each_pair { |k,v|
- a << "#{k}=#{v}"
+ a << "#{k}#{sep}#{v}"
}
return a.join ' '
end
diff --git a/lib/nlp_ruby/bleu.rb b/lib/nlp_ruby/bleu.rb
index 42be45e..ee91985 100644
--- a/lib/nlp_ruby/bleu.rb
+++ b/lib/nlp_ruby/bleu.rb
@@ -79,12 +79,12 @@ def BLEU::get_counts hypothesis, reference, n, times=1
return p
end
-def BLEU::brevity_penalty(c, r)
- if c > r then return 1.0 end
+def BLEU::brevity_penalty c, r
+ return 1.0 if c>r
return Math.exp(1-r/c)
end
-def BLEU::bleu(counts, n, debug=false)
+def BLEU::bleu counts, n, debug=false
corpus_stats = NgramCounts.new n
counts.each { |i| corpus_stats.plus_eq i }
sum = 0.0
@@ -105,6 +105,25 @@ def BLEU::hbleu counts, n, debug=false
(100*bleu(counts, n, debug)).round(3)
end
+def BLEU::per_sentence_bleu hypothesis, reference, n=4
+ h_ng = {}; r_ng = {}
+ (1).upto(n) {|i| h_ng[i] = []; r_ng[i] = []}
+ ngrams(hypothesis, n) {|i| h_ng[i.size] << i}
+ ngrams(reference, n) {|i| r_ng[i.size] << i}
+ m = [n, reference.split.size].min
+ weight = 1.0/m
+ add = 0.0
+ sum = 0
+ (1).upto(m) { |i|
+ counts_clipped = 0
+ counts_sum = h_ng[i].size
+ h_ng[i].uniq.each {|j| counts_clipped += r_ng[i].count(j)}
+ add = 1.0 if i >= 2
+ sum += weight * Math.log((counts_clipped + add)/(counts_sum + add));
+ }
+ return brevity_penalty(hypothesis.strip.split.size, reference.strip.split.size) * Math.exp(sum)
+end
+
end
diff --git a/lib/nlp_ruby/misc.rb b/lib/nlp_ruby/misc.rb
index 9a4064f..1fa3878 100644
--- a/lib/nlp_ruby/misc.rb
+++ b/lib/nlp_ruby/misc.rb
@@ -2,5 +2,31 @@ class Array
def max_index
self.index(self.max)
end
+
+ def is_subset_of? other
+ self.each { |i|
+ if other.include? i
+ return false
+ end
+ }
+ return true
+ end
+end
+
+def spawn_with_timeout cmd, t=4, debug=false
+ require 'timeout'
+ STDERR.write cmd+"\n" if debug
+ pipe_in, pipe_out = IO.pipe
+ pid = Process.spawn(cmd, :out => pipe_out)
+ begin
+ Timeout.timeout(t) { Process.wait pid }
+ rescue Timeout::Error
+ return ""
+ # accept the zombies
+ #Process.kill('TERM', pid)
+ end
+ pipe_out.close
+ return pipe_in.read
end
+
diff --git a/lib/nlp_ruby/stringutil.rb b/lib/nlp_ruby/stringutil.rb
index 4091994..d7381bb 100644
--- a/lib/nlp_ruby/stringutil.rb
+++ b/lib/nlp_ruby/stringutil.rb
@@ -48,3 +48,8 @@ def read_cfg fn
return cfg
end
+def bag_of_words s, stopwords=[]
+ s.strip.split.uniq.sort.reject{ |w| stopwords.include? w }
+end
+
+
diff --git a/lib/nlp_ruby/ttable.rb b/lib/nlp_ruby/ttable.rb
index 598e318..14d6c5d 100644
--- a/lib/nlp_ruby/ttable.rb
+++ b/lib/nlp_ruby/ttable.rb
@@ -15,18 +15,21 @@ def read_phrase_table fn
return table
end
+# FIXME
class Translation
- attr_accessor :id, :s, :raw, :f, :score
+ attr_accessor :id, :s, :raw, :f, :score, :rank, :other_score
- def initialize id=nil, raw=nil, s=nil, f=nil, score=nil
+ def initialize id=nil, raw=nil, s=nil, f=nil, score=nil, rank=nil, other_score=nil
@id = id
@raw = raw
@s = s
@f = f
@score = score
+ @rank = rank
+ @other_score = other_score
end
- def from_s t, strip_alignment=true
+ def from_s t, strip_alignment=true, rank=nil
id, raw, features, score = splitpipe(t, 3)
raw.strip!
@raw = raw
@@ -39,10 +42,16 @@ class Translation
@id = id.to_i
@f = read_feature_string features
@score = score.to_f
+ @rank = rank
+ @other_score = nil
end
def to_s
- [id, s, f.to_kv, score].join ' ||| '
+ [@id, @s, @f.to_kv, @score].join ' ||| '
+ end
+
+ def to_s2
+ [@rank, @s, @f.to_kv, @score, @other_score].join ' ||| '
end
end