From 06a270c8d3f061bc7fed062cb28605dd8c6e3a8f Mon Sep 17 00:00:00 2001 From: Patrick Simianer Date: Tue, 29 Nov 2011 23:35:51 +0100 Subject: avg.rb lplp.rb --- dtrain/hstreaming/avg.rb | 31 ++++++++++++ dtrain/hstreaming/lplp.rb | 112 ++++++++++++++++++++++++++----------------- dtrain/hstreaming/red-all.rb | 26 ---------- dtrain/hstreaming/red-avg.rb | 27 ----------- 4 files changed, 100 insertions(+), 96 deletions(-) create mode 100755 dtrain/hstreaming/avg.rb delete mode 100755 dtrain/hstreaming/red-all.rb delete mode 100755 dtrain/hstreaming/red-avg.rb (limited to 'dtrain/hstreaming') diff --git a/dtrain/hstreaming/avg.rb b/dtrain/hstreaming/avg.rb new file mode 100755 index 00000000..e0899144 --- /dev/null +++ b/dtrain/hstreaming/avg.rb @@ -0,0 +1,31 @@ +# avg.rb + +shard_count_key = "__SHARD_COUNT__" + +STDIN.set_encoding 'utf-8' +STDOUT.set_encoding 'utf-8' + +w = {} +c = {} +w.default = 0 +c.default = 0 +while line = STDIN.gets + key, val = line.split /\s/ + w[key] += val.to_f + c[key] += 1 +end + +if ARGV.size == 0 + shard_count = w["__SHARD_COUNT__"] +else + shard_count = ARGV[0].to_f +end +w.each_key { |k| + if k == shard_count_key + puts "# shard count: #{shard_count.to_i}" + else + puts "#{k}\t#{w[k]/shard_count}" + puts "# #{c[k]}" + end +} + diff --git a/dtrain/hstreaming/lplp.rb b/dtrain/hstreaming/lplp.rb index edb93e77..0ec21a46 100755 --- a/dtrain/hstreaming/lplp.rb +++ b/dtrain/hstreaming/lplp.rb @@ -2,15 +2,15 @@ # norms def l0(feature_column, n) - if feature_column.size == n then return 1 else return 0 end + if feature_column.size >= n then return 1 else return 0 end end def l1(feature_column, n=-1) - return feature_column.reduce { |sum, i| i.abs } + return feature_column.map { |i| i.abs }.reduce { |sum,i| sum+i } end def l2(feature_column, n=-1) - return Math.sqrt feature_column.reduce { |sum, i| i**2 } + return Math.sqrt feature_column.map { |i| i.abs2 }.reduce { |sum,i| sum+i } end def linfty(feature_column, n=-1) @@ -18,7 +18,7 @@ def linfty(feature_column, n=-1) end # stats -def M(feature_column, n) +def median(feature_column, n) return feature_column.concat(0.step(n-feature_column.size-1).map{|i|0}).sort[feature_column.size/2] end @@ -27,22 +27,73 @@ def mean(feature_column, n) end # selection -def select_k(weights, normfn, n, k=10000) - weights.sort{|a,b| normfn.call(b[1], n) <=> normfn.call(a[1], n)}.each { |p| +def select_k(weights, norm_fun, n, k=10000) + weights.sort{|a,b| norm_fun.call(b[1], n) <=> norm_fun.call(a[1], n)}.each { |p| puts "#{p[0]}\t#{mean(p[1], n)}" k -= 1 if k == 0 then break end } end -def cut(weights, normfn, n, epsilon=0.0001) +def cut(weights, norm_fun, n, epsilon=0.0001) weights.each { |k,v| - if normfn.call(v).abs > epsilon + if norm_fun.call(v, n).abs > epsilon puts "#{k}\t#{mean(v, n)}" end } end +# test +def _test() + puts + w = {} + w["a"] = [1, 2, 3] + w["b"] = [1, 2] + w["c"] = [66] + w["d"] = [10, 20, 30] + n = 3 + puts w.to_s + puts + puts "select_k" + puts "l0 expect ad" + select_k(w, method(:l0), n, 2) + puts "l1 expect cd" + select_k(w, method(:l1), n, 2) + puts "l2 expect c" + select_k(w, method(:l2), n, 1) + puts + puts "cut" + puts "l1 expect cd" + cut(w, method(:l1), n, 7) + puts + puts "median" + a = [1,2,3,4,5] + puts a.to_s + puts median(a, 5) + puts + puts "#{median(a, 7)} <- that's because we add missing 0s:" + puts a.concat(0.step(7-a.size-1).map{|i|0}).to_s + puts + puts "mean expect bc" + w.clear + w["a"] = [2] + w["b"] = [2.1] + w["c"] = [2.2] + cut(w, method(:mean), 1, 2.05) + exit +end +_test() + +# actually do something +def usage() + puts "lplp.rb [n] < " + exit +end + +if ARGV.size < 3 then usage end +norm_fun = method(ARGV[0].to_sym) +type = ARGV[1] +x = ARGV[2].to_f shard_count_key = "__SHARD_COUNT__" @@ -53,49 +104,24 @@ w = {} shard_count = 0 while line = STDIN.gets key, val = line.split /\t/ - if k = shard_count_key + if key == shard_count_key shard_count += 1 next end if w.has_key? key - w[key].push val + w[key].push val.to_f else - w[key] = [val] + w[key] = [val.to_f] end end -select_k(w, method(:l1), shard_count, 100000) +if ARGV.size == 4 then shard_count = ARGV[3].to_f end -def _test() -puts -w = {} -w["a"] = [1, 2, 3] -w["b"] = [1, 2] -w["c"] = [66] -w["d"] = [10, 20, 30] -n = 3 -puts w.to_s -puts -puts "select_k" -puts "l0 expect ad" -select_k(w, method(:l0), n, 2) -puts "l1 expect c" -select_k(w, method(:l1), n, 1) -puts "l2 expect d" -select_k(w, method(:l2), n, 1) -puts -puts "cut" -puts "l1 expect cd" -cut(w, method(:l1), n, 7) -puts -puts "M" -a = [1,3,4,5,6] -puts a.to_s -puts M(a, 7) -puts "that's because we add missing 0s" -puts a.concat(0.step(7-a.size-1).map{|i|0}).to_s -puts +if type == 'cut' + cut(w, norm_fun, shard_count, x) +elsif type == 'select_k' + select_k(w, norm_fun, shard_count, x) +else + puts "oh oh" end -#_test() - diff --git a/dtrain/hstreaming/red-all.rb b/dtrain/hstreaming/red-all.rb deleted file mode 100755 index bbc65945..00000000 --- a/dtrain/hstreaming/red-all.rb +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env ruby1.9.1 - - -shard_count_key = "__SHARD_COUNT__" - -STDIN.set_encoding 'utf-8' -STDOUT.set_encoding 'utf-8' - -w = {} -c = {} -w.default = 0 -c.default = 0 -while line = STDIN.gets - key, val = line.split /\t/ - w[key] += val.to_f - c[key] += 1 -end - -puts "# dtrain reducer: active on all" -shard_count = w["__SHARD_COUNT__"] -puts "shard count #{shard_count}" -w.each_key { |k| - if k == shard_count_key then next end - if c[k] == shard_count then puts "#{k}\t#{w[k]/shard_count}" end -} - diff --git a/dtrain/hstreaming/red-avg.rb b/dtrain/hstreaming/red-avg.rb deleted file mode 100755 index 9326ffbe..00000000 --- a/dtrain/hstreaming/red-avg.rb +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env ruby1.9.1 - -shard_count_key = "__SHARD_COUNT__" - -STDIN.set_encoding 'utf-8' -STDOUT.set_encoding 'utf-8' - -w = {} -c = {} -w.default = 0 -c.default = 0 -while line = STDIN.gets - key, val = line.split /\t/ - w[key] += val.to_f - c[key] += 1 -end - -puts "# dtrain reducer: average" -shard_count = w["__SHARD_COUNT__"] -w.each_key { |k| - if k == shard_count_key - puts "# shard count: #{shard_count.to_i}" - else - puts "#{k}\t#{w[k]/shard_count}\t# #{c[k]}" - end -} - -- cgit v1.2.3