summaryrefslogtreecommitdiff
path: root/dtrain/hstreaming/lplp.rb
blob: 57353adb64fb71ca3f43378dfc387b6957631f58 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
# lplp.rb

# norms
def l0(feature_column, n)
  if feature_column.size >= n then return 1 else return 0 end
end

def l1(feature_column, n=-1)
  return feature_column.map { |i| i.abs }.reduce { |sum,i| sum+i }
end

def l2(feature_column, n=-1)
  return Math.sqrt feature_column.map { |i| i.abs2 }.reduce { |sum,i| sum+i }
end

def linfty(feature_column, n=-1)
  return feature_column.map { |i| i.abs }.max
end

# stats
def median(feature_column, n)
  return feature_column.concat(0.step(n-feature_column.size-1).map{|i|0}).sort[feature_column.size/2]
end

def mean(feature_column, n)
  return feature_column.reduce { |sum, i| sum+i } / n
end

# selection
def select_k(weights, norm_fun, n, k=10000)
  weights.sort{|a,b| norm_fun.call(b[1], n) <=> norm_fun.call(a[1], n)}.each { |p|
    puts "#{p[0]}\t#{mean(p[1], n)}" 
    k -= 1
    if k == 0 then break end
  }
end

def cut(weights, norm_fun, n, epsilon=0.0001)
  weights.each { |k,v|
    if norm_fun.call(v, n).abs >= epsilon
      puts "#{k}\t#{mean(v, n)}"
    end
  }
end

# test
def _test()
  puts
  w = {}
  w["a"] = [1, 2, 3]
  w["b"] = [1, 2]
  w["c"] = [66]
  w["d"] = [10, 20, 30]
  n = 3
  puts w.to_s
  puts
  puts "select_k"
  puts "l0 expect ad"
  select_k(w, method(:l0), n, 2)
  puts "l1 expect cd"
  select_k(w, method(:l1), n, 2)
  puts "l2 expect c"
  select_k(w, method(:l2), n, 1)
  puts
  puts "cut"
  puts "l1 expect cd"
  cut(w, method(:l1), n, 7)
  puts
  puts "median"
  a = [1,2,3,4,5]
  puts a.to_s
  puts median(a, 5)
  puts
  puts "#{median(a, 7)} <- that's because we add missing 0s:"
  puts a.concat(0.step(7-a.size-1).map{|i|0}).to_s
  puts
  puts "mean expect bc"
  w.clear
  w["a"] = [2]
  w["b"] = [2.1]
  w["c"] = [2.2]
  cut(w, method(:mean), 1, 2.05)
 exit
end
#_test()

# actually do something
def usage()
  puts "lplp.rb <l0,l1,l2,linfty,mean,median> <cut|select_k> <k|threshold> [n] < <input>"
  puts "   l0...: norms for selection"
  puts "select_k: only output top k (according to the norm of their column vector) features"
  puts "     cut: output features with weight >= threshold"
  puts "       n: if we do not have a shard count use this number for averaging"
  exit
end

if ARGV.size < 3 then usage end
norm_fun = method(ARGV[0].to_sym)
type = ARGV[1]
x = ARGV[2].to_f

shard_count_key = "__SHARD_COUNT__"

STDIN.set_encoding 'utf-8'
STDOUT.set_encoding 'utf-8'

w = {}
shard_count = 0
while line = STDIN.gets
  key, val = line.split /\s+/
  if key == shard_count_key
    shard_count += 1
    next
  end
  if w.has_key? key
    w[key].push val.to_f
  else
    w[key] = [val.to_f]
  end
end

if ARGV.size == 4 then shard_count = ARGV[3].to_f end

if type == 'cut'
  cut(w, norm_fun, shard_count, x)
elsif type == 'select_k'
  select_k(w, norm_fun, shard_count, x)
else
  puts "oh oh"
end