summaryrefslogtreecommitdiff
path: root/dtrain/hstreaming/red-all.rb
diff options
context:
space:
mode:
Diffstat (limited to 'dtrain/hstreaming/red-all.rb')
-rwxr-xr-xdtrain/hstreaming/red-all.rb26
1 files changed, 26 insertions, 0 deletions
diff --git a/dtrain/hstreaming/red-all.rb b/dtrain/hstreaming/red-all.rb
new file mode 100755
index 00000000..bbc65945
--- /dev/null
+++ b/dtrain/hstreaming/red-all.rb
@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby1.9.1
+
+
+shard_count_key = "__SHARD_COUNT__"
+
+STDIN.set_encoding 'utf-8'
+STDOUT.set_encoding 'utf-8'
+
+w = {}
+c = {}
+w.default = 0
+c.default = 0
+while line = STDIN.gets
+ key, val = line.split /\t/
+ w[key] += val.to_f
+ c[key] += 1
+end
+
+puts "# dtrain reducer: active on all"
+shard_count = w["__SHARD_COUNT__"]
+puts "shard count #{shard_count}"
+w.each_key { |k|
+ if k == shard_count_key then next end
+ if c[k] == shard_count then puts "#{k}\t#{w[k]/shard_count}" end
+}
+