diff options
author | Patrick Simianer <p@simianer.de> | 2011-11-13 22:12:46 +0100 |
---|---|---|
committer | Patrick Simianer <p@simianer.de> | 2011-11-13 22:12:46 +0100 |
commit | 7b79fc9e6e6c9c2bb7f977978e319abe2143bbd9 (patch) | |
tree | d7272bb7db6c214f9cf66f665f142470e5aee9ed /dtrain/hstreaming/red-all.rb | |
parent | c57425bbdbb9cbc437ed6cd2774702ecb2617d22 (diff) |
new reducer: active on all tasks
Diffstat (limited to 'dtrain/hstreaming/red-all.rb')
-rwxr-xr-x | dtrain/hstreaming/red-all.rb | 26 |
1 files changed, 26 insertions, 0 deletions
diff --git a/dtrain/hstreaming/red-all.rb b/dtrain/hstreaming/red-all.rb new file mode 100755 index 00000000..bbc65945 --- /dev/null +++ b/dtrain/hstreaming/red-all.rb @@ -0,0 +1,26 @@ +#!/usr/bin/env ruby1.9.1 + + +shard_count_key = "__SHARD_COUNT__" + +STDIN.set_encoding 'utf-8' +STDOUT.set_encoding 'utf-8' + +w = {} +c = {} +w.default = 0 +c.default = 0 +while line = STDIN.gets + key, val = line.split /\t/ + w[key] += val.to_f + c[key] += 1 +end + +puts "# dtrain reducer: active on all" +shard_count = w["__SHARD_COUNT__"] +puts "shard count #{shard_count}" +w.each_key { |k| + if k == shard_count_key then next end + if c[k] == shard_count then puts "#{k}\t#{w[k]/shard_count}" end +} + |