diff options
author | Patrick Simianer <simianer@cl.uni-heidelberg.de> | 2011-09-27 00:10:48 +0200 |
---|---|---|
committer | Patrick Simianer <simianer@cl.uni-heidelberg.de> | 2011-09-27 00:10:48 +0200 |
commit | 40c884b3d74a1779be80974fc6fc926b0812813c (patch) | |
tree | fca25c58549907528b201a14c4014f6a868d487f /dtrain/hstreaming/red-avg.rb | |
parent | 36de7283576dd22a91577ef175c62434f3d933b4 (diff) |
some updates for hadoop streaming
Diffstat (limited to 'dtrain/hstreaming/red-avg.rb')
-rwxr-xr-x | dtrain/hstreaming/red-avg.rb | 26 |
1 files changed, 26 insertions, 0 deletions
diff --git a/dtrain/hstreaming/red-avg.rb b/dtrain/hstreaming/red-avg.rb new file mode 100755 index 00000000..11dc0d71 --- /dev/null +++ b/dtrain/hstreaming/red-avg.rb @@ -0,0 +1,26 @@ +#!/usr/bin/env ruby1.9.1 + + +STDIN.set_encoding 'utf-8' + +shard_count_key = "__SHARD_COUNT__" + +w = {} +c = {} +w.default = 0 +c.default = 0 +while line = STDIN.gets + key, val = line.split /\t/ + w[key] += val.to_f + c[key] += 1.0 +end + +shard_count = w["__SHARD_COUNT__"] + +w.each_key { |k| + if k == shard_count_key then next end + puts "#{k}\t{w[k]/shard_count}" +} + +puts "#{shard_count_key}\t#{w[shard_count_key]}" + |