diff options
author | Patrick Simianer <simianer@cl.uni-heidelberg.de> | 2011-09-27 00:10:48 +0200 |
---|---|---|
committer | Patrick Simianer <simianer@cl.uni-heidelberg.de> | 2011-09-27 00:10:48 +0200 |
commit | 40c7d43f8cd5ccaef22f098937b985d4101368f9 (patch) | |
tree | 6a039583398f18b4ec1295bba156e62078e53cb3 /dtrain/hstreaming/red-avg.rb | |
parent | 8bb00a2a2775442418f1cb7c041f7cba5d6e0d42 (diff) |
some updates for hadoop streaming
Diffstat (limited to 'dtrain/hstreaming/red-avg.rb')
-rwxr-xr-x | dtrain/hstreaming/red-avg.rb | 26 |
1 files changed, 26 insertions, 0 deletions
diff --git a/dtrain/hstreaming/red-avg.rb b/dtrain/hstreaming/red-avg.rb new file mode 100755 index 00000000..11dc0d71 --- /dev/null +++ b/dtrain/hstreaming/red-avg.rb @@ -0,0 +1,26 @@ +#!/usr/bin/env ruby1.9.1 + + +STDIN.set_encoding 'utf-8' + +shard_count_key = "__SHARD_COUNT__" + +w = {} +c = {} +w.default = 0 +c.default = 0 +while line = STDIN.gets + key, val = line.split /\t/ + w[key] += val.to_f + c[key] += 1.0 +end + +shard_count = w["__SHARD_COUNT__"] + +w.each_key { |k| + if k == shard_count_key then next end + puts "#{k}\t{w[k]/shard_count}" +} + +puts "#{shard_count_key}\t#{w[shard_count_key]}" + |