From c69080adb7cf6dbf25c0ed1129fe988163bc26fd Mon Sep 17 00:00:00 2001 From: Patrick Simianer Date: Wed, 23 Dec 2015 13:53:03 +0100 Subject: hadoop_uniq: uniq with hadoop-streaming --- hadoop_uniq | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 hadoop_uniq (limited to 'hadoop_uniq') diff --git a/hadoop_uniq b/hadoop_uniq new file mode 100755 index 0000000..5052419 --- /dev/null +++ b/hadoop_uniq @@ -0,0 +1,11 @@ +#!/bin/zsh + +HADOOP_HOME=/usr/lib/hadoop + +$HADOOP_HOME/bin/hadoop jar /usr/lib/hadoop-mapreduce/hadoop-streaming.jar \ + -D mapred.reduce.tasks=98 \ + -input d \ + -output d.uniq \ + -mapper 'cut -d " " -f 1' \ + -reducer /usr/bin/uniq + -- cgit v1.2.3