hadoop-uniq


1
2
3
4
5
6
7
8
9
10
11

#!/bin/zsh

HADOOP_HOME=/usr/lib/hadoop

$HADOOP_HOME/bin/hadoop  jar /usr/lib/hadoop-mapreduce/hadoop-streaming.jar \
    -D mapred.reduce.tasks=98 \
    -input d \
    -output d.uniq \
    -mapper 'cut -d " " -f 1' \
    -reducer /usr/bin/uniq