summaryrefslogtreecommitdiff
path: root/hadoop/streaming
diff options
context:
space:
mode:
Diffstat (limited to 'hadoop/streaming')
-rw-r--r--hadoop/streaming/mapper/input10
-rw-r--r--hadoop/streaming/mapper/mapper_test.py9
-rwxr-xr-xhadoop/streaming/mapper/mapper_test.sh23
-rw-r--r--hadoop/streaming/mapper/mapper_test1.py9
-rwxr-xr-xhadoop/streaming/mapper/mapper_test1.sh24
-rw-r--r--hadoop/streaming/no_reducer/input8
-rwxr-xr-xhadoop/streaming/no_reducer/no_reducer.rb9
-rwxr-xr-xhadoop/streaming/no_reducer/no_reducer.sh23
-rw-r--r--hadoop/streaming/partitioner/input8
-rwxr-xr-xhadoop/streaming/partitioner/partitioner_test.sh22
-rwxr-xr-xhadoop/streaming/partitioner/partitioner_test1.sh27
-rw-r--r--hadoop/streaming/secondary_sort/input8
-rwxr-xr-xhadoop/streaming/secondary_sort/secondary_sort.sh30
-rw-r--r--hadoop/streaming/test/input10
-rwxr-xr-xhadoop/streaming/test/test.sh23
15 files changed, 243 insertions, 0 deletions
diff --git a/hadoop/streaming/mapper/input b/hadoop/streaming/mapper/input
new file mode 100644
index 0000000..338fd87
--- /dev/null
+++ b/hadoop/streaming/mapper/input
@@ -0,0 +1,10 @@
+0 1 a c
+0 2 b c
+0 3 c c
+1 4 a c
+1 5 b c
+1 6 c c
+1 7 d c
+2 8 a c
+2 9 b c
+2 10 c c
diff --git a/hadoop/streaming/mapper/mapper_test.py b/hadoop/streaming/mapper/mapper_test.py
new file mode 100644
index 0000000..d358bda
--- /dev/null
+++ b/hadoop/streaming/mapper/mapper_test.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python2
+
+import sys
+
+
+if __name__ == "__main__":
+ for line in sys.stdin:
+ print line.upper()
+
diff --git a/hadoop/streaming/mapper/mapper_test.sh b/hadoop/streaming/mapper/mapper_test.sh
new file mode 100755
index 0000000..f0c5da3
--- /dev/null
+++ b/hadoop/streaming/mapper/mapper_test.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+hadoop dfs -put input input
+
+HADOOP_HOME=/usr/lib/hadoop
+HADOOP_VERSION=0.20.2-cdh3u1
+JAR=contrib/streaming/hadoop-streaming-$HADOOP_VERSION.jar
+HSTREAMING="$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/$JAR"
+
+OUT=mapper_test_out
+
+$HSTREAMING \
+ -input input \
+ -output $OUT \
+ -mapper "python mapper.py" \
+ -file mapper_test.py \
+ -reducer org.apache.hadoop.mapred.lib.IdentityReducer \
+ -jobconf mapred.reduce.tasks=3
+
+hadoop dfs -get $OUT .
+hadoop dfs -rmr $OUT
+hadoop dfs -rm input
+
diff --git a/hadoop/streaming/mapper/mapper_test1.py b/hadoop/streaming/mapper/mapper_test1.py
new file mode 100644
index 0000000..79c8aa6
--- /dev/null
+++ b/hadoop/streaming/mapper/mapper_test1.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python2
+
+import sys
+
+
+if __name__ == "__main__":
+ for line in sys.stdin:
+ print "MYKEY\t%s"%line.upper()
+
diff --git a/hadoop/streaming/mapper/mapper_test1.sh b/hadoop/streaming/mapper/mapper_test1.sh
new file mode 100755
index 0000000..475699f
--- /dev/null
+++ b/hadoop/streaming/mapper/mapper_test1.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+
+hadoop dfs -put input input
+
+HADOOP_HOME=/usr/lib/hadoop
+HADOOP_VERSION=0.20.2-cdh3u1
+JAR=contrib/streaming/hadoop-streaming-$HADOOP_VERSION.jar
+HSTREAMING="$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/$JAR"
+
+OUT=mapper_test1_out
+
+$HSTREAMING \
+ -input input \
+ -output $OUT \
+ -mapper "python mapper1.py" \
+ -file mapper_test1.py \
+ -reducer org.apache.hadoop.mapred.lib.IdentityReducer \
+ -jobconf mapred.reduce.tasks=3
+
+hadoop dfs -get $OUT .
+hadoop dfs -rmr $OUT
+hadoop dfs -rm input
+
diff --git a/hadoop/streaming/no_reducer/input b/hadoop/streaming/no_reducer/input
new file mode 100644
index 0000000..71ac1b5
--- /dev/null
+++ b/hadoop/streaming/no_reducer/input
@@ -0,0 +1,8 @@
+a
+b
+c
+d
+e
+f
+g
+h
diff --git a/hadoop/streaming/no_reducer/no_reducer.rb b/hadoop/streaming/no_reducer/no_reducer.rb
new file mode 100755
index 0000000..4410b93
--- /dev/null
+++ b/hadoop/streaming/no_reducer/no_reducer.rb
@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+
+i = 0
+while l = STDIN.gets
+ puts "line #{i} (#{l.strip})"
+ i+=1
+end
+
diff --git a/hadoop/streaming/no_reducer/no_reducer.sh b/hadoop/streaming/no_reducer/no_reducer.sh
new file mode 100755
index 0000000..c32bfdd
--- /dev/null
+++ b/hadoop/streaming/no_reducer/no_reducer.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+
+hadoop dfs -put input input
+
+HADOOP_HOME=/usr/lib/hadoop
+HADOOP_VERSION=0.20.2-cdh3u1
+JAR=contrib/streaming/hadoop-streaming-$HADOOP_VERSION.jar
+HSTREAMING="$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/$JAR"
+
+OUT=no_reducer_out
+
+$HSTREAMING \
+ -input input \
+ -output $OUT \
+ -mapper "no_reducer.rb" \
+ -file "no_reducer.rb" \
+ -reducer NONE
+
+hadoop dfs -get $OUT .
+hadoop dfs -rmr $OUT
+hadoop dfs -rm input
+
diff --git a/hadoop/streaming/partitioner/input b/hadoop/streaming/partitioner/input
new file mode 100644
index 0000000..06c6a28
--- /dev/null
+++ b/hadoop/streaming/partitioner/input
@@ -0,0 +1,8 @@
+1.1 a
+2.2 b
+3.1 c
+4.2 d
+4.1 e
+1.2 x
+2.1 y
+4.3 q
diff --git a/hadoop/streaming/partitioner/partitioner_test.sh b/hadoop/streaming/partitioner/partitioner_test.sh
new file mode 100755
index 0000000..bfb2185
--- /dev/null
+++ b/hadoop/streaming/partitioner/partitioner_test.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+
+hadoop dfs -put input input
+
+HADOOP_HOME=/usr/lib/hadoop
+HADOOP_VERSION=0.20.2-cdh3u1
+JAR=contrib/streaming/hadoop-streaming-$HADOOP_VERSION.jar
+HSTREAMING="$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/$JAR"
+
+OUT=partitioner_test_out
+
+$HSTREAMING \
+ -input input \
+ -output $OUT \
+ -mapper /bin/cat \
+ -jobconf mapred.reduce.tasks=2
+
+hadoop dfs -get $OUT .
+hadoop dfs -rmr $OUT
+hadoop dfs -rm input
+
diff --git a/hadoop/streaming/partitioner/partitioner_test1.sh b/hadoop/streaming/partitioner/partitioner_test1.sh
new file mode 100755
index 0000000..759f823
--- /dev/null
+++ b/hadoop/streaming/partitioner/partitioner_test1.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+
+hadoop dfs -put input input
+
+HADOOP_HOME=/usr/lib/hadoop
+HADOOP_VERSION=0.20.2-cdh3u1
+JAR=contrib/streaming/hadoop-streaming-$HADOOP_VERSION.jar
+HSTREAMING="$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/$JAR"
+
+OUT=partitioner_test1_out
+
+$HSTREAMING \
+ -input input \
+ -output $OUT \
+ -mapper /bin/cat \
+ -jobconf mapred.reduce.tasks=2 \
+ -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner \
+ -jobconf stream.map.output.field.separator="\t" \
+ -jobconf map.output.key.field.separator=. \
+ -jobconf stream.num.map.output.key.fields=2 \
+ -jobconf mapred.text.key.partitioner.options=-k1
+
+hadoop dfs -get $OUT .
+hadoop dfs -rmr $OUT
+hadoop dfs -rm input
+
diff --git a/hadoop/streaming/secondary_sort/input b/hadoop/streaming/secondary_sort/input
new file mode 100644
index 0000000..5aa7ec9
--- /dev/null
+++ b/hadoop/streaming/secondary_sort/input
@@ -0,0 +1,8 @@
+0-*-1 a:1 a:2 a:3
+0-*-2 a:1 a:2 a:3
+0-*-10 a:1 a:2 a:3
+1-*-2 a:1 a:2 a:3
+2-*-0 a:1 a:2 a:3
+2-*-2 a:1 a:2 a:3
+3-*-3 a:1 a:2 a:3
+10-*-0 a:1 a:2 a:3
diff --git a/hadoop/streaming/secondary_sort/secondary_sort.sh b/hadoop/streaming/secondary_sort/secondary_sort.sh
new file mode 100755
index 0000000..c45f381
--- /dev/null
+++ b/hadoop/streaming/secondary_sort/secondary_sort.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+
+hadoop dfs -put input input
+
+HADOOP_HOME=/usr/lib/hadoop
+HADOOP_VERSION=0.20.2-cdh3u1
+JAR=contrib/streaming/hadoop-streaming-$HADOOP_VERSION.jar
+HSTREAMING="$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/$JAR"
+
+OUT=secondary_sort_out
+
+$HSTREAMING \
+ -input input \
+ -output $OUT \
+ -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner \
+ -jobconf map.output.key.field.separator="-*-" \
+ -jobconf mapred.text.key.partitioner.options="-k1,1n" \
+ -mapper /bin/cat \
+ -reducer org.apache.hadoop.mapred.lib.IdentityReducer \
+ -jobconf mapred.output.key.comparator.class=org.apache.hadoop.mapred.lib.KeyFieldBasedComparator \
+ -jobconf stream.num.map.output.key.fields=2 \
+ -jobconf stream.map.output.field.separator="\t" \
+ -jobconf mapred.text.key.comparator.options="-k1,1n -k2,2nr" \
+ -jobconf mapred.reduce.tasks=3
+
+hadoop dfs -get $OUT .
+hadoop dfs -rmr $OUT
+hadoop dfs -rm input
+
diff --git a/hadoop/streaming/test/input b/hadoop/streaming/test/input
new file mode 100644
index 0000000..338fd87
--- /dev/null
+++ b/hadoop/streaming/test/input
@@ -0,0 +1,10 @@
+0 1 a c
+0 2 b c
+0 3 c c
+1 4 a c
+1 5 b c
+1 6 c c
+1 7 d c
+2 8 a c
+2 9 b c
+2 10 c c
diff --git a/hadoop/streaming/test/test.sh b/hadoop/streaming/test/test.sh
new file mode 100755
index 0000000..2f09bac
--- /dev/null
+++ b/hadoop/streaming/test/test.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+
+hadoop dfs -put input input
+
+HADOOP_HOME=/usr/lib/hadoop
+HADOOP_VERSION=0.20.2-cdh3u1
+JAR=contrib/streaming/hadoop-streaming-$HADOOP_VERSION.jar
+HSTREAMING="$HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/$JAR"
+
+OUT=test_out
+
+$HSTREAMING \
+ -input input \
+ -output $OUT \
+ -mapper /bin/cat \
+ -jobconf "mapred.reduce.tasks=3" \
+ -reducer org.apache.hadoop.mapred.lib.IdentityReducer
+
+hadoop dfs -get $OUT .
+hadoop dfs -rmr $OUT
+hadoop dfs -rm input
+
generated by cgit v1.2.3 (git 2.39.1) at 2024-08-22 00:50:37 +0000