From 26c490f404731d053a6205719b6246502c07b449 Mon Sep 17 00:00:00 2001
From: Patrick Simianer
Date: Sat, 14 Jun 2014 16:46:27 +0200
Subject: init
---
hadoop/wordcount/WordCount.java | 57 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 57 insertions(+)
create mode 100644 hadoop/wordcount/WordCount.java
(limited to 'hadoop/wordcount/WordCount.java')
diff --git a/hadoop/wordcount/WordCount.java b/hadoop/wordcount/WordCount.java
new file mode 100644
index 0000000..8917aee
--- /dev/null
+++ b/hadoop/wordcount/WordCount.java
@@ -0,0 +1,57 @@
+package org.myorg;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.util.*;
+
+public class WordCount {
+
+ // Mapper
+ public static class Map extends MapReduceBase implements Mapper {
+ private final static IntWritable one = new IntWritable(1);
+ private Text word = new Text();
+ public void map(LongWritable key, Text value, OutputCollector output, Reporter reporter) throws IOException {
+ String line = value.toString(); // TextInputFormat
+ StringTokenizer tokenizer = new StringTokenizer(line);
+ while (tokenizer.hasMoreTokens()) {
+ word.set(tokenizer.nextToken());
+ output.collect(word, one);
+ }
+ }
+ }
+
+ // Reducer
+ public static class Reduce extends MapReduceBase implements Reducer {
+ public void reduce(Text key, Iterator values, OutputCollector output, Reporter reporter) throws IOException {
+ int sum = 0;
+ while (values.hasNext()) {
+ sum += values.next().get();
+ }
+ output.collect(key, new IntWritable(sum));
+ }
+ }
+
+ // Driver
+ public static void main(String[] args) throws Exception {
+ JobConf conf = new JobConf(WordCount.class);
+ conf.setJobName("wordcount");
+ conf.setNumReduceTasks(200);
+ conf.setOutputKeyClass(Text.class);
+ conf.setOutputValueClass(IntWritable.class);
+ conf.setMapperClass(Map.class);
+ conf.setCombinerClass(Reduce.class); // Combiner
+ conf.setReducerClass(Reduce.class);
+ conf.setInputFormat(TextInputFormat.class);
+ conf.setOutputFormat(TextOutputFormat.class);
+ FileInputFormat.setInputPaths(conf, new Path(args[0]));
+ FileOutputFormat.setOutputPath(conf, new Path(args[1]));
+ JobClient.runJob(conf);
+ }
+
+}
+
--
cgit v1.2.3