MapReduce WordCount

/** * */ /** * @author hadoop * */ package MapReduce; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; //o public class WordCount { public static void main(String[] args) throws Exception { Configuration conf=new Configuration();//wordcounter zong guanjia=>job Job job = Job.getInstance(conf); job.setMapperClass(MyMapper.class); job.setReducerClass(MyReducer.class); job.setJarByClass(WordCount.class); //map input default,not need to set,so just set output job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); //reduce:map's output is reduce's input,so just set reduce;s output //reduce's output is the final output job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); //set data source and where to put FileInputFormat.setInputPaths(job, new Path("hdfs://localhost:9000/new/WordCount.txt"));//represent hdfs root content,after "/source.txt",source exists FileOutputFormat.setOutputPath(job, new Path("hdfs://localhost:9000/output.txt"));//output where,after"/",book.txt,book can be not exist job.waitForCompletion(true);//start } } class MyMapper extends Mapper<Object,Text,Text,IntWritable>{ //mapper-alt+/-oracle.apache.hadoop.mapreduce related class-<KEYIN,VALUEIN,KEYOUT,VALUEOUT> //alt+/-double click"map"-delete a little: protected void map(Object key,Text value,Context context)throws IOException,InterruptedException{ //the variable is because the before we set //turn text to String,which backs String String line=value.toString(); String[] words=line.split(" ");//back to String num group for(int i=0;i<words.length;i++) {//there are lots of String,so we one to one //cuz our output is text,so turn it into String Text word=new Text(words[i]); //next we ouput the number 1,Intwritable IntWritable one=new IntWritable(1);//this value always =1 to any word context.write(word,one); }//for }//protected }//class map //<hello,1><hello,1> =>guibing <hello,<1,1,1,1,1,1> > class MyReducer extends Reducer<Text,Writable,Text,Writable>{ //r-reduce method-double clicks //and we leave: protected void reduce(Text keyIn,Iterable<IntWritable> valueIn,Context context)throws IOException,InterruptedException{ //Iterable rongqi,it is used to put like<1,1,1,1,1,1,1> //so next we need to count the <1,1,1,1,1,1,1> int sum=0; //zeng qiang de for for(IntWritable n : valueIn) { sum+=n.get(); }//for IntWritable result=new IntWritable(sum); context.write(keyIn,result); }//protected }//class reduce