基于Hadoop的文本相似度计算

package com.bjtu.process; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.GenericOptionsParser; import com.bjtu.tfid.AnalyseMapper; import com.bjtu.tfid.AnalyseReducer; /** * <code>AnayseProcess</code> is used to analyze text of files<br /> * it uses {@link com.bjtu.tfid.AnalyseMapper} and {@link com.bjtu.tfid.AnalyseReducer} to * analyze. * The data files is in "hdfs://localhost:9002/tmp/hello/". * And output path is "hdfs://localhost:9002/tmp/analyze/analyse-map-reduce" * The output file contains file name , words, and the number of words in this text. * * @author wuyadong & tangxu */ class AnalyseProcess { public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException { args = new String[]{"hdfs://localhost:9002/tmp/hello/", "hdfs://localhost:9002/tmp"}; Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args) .getRemainingArgs(); if (otherArgs.length != 2) { System.err.println("Usage: Tf-idf <in> <out>"); System.exit(2); } AnalyseProcess process = new AnalyseProcess(); process.analyze(conf, otherArgs[0],"hdfs://localhost:9002/tmp/analyze/analyse-map-reduce"); } /** * @throws IOException * @throws ClassNotFoundException * @throws InterruptedException */ public void analyze(Configuration conf, String inFileName, String outFileName) throws IOException, InterruptedException, ClassNotFoundException { FileSystem hdfs = FileSystem.get(conf); FileStatus p[] = hdfs.listStatus(new Path(inFileName)); Job job = new Job(conf, "analyze"); job.setJarByClass(AnalyseProcess.class); job.setMapperClass(AnalyseMapper.class); job.setReducerClass(AnalyseReducer.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setNumReduceTasks(p.length); FileInputFormat.addInputPath(job, new Path(inFileName)); FileOutputFormat.setOutputPath(job, new Path(outFileName)); job.waitForCompletion(true); } }