Tf-idf.zip_tfidf

package tfidf; import java.io.*; import java.util.*; import org.wltea.analyzer.lucene.IKAnalyzer; public class tfidf{ /** * @param args * */ private static ArrayList<String> FileList = new ArrayList<String>(); // the list of file //get list of file for the directory, including sub-directory of it public static List<String> readDirs(String filepath) throws FileNotFoundException, IOException { try { File file = new File(filepath); //判断是否为目录 if(!file.isDirectory()) { System.out.println("输入的[]"); System.out.println("filepath:" + file.getAbsolutePath()); } else { String[] flist = file.list();//获得file的文件 for(int i = 0; i < flist.length; i++) { File newfile = new File(filepath + "/" + flist[i]); if(!newfile.isDirectory()) { FileList.add(newfile.getAbsolutePath()); } else if(newfile.isDirectory()) //if file is a directory, call ReadDirs { readDirs(filepath + "/" + flist[i]); } } } }catch(FileNotFoundException e) { System.out.println(e.getMessage()); } return FileList; } //read file public static String readFile(String file) throws FileNotFoundException, IOException { StringBuffer strSb = new StringBuffer(); //String is constant， StringBuffer can be changed. InputStreamReader inStrR = new InputStreamReader(new FileInputStream(file), "gbk"); //byte streams to character streams BufferedReader br = new BufferedReader(inStrR); String line = br.readLine(); while(line != null){ strSb.append(line).append("\r\n"); line = br.readLine(); } return strSb.toString(); } public static ArrayList<String> Getwords(String file) throws FileNotFoundException, IOException { ArrayList<String> w = new ArrayList<String>(); BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8")); String line; while ((line = br.readLine()) != null) { String[] words = line.split(" "); for (String word : words) { if (word.trim().length() < 2) continue;//去空格 w.add(word); } } br.close(); return w; } //word segmentation public static ArrayList<String> cutWords(String file) throws IOException{ ArrayList<String> words = new ArrayList<String>(); String text = tfidf.readFile(file); IKAnalyzer analyzer = new IKAnalyzer();//分词 words = analyzer.split(text); return words; } //term frequency in a file, times for each word次数 public static HashMap<String, Integer> normalTF(ArrayList<String> cutwords){ HashMap<String, Integer> resTF = new HashMap<String, Integer>(); for(String word : cutwords){ if(resTF.get(word) == null){ resTF.put(word, 1);//未出现到词value为1，一旦出现已分配value，则+1 //System.out.println(word); } else{ resTF.put(word, resTF.get(word) + 1); //System.out.println(word.toString()); } } return resTF; } //term frequency in a file, frequency of each word频率 public static HashMap<String, Float> tf(ArrayList<String> cutwords){ HashMap<String, Float> resTF = new HashMap<String, Float>(); int wordLen = cutwords.size(); HashMap<String, Integer> intTF = tfidf.normalTF(cutwords); Iterator iter = intTF.entrySet().iterator(); //iterator for that get from TF每个词在此篇文章中出现到频率 while(iter.hasNext()){ Map.Entry entry = (Map.Entry)iter.next(); resTF.put(entry.getKey().toString(), Float.parseFloat(entry.getValue().toString()) / wordLen); //System.out.println(entry.getKey().toString() + " = "+ Float.parseFloat(entry.getValue().toString()) / wordLen); } return resTF; } //tf times for file public static HashMap<String, HashMap<String, Integer>> normalTFAllFiles(String dirc) throws IOException{ HashMap<String, HashMap<String, Integer>> allNormalTF = new HashMap<String, HashMap<String,Integer>>(); List<String> filelist = tfidf.readDirs(dirc); for(String file : filelist){ HashMap<String, Integer> dict = new HashMap<String, Integer>(); ArrayList<String> cutwords = tfidf.Getwords(file); //get cut word for one file dict = tfidf.normalTF(cutwords); allNormalTF.put(file, dict); } return allNormalTF; } //tf for all file public static HashMap<String,HashMap<String, Float>> tfAllFiles(String dirc) throws IOException{ HashMap<String, HashMap<String, Float>> allTF = new HashMap<String, HashMap<String, Float>>(); List<String> filelist = tfidf.readDirs(dirc); for(String file : filelist){ HashMap<String, Float> dict = new HashMap<String, Float>(); ArrayList<String> cutwords = tfidf.Getwords(file); //get cut words for one file dict = tfidf.tf(cutwords);//每个词在这篇文章中出现到频率 allTF.put(file, dict); } return allTF;//总和词在他所在那篇文章中的频率 } public static HashMap<String, Float> idf(HashMap<String,HashMap<String, Float>> all_tf){ HashMap<String, Float> resIdf = new HashMap<String, Float>(); HashMap<String, Integer> dict = new HashMap<String, Integer>(); int docNum = FileList.size(); for(int i = 0; i < docNum; i++){ HashMap<String, Float> temp = all_tf.get(FileList.get(i)); Iterator iter = temp.entrySet().iterator(); while(iter.hasNext()){ Map.Entry entry = (Map.Entry)iter.next(); String word = entry.getKey().toString(); if(dict.get(word) == null){ dict.put(word, 1); }else { dict.put(word, dict.get(word) + 1); } } }//统计某一个词出现在整个文档中的次数 //System.out.println("IDF for every word is:"); Iterator iter_dict = dict.entrySet().iterator(); while(iter_dict.hasNext()){ Map.Entry entry = (Map.Entry)iter_dict.next(); float value = (float)Math.log(docNum / Float.parseFloat(entry.getValue().toString())); resIdf.put(entry.getKey().toString(), value); //System.out.println(entry.getKey().toString() + " = " + value); }//每个词在整个文档中占的频率 return resIdf; } public static void tf_idf(HashMap<String,HashMap<String, Float>> all_tf,HashMap<String, Float> idfs){ HashMap<String, HashMap<String, Float>> resTfIdf = new HashMap<String, HashMap<String, Float>>(); int docNum = FileList.size(); for(int i = 0; i < docNum; i++){ String filepath = FileList.get(i); HashMap<String, Float> tfidf = new HashMap<String, Float>(); HashMap<String, Float> temp = all_tf.get(filepath); Iterator iter = temp.entrySet().iterator(); while(iter.h