开发自己的搜索引擎

package com.luceneheritrixbook.extractor; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.htmlparser.Parser; import com.luceneheritrixbook.extractor.com163.Extract163Moblie; import com.luceneheritrixbook.extractor.pconline.mobile.ExtractPconlineMoblie; public abstract class Extractor { protected static final String NEWLINE = "\r\n"; /** * ��ʾ���н������·�� */ private String outputPath = ""; /** * ��ʾ��ǰ���ڱ�������ļ� */ private String inuputFilePath; /** * ��ʾ��ǰ���б�ץȡ����ҳ�ľ����Ŀ¼ ��Heritrix��mirrorĿ¼��ʾ */ private String mirrorDir = ""; /** * ���ڴ�ű�������IJ�ڵ�ͼƬ��Ŀ¼ */ private String imageDir = ""; /** * HTMLParser��ʵ�� */ private Parser parser; /** * ��ͼƬ·�����й�ϣ���㷨���������MD5�㷨 */ protected static final String HASH_ALGORITHM = "md5"; /** * �ָ�� */ public static final String SEPARATOR = "======================"; /** * װ����Ҫ����ҳ�ļ� * */ public void loadFile(String path) { try { parser = new Parser(path); inuputFilePath = path; parser.setEncoding("GBK"); } catch (Exception e) { e.printStackTrace(); } } /** * ��ȡ����·�� */ public String getOutputPath() { return outputPath; } /** * ��������·����ͨ���ڳ�ʼ��Extractorʱ��Ӧ���� */ public void setOutputPath(String outputPath) { this.outputPath = outputPath; } public Parser getParser() { return parser; } /** * ʹ������4ƥ�䲢�����ҳ�е��ַ� */ protected String getProp(String pattern, String match, int index) { Pattern sp = Pattern.compile(pattern); Matcher matcher = sp.matcher(match); while (matcher.find()) { return matcher.group(index); } return null; } /** * ���󷽷������ڹ�����ʵ�֡� �书����Ҫ�ǽ�����ҳ�ļ� ����Ʒ��Ϣ���浽 * */ public abstract void extract(); /** * ��ȡ���ڴ�����ļ���·��手机 */ public String getInuputFilePath() { return inuputFilePath; } /** * ��mirrorĿ¼�¿����ļ������趨��ͼƬĿ¼ * �÷���������Ҫ���ı� */ protected boolean copyImage(String image_url, String new_image_file) { String dirs = image_url.substring(7); try { // instance the File as file_in and file_out File file_in = new File(new File(mirrorDir), dirs); if (file_in == null || !file_in.exists()) { file_in = new File("e:\\noimage.jpg"); } File file_out = new File(new File(imageDir), new_image_file); FileInputStream in1 = new FileInputStream(file_in); FileOutputStream out1 = new FileOutputStream(file_out); byte[] bytes = new byte[1024]; int c; while ((c = in1.read(bytes)) != -1) out1.write(bytes, 0, c); // close in1.close(); out1.close(); return (true); // if success then return true } catch (Exception e) { e.printStackTrace(); return (false); // if fail then return false } } public String getImageDir() { return imageDir; } public void setImageDir(String imageDir) { this.imageDir = imageDir; } public String getMirrorDir() { return mirrorDir; } public void setMirrorDir(String mirrorDir) { this.mirrorDir = mirrorDir; } public void setInuputFilePath(String inuputFilePath) { this.inuputFilePath = inuputFilePath; } // public static void main(String[] args) throws Exception { // // Extractor extractor = new Extract163Moblie(); // extractor.setOutputPath("c:\\product\\test\\mobile\\"); // extractor.setImageDir("c:\\product\\test\\image\\"); // extractor.setMirrorDir("F:\\data\\163�ֻ�\\mirror\\"); // // traverse(extractor, new File("F:\\data\\163�ֻ�\\mirror\\mobile.163.com\\0011\\product\\0011000B\\product")); // System.out.println(count); // // } static int count = 0; public static void main(String[] args) throws Exception { Extractor extractor = new ExtractPconlineMoblie(); extractor.setOutputPath("c:\\product\\mobile\\"); extractor.setImageDir("c:\\product\\image\\"); extractor.setMirrorDir("F:\\"); traverse(extractor, new File("F:\\data\\product.pconline.com.cn\\product")); System.out.println(count); } public static void traverse(Extractor extractor, File path) throws Exception { if (path == null) { return; } if (path.isDirectory()) { String[] files = path.list(); for (int i = 0; i < files.length; i++) { traverse(extractor, new File(path, files[i])); } } else { if (path.getAbsolutePath().endsWith(".html") && path.getAbsolutePath().indexOf("_") == -1) { System.out.println(path); count++; extractor.loadFile(path.getAbsolutePath()); extractor.extract(); } } } }