鄙人自制csdn博客爬虫

package cn.crawler.lucene; import java.io.File; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Version; import cn.crawler.lucene.bean.BlogBean; public class SearchIndexs { private static String indexPath = "./data"; static Log log = LogFactory.getLog("log4j"); /** * 显示多少条 */ private static int pages=10; /** * 内容摘要长度<br>单位：字 */ private static int content_lenght=300; private SearchIndexs() {} public static void main(String[] args) throws Exception { log.info(search("csdn")); } @SuppressWarnings("deprecation") public synchronized static List<BlogBean> search(String keyword){ List<BlogBean> list=new ArrayList<>(); try { IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(indexPath))); IndexSearcher searcher = new IndexSearcher(reader); Analyzer analyzer = new SmartChineseAnalyzer(Version.LUCENE_40); String[] fields={"url","title","content"}; QueryParser parser = new MultiFieldQueryParser(Version.LUCENE_40,fields, analyzer); Query query = parser.parse(keyword); //高亮显示 Date start = new Date(); TopDocs results = searcher.search(query, pages); Date end = new Date(); log.info("检索完成，用时" + ((end.getTime() - start.getTime())/1000.0) + "毫秒"); ScoreDoc[] hits = results.scoreDocs; Document doc=null; for(int i=0;hits!=null&&i<hits.length;i++){ doc=searcher.doc(hits[i].doc); list.add(BlogBean.toBolgBean(doc)); } shortContent(list); highLighter(list,keyword); reader.close(); analyzer.close(); } catch (Exception e) { e.printStackTrace(); log.error("检索"+keyword+"出现异常："+e.getMessage()); return null; } return list.size()==0?null:list; } /** * 高亮显示 * @param list * @param keyword */ private static void highLighter(List<BlogBean> list,String keyword) { if(list==null) return ; for(BlogBean bBean:list){ bBean.setContent(bBean.getContent().replace(keyword,"<font color='red' >"+keyword+"</font>")); bBean.setTitle(bBean.getTitle().replace(keyword,"<font color='red' >"+keyword+"</font>")); } } /** * 内容摘要 * @param list */ private static void shortContent(List<BlogBean> list) { if(list==null) return ; for(BlogBean bBean:list){ if(bBean.getContent().length()<content_lenght) continue; bBean.setContent(bBean.getContent().substring(0,content_lenght)); } } /** * java实现不区分大小写替换 * @param source * @param oldstring * @param newstring * @return */ public static String IgnoreCaseReplace(String source, String oldstring,String newstring) { Pattern p = Pattern.compile(oldstring, Pattern.CASE_INSENSITIVE); Matcher m = p.matcher(source); String ret = m.replaceAll(newstring); return ret; } }