Webmagic爬取数据导入到ES

package com.fendo.service.impl; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.text.Text; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import com.fendo.common.Page; import com.fendo.entity.CsdnBlog; import com.fendo.entity.News; import com.fendo.mapper.CsdnBlogMapper; import com.fendo.service.CsdnBlogService; import com.google.gson.JsonObject; import io.searchbox.client.JestClient; import io.searchbox.core.*; import io.searchbox.core.SearchResult.Hit; import io.searchbox.indices.CreateIndex; import io.searchbox.indices.DeleteIndex; @Service public class CsdnBlogServiceImpl implements CsdnBlogService{ public static final Logger LOGGER = Logger.getLogger(CsdnBlogServiceImpl.class); @Autowired private JestClient jestClient; @Autowired private CsdnBlogMapper CsdnBlogMapper; private int num = 100000; @Override public void save(CsdnBlog csdnBlog) { try { System.out.println("jinlai"); CsdnBlogMapper.insertSelective(csdnBlog); LOGGER.info("插入数据成功!!!"); } catch (Exception e) { LOGGER.error(e); } } @Override public void get() { CsdnBlogMapper.selectByPrimaryKey(1); List<CsdnBlog> csdnlist =CsdnBlogMapper.selectAll(); System.out.println(csdnlist.size()); } @Override public void createSearchIndex() { long start = System.currentTimeMillis(); try { // 如果索引存在,删除索引 Bulk bulk = new Bulk.Builder().defaultIndex("article") .defaultType("article") .addAction(Arrays.asList( new Index.Builder("").build(), new Index.Builder("").build(), new Index.Builder("").build(), new Index.Builder("").build() )).build(); jestClient.execute(bulk); } catch (Exception e) { e.printStackTrace(); } long end = System.currentTimeMillis(); System.out.println("创建索引时间:数据量是 " + num + "记录,共用时间 -->> " + (end - start) + " 毫秒"); } @Override public List<CsdnBlog> search(String param) throws Exception { long start = System.currentTimeMillis(); Map<String, Object> maps = search("all","java","",1,15); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.size(15); searchSourceBuilder.from(0); String query = searchSourceBuilder.toString(); // System.out.println(query); Search search = new Search.Builder(query) .addIndex("csdnblog") .addType("article") .build(); SearchResult result = jestClient.execute(search); List<Hit<CsdnBlog, Void>> hits = result.getHits(CsdnBlog.class); System.out.println("Size:" + hits.size()); for (Hit<CsdnBlog, Void> hit : hits) { CsdnBlog csdnBlog = hit.source; System.out.println(hit.highlight); System.out.println(csdnBlog.getId()); } long end = System.currentTimeMillis(); System.out.println("在" + num + "条记录中,搜索新闻,共用时间 -->> " + (end - start) + " 毫秒"); return result.getSourceAsObjectList(CsdnBlog.class); } /** * 检索 * * @param field * @param queryString * @param older * @param pageNumber * @param pageSize * @return * @throws Exception */ public Map<String, Object> search(String field, String queryString, String older, int pageNumber, int pageSize) throws Exception { List<CsdnBlog> csdnblogs = new ArrayList<CsdnBlog>(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); // 构建查询 if ("all".equals(field)) { searchSourceBuilder.query(QueryBuilders.queryStringQuery(queryString)); } else { searchSourceBuilder.query(QueryBuilders.termQuery(field, queryString)); // 设置排序 // searchSourceBuilder.sort(field, "asc".equals(older) ? SortOrder.ASC : SortOrder.DESC); // 设置排序字段及排序顺序 } // 设置高亮字段 HighlightBuilder highlightBuilder = new HighlightBuilder().field("*").requireFieldMatch(false); highlightBuilder.field("titles");//高亮title highlightBuilder.preTags("<em>").postTags("</em>");//高亮标签 highlightBuilder.fragmentSize(200);//高亮内容长度 searchSourceBuilder.highlighter(highlightBuilder); searchSourceBuilder.explain(true); //设置是否按查询匹配度排序 // 设置分页 searchSourceBuilder.from((pageNumber - 1) * pageSize);//设置起始页 searchSourceBuilder.size(pageSize);//设置页大小 System.out.println(searchSourceBuilder.highlighter()); Search search = new Search.Builder(searchSourceBuilder.toString()) .addIndex("csdnblog")// 索引名称 .build(); SearchResult result = jestClient.execute(search); System.out.println("本次查询共查到："+result.getTotal()+"篇文章！"); List<Hit<CsdnBlog,Void>> hitss = result.getHits(CsdnBlog.class); System.out.println("---------"+hitss); // for (Hit<CsdnBlog, Void> hit : hitss) { // CsdnBlog source = hit.source; // // //获取高亮后的内容 // Map<String, List<String>> highlight = hit.highlight; // List<String> titlelist = highlight.get("titles");//高亮后的title // if(titlelist!=null){ // source.setTitles(titlelist.get(0)); // } // List<String> contentlist = highlight.get("content");//高亮后的content // if(contentlist!=null){ // source.setContent(contentlist.get(0)); // } // System.out.println("标题："+source.getTitles()); // System.out.println("内容："+source.getContent()); // System.out.println("url："+source.getUrl()); // } // 自动解析 // parseSearchResult(articles, result); // 手动解析 JsonObject jsonObject = result.getJsonObject(); JsonObject hitsobject = jsonObject.getAsJsonObject("hits"); System.out.println(hitsobject); long took = jsonObject.get("took").getAsLong(); long total = hitsobject.get("total").getAsLong(); List<SearchResult.Hit<CsdnBlog, Void>> hits = result.getHits(CsdnBlog.class); System.out.println(hitsobject); for (SearchResult.Hit<CsdnBlog, Void> hit : hits) { CsdnBlog source = hit.source; //获取高亮后的内容 System.out.println("-------------"+hit.source); Map<String, List<String>> highlight = hit.highlight; System.out.println("-----"+hit.highlight); System.out.println("-----------"+highlight); List<String> titlelist = highlight.get("titles");//高亮后的