协同过滤服务+源代码+文档说明

package utils; import com.mada.pojo.*; import org.apache.http.NameValuePair; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.safety.Whitelist; import org.jsoup.select.Elements; import javax.script.ScriptException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.List; /* * 爬取豆瓣读书需要用到的公共方法 * * * */ public class bookCommons { public static author authorGet(String authorUrl, CloseableHttpClient httpClient) throws IOException { int aId; String name; String sex = ""; String birth = ""; String area = ""; String aImagePath = ""; String aboutAuthor = ""; aId = Integer.parseInt(authorUrl.substring(authorUrl.lastIndexOf("/", authorUrl.length() - 2) + 1, authorUrl.length() - 1)); CloseableHttpResponse authorResponse; authorResponse = commonUtils.sendGet(authorUrl, httpClient); // bookResponse=commonUtils.sendGet("https://book.douban.com/subject/1084336/",httpClient); Document docAuthorTemp = Jsoup.parse(EntityUtils.toString(authorResponse.getEntity(), "UTF-8")); //写入文件 //writeStringToDisk.writeHtml(doc.toString(), "1"); Element info = docAuthorTemp.getElementById("headline");//作者头部信息节点 try { aImagePath = info.select("div.pic").get(0).getElementsByTag("img").get(0).attr("src"); } catch (NullPointerException e) { aImagePath = ""; System.out.println("找不到头像" + authorUrl); } name = docAuthorTemp.getElementById("content").getElementsByTag("h1").get(0).text(); Elements infoEntityList = info.getElementsByTag("li"); for (Element infoEntity : infoEntityList) { String infoEntityString = infoEntity.text(); if (infoEntityString.contains("性别")) { sex = infoEntityString.split(":")[1]; } else if (infoEntityString.contains("出生日期") || infoEntityString.contains("生卒日期")) { birth = infoEntityString.split(":")[1]; } else if (infoEntityString.contains("出生地") || infoEntityString.contains("国家/地区")) { area = infoEntityString.split(":")[1]; } } Element intro = docAuthorTemp.getElementById("intro");//作者简介信息节点 if (intro.select("span[class='all hidden']").size() != 0) { aboutAuthor = Jsoup.clean(intro.select("span[class='all hidden']").get(0).html(), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false)); } else { aboutAuthor = Jsoup.clean(intro.select("div[class='bd']").get(0).html(), "", Whitelist.none(), new Document.OutputSettings().prettyPrint(false)); } author authorEntity = new author(); authorEntity.setAId(aId); authorEntity.setName(name); authorEntity.setSex(sex); authorEntity.setBirth(birth); authorEntity.setArea(area); authorEntity.setAImagePath(aImagePath); authorEntity.setAboutAuthor(aboutAuthor); // System.out.println("aId:" + aId); //// System.out.println("name:" + name); //// System.out.println("sex:" + sex); //// System.out.println("birth:" + birth); //// System.out.println("area:" + area); //// System.out.println("aImagePath:" + aImagePath); //// System.out.println("aboutAuthor:" + aboutAuthor); return authorEntity; } public static book bookGet(String bookUrl, CloseableHttpClient httpClient) throws URISyntaxException, IOException, ScriptException { CloseableHttpResponse bookResponse, authorResponse; int bId; String isbn = ""; String name = ""; String coverPath = ""; String authorName = ""; String publishingHouse = ""; String publishingYear = ""; String labels = ""; String briefIntroduction = "这本书还没有介绍！"; List<String> authorUrlList = new ArrayList<>(); List<String> recommonedUrlList = new ArrayList<>(); float score; int pointNumber; bId = Integer.parseInt(bookUrl.substring(bookUrl.lastIndexOf("/", bookUrl.length() - 2) + 1, bookUrl.length() - 1)); //bookUrl="https://book.douban.com/subject/1084336/"; bookResponse = commonUtils.sendGet(bookUrl, httpClient); // bookResponse=commonUtils.sendGet("https://book.douban.com/subject/1084336/",httpClient); Document docBookTemp = Jsoup.parse(EntityUtils.toString(bookResponse.getEntity(), "UTF-8")); String debug = docBookTemp.toString(); // writeStringToDisk.writeHtml(debug, "book" + bId); try { score = Float.parseFloat(docBookTemp.select("#interest_sectl div.rating_self.clearfix strong").get(0).text().trim()); pointNumber = Integer.parseInt(docBookTemp.select("#interest_sectl > div > div.rating_self.clearfix > div > div.rating_sum > span > a > span").get(0).text().trim()); } catch (Exception e) { score = 0; pointNumber = 0; // System.out.println("书籍信息不足：" + bookUrl); // System.out.println(e); // return null; } Element mainpic = docBookTemp.getElementById("mainpic");//书籍图片信息节点 Element info = docBookTemp.getElementById("info");//书籍信息节点 coverPath = mainpic.child(0).attr("href");//书籍封面url; name = docBookTemp.select("#wrapper>h1").text(); //处理作者信息 //可能会出现多个作者，有的是作者的直接页面，有的不是直接的作何页面，而是作者的search， Elements authorUrls = info.getElementsByTag("a");//id为info的节点下第一个a标签必为作者标签，不过当链接中有search时不是直接的作者页面，需再处理，只有链接中含有author时才是直接的作者页面 boolean flag = false;//记录是否搜索到作者链接 URI base = new URI(bookUrl);//处理链接中的相对路径 for (Element authorE : authorUrls) { String authorUrl = authorE.attr("href"); if (authorUrl.contains("author"))//直接是作者页面 { flag = true; authorUrlList.add(base.resolve(authorUrl).toString()); } else if (false && authorUrl.contains("search"))//得到搜索页面 { authorResponse = commonUtils.sendGet(base.resolve(authorUrl).toString(), null); if (authorResponse.getStatusLine().getStatusCode() == 302) { String redirectUrl = authorResponse.getHeaders("Location")[0].getValue().replace(" ", ""); authorResponse = commonUtils.sendGet(redirectUrl, null); } Document authorSearchTemp = Jsoup.parse(EntityUtils.toString(authorResponse.getEntity(), "UTF-8")); //解密 Elements scripts = authorSearchTemp.getElementsByTag("script"); String windowDATA = ""; for (Element script : scripts) { if (script.toString().contains("window.__DATA__")) { windowDATA = script.toString().replaceAll("\\s+", "").replaceAll("<scripttype=\"text/javascript\">window.__DATA__=", "").replaceAll("window.__USER__=\\{\\}</script>", ""); } } String[] searchResultArray = {}; try { List<Nam