高级爬虫进阶：HtmlUnit+多线线程+消息队列快速抓取大量信息数据

package com.gtja.util; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import com.gargoylesoftware.htmlunit.BrowserVersion; import com.gargoylesoftware.htmlunit.ProxyConfig; import com.gargoylesoftware.htmlunit.WebClient; import com.gargoylesoftware.htmlunit.html.DomNode; import com.gargoylesoftware.htmlunit.html.HtmlDivision; import com.gargoylesoftware.htmlunit.html.HtmlPage; import com.gargoylesoftware.htmlunit.html.HtmlTable; import com.gargoylesoftware.htmlunit.html.HtmlTableCell; import com.gargoylesoftware.htmlunit.html.HtmlTableRow; import com.gargoylesoftware.htmlunit.javascript.host.dom.Document; import com.gargoylesoftware.htmlunit.javascript.host.dom.Node; import com.gtja.bean.QsztBean; import com.gtja.bean.SmjjBean; import com.gtja.bean.SmjjGlrBean; public class PageMessage { /** * 私募基金管理人分类公示 * @param smjglr * @return */ public static SmjjGlrBean SmjjGlrMessage(SmjjGlrBean smjglr){ // TODO Auto-generated method stub WebClient wc = new WebClient(BrowserVersion.FIREFOX_38); wc.getOptions().setUseInsecureSSL(true); wc.getOptions().setJavaScriptEnabled(false); // 启用JS解释器，默认为true wc.getOptions().setCssEnabled(false); // 禁用css支持 wc.getOptions().setThrowExceptionOnScriptError(false); // js运行错误时，是否抛出异常 wc.getOptions().setTimeout(10000); // 设置连接超时时间 ，这里是10S。如果为0，则无限期等待 wc.getOptions().setDoNotTrackEnabled(false); //服务器代理 ProxyConfig pro =wc.getOptions().getProxyConfig(); pro.setProxyHost("10.88.100.100"); pro.setProxyPort(8080); HtmlPage page = null; try { page = wc.getPage("http://gs.amac.org.cn/amac-infodisc/res/pof/manager/"+smjglr.getUrl()+""); String url = page.getUrl().toString(); smjglr.setUrl(url); //分析最终的页面信息 HtmlTable table=(HtmlTable) page.getDocumentElement().getElementsByTagName("table").get(0); //基金管理人全称 HtmlDivision allName = (HtmlDivision) page.getElementById("complaint1"); smjglr.setFullName(Template.trimVal(allName.asText()).trim()); for(HtmlTableRow row:table.getRows()){ // 行 for(HtmlTableCell cell:row.getCells()){ // 列 if("机构诚信信息:".equals(cell.asText())){ String cxm = cell.getNextElementSibling().asText(); smjglr.setCxm(Template.trimVal(cxm)); }else if("基金管理人全称(英文):".equals(cell.asText())){ String enName = cell.getNextElementSibling().asText(); smjglr.setEnName(Template.trimVal(enName)); }else if("登记编号:".equals(cell.asText())){ String regiNumber = cell.getNextElementSibling().asText(); smjglr.setRegiNumber(Template.trimVal(regiNumber)); }else if("组织机构代码:".equals(cell.asText())){ String meCode = cell.getNextElementSibling().asText(); smjglr.setMeCode(Template.trimVal(meCode)); }else if("登记时间:".equals(cell.asText())){ String regiDate = cell.getNextElementSibling().asText(); smjglr.setRegiDate(Template.trimVal(regiDate)); }else if("成立时间:".equals(cell.asText())){ String creDate = cell.getNextElementSibling().asText(); smjglr.setCreDate(Template.trimVal(creDate)); }else if("注册地址:".equals(cell.asText())){ String regiAddress = cell.getNextElementSibling().asText(); smjglr.setRegiAddress(Template.trimVal(regiAddress)); }else if("办公地址:".equals(cell.asText())){ String offAddress = cell.getNextElementSibling().asText(); smjglr.setOffAddress(Template.trimVal(offAddress)); }else if("注册资本(万元)(人民币):".equals(cell.asText())){ String regiCapital = cell.getNextElementSibling().asText(); smjglr.setRegiCapital(Template.trimVal(regiCapital)); }else if("实缴资本(万元)(人民币):".equals(cell.asText())){ String paiCapital = cell.getNextElementSibling().asText(); smjglr.setPaiCapital(Template.trimVal(paiCapital)); }else if("企业性质:".equals(cell.asText())){ String natPrise = cell.getNextElementSibling().asText(); smjglr.setNatPrise(Template.trimVal(natPrise)); }else if("注册资本实缴比例:".equals(cell.asText())){ String proRegiCapital = cell.getNextElementSibling().asText(); smjglr.setProRegiCapital(Template.trimVal(proRegiCapital)); }else if("机构类型:".equals(cell.asText())){ String meanType = cell.getNextElementSibling().asText(); smjglr.setMeanType(Template.trimVal(meanType)); }else if("业务类型:".equals(cell.asText())){ String busiType = cell.getNextElementSibling().asText(); smjglr.setBusiType(Template.trimVal(busiType)); }else if("员工人数:".equals(cell.asText())){ String employNumber = cell.getNextElementSibling().asText(); smjglr.setEmployNumber(Template.trimVal(employNumber)); }else if("机构网址:".equals(cell.asText())){ String insWebsite = cell.getNextElementSibling().asText(); smjglr.setInsWebsite(Template.trimVal(insWebsite)); }else if("是否为会员:".equals(cell.asText())){ String isMember = cell.getNextElementSibling().asText(); smjglr.setIsMember(Template.trimVal(isMember)); }else if("当前会员类型:".equals(cell.asText())){ String memberType = cell.getNextElementSibling().asText(); smjglr.setMemberType(Template.trimVal(memberType)); }else if("入会时间:".equals(cell.asText())){ String admissTime = cell.getNextElementSibling().asText(); smjglr.setAdmissTime(Template.trimVal(admissTime)); }else if("法律意见书状态:".equals(cell.asText())){ String legionStatus = cell.getNextElementSibling().asText(); smjglr.setLegionStatus(Template.trimVal(legionStatus)); }else if("律师事务所名称:".equals(cell.asText())){ String swsName = cell.getNextElementSibling().asText(); smjglr.setSwsName(Template.trimVal(swsName)); }else if("律师姓名:".equals(cell.asText())){ String lsName = cell.getNextElementSibling().asText(); smjglr.setLsName(Template.trimVal(lsName)); }else if("法定代表人/执行事务合伙人(委派代表)姓名:".equals(cell.asText())){ String legalRepre = cell.getNextElementSibling().asText(); smjglr.setLegalRepre(Template.trimVal(legalRepre)); }else if("是否有从业资格:".equals(cell.asText())){ String isQualifition = cell.getNextElementSibling().asText(); smjglr.setIsQualifition(Template.trimVal(isQualifition)); }else if("资格取得方式:".equals(cell.asText())){ String eligMethod = cell.getNextElementSibling().asText(); smjglr.setEligMethod(Template.trimVal(eligMethod)); }else if("机构信息最后更新时间:".equals(cell.asText())){ String laupTime = cell.getNextElementSibling().asText(); smjglr.setLaupTime(Template.trimVal(laupTime)); }else if("特别提示信息:".equals(cell.asText())){ String specInformation = cell.getNextElementSibling().asT