java多线程代理IP池

package com.zxy.request; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; public class findIpProxy { public String findNewIpProxyContent(String addr, Integer port, Integer i) { CloseableHttpClient closeableHttpClient = HttpClients.createDefault(); HttpHost httpHost = null; if (addr != null && port != null) { httpHost = new HttpHost(addr, port); System.out.println("本次请求的IP为:" + addr+":"+port); } HttpGet httpGet = new HttpGet("http://www.xicidaili.com/nn/" + i + ""); if (addr != null && port != null) { RequestConfig config = RequestConfig.custom().setProxy(httpHost) .setConnectTimeout(5000).setSocketTimeout(5000).build(); httpGet.setConfig(config); } httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;" + "q=0.9,image/webp,*/*;q=0.8"); httpGet.setHeader("Accept-Encoding", "gzip, deflate, sdch"); httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.8"); httpGet.setHeader("Cache-Control", "no-cache"); httpGet.setHeader("Connection", "keep-alive"); httpGet.setHeader("Host", "www.xicidaili.com"); httpGet.setHeader("Pragma", "no-cache"); httpGet.setHeader("Upgrade-Insecure-Requests", "1"); httpGet.setHeader( "User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"); try { String responseContent; HttpResponse httpResponse = closeableHttpClient.execute(httpGet); responseContent = EntityUtils.toString(httpResponse.getEntity(), "utf-8"); if (closeableHttpClient != null) { closeableHttpClient.close(); } return responseContent; } catch (Exception e) { System.out.println("本轮网页返回为空,有废弃ip"); } return null; } public String findNewIpProxy_QuickDLContent(String addr, Integer port, Integer i) { CloseableHttpClient closeableHttpClient = HttpClients.createDefault(); HttpHost httpHost = null; if (addr != null && port != null ) { httpHost = new HttpHost(addr, port); System.out.println("本次请求的IP为:" + addr+":"+port); } HttpGet httpGet = new HttpGet("http://www.kuaidaili.com/free/inha/" + i + ""); if (addr != null && port != null) { RequestConfig config = RequestConfig.custom().setProxy(httpHost) .setConnectTimeout(5000).setSocketTimeout(5000).build(); httpGet.setConfig(config); } httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;" + "q=0.9,image/webp,*/*;q=0.8"); httpGet.setHeader("Accept-Encoding", "gzip, deflate, sdch"); httpGet.setHeader("Accept-Language", "zh-CN,zh;q=0.8"); httpGet.setHeader("Cache-Control", "no-cache"); httpGet.setHeader("Connection", "keep-alive"); httpGet.setHeader("Host", "www.kuaidaili.com"); httpGet.setHeader("Referer", "http://www.kuaidaili.com/free/inha/"+i+"/"); httpGet.setHeader("Pragma", "no-cache"); httpGet.setHeader("Upgrade-Insecure-Requests", "1"); httpGet.setHeader( "User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"); try { String responseContent; HttpResponse httpResponse = closeableHttpClient.execute(httpGet); responseContent = EntityUtils.toString(httpResponse.getEntity(), "utf-8"); if (closeableHttpClient != null) { closeableHttpClient.close(); } return responseContent; } catch (Exception e) { System.err.println("本轮网页返回为空,有废弃ip"); } return null; } }