基于SpringBoot快速开发的爬虫项目源码+项目使用说明+sql数据库.zip

package com.aposs.box.spider.utils.net; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.conn.ClientConnectionManager; import org.apache.http.conn.scheme.Scheme; import org.apache.http.conn.scheme.SchemeRegistry; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLSocketFactory; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; import java.io.IOException; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.List; import java.util.Map; /** * @author Aaron * @date 2020/12/21 */ public class HttpClientUtil { public static void main(String[] args) { String url = "http://aposs.cn:8083/network/v1/ip"; HttpHost proxy = new HttpHost("213.110.230.247", 8080, "http"); System.out.println(doGetHttp(url, proxy)); // String url1 = "https://bendi.news.163.com/guangdong/special/04178NDR/news_sz.js"; // HttpHost proxy1 = new HttpHost("213.110.230.247", 8080, "https"); // System.out.println(doGetHttps(url1, proxy1)); } public static String doGetHttp(String url, HttpHost proxy) { HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); if (proxy != null) httpClientBuilder.setProxy(proxy); CloseableHttpClient httpClient = httpClientBuilder.build(); // 创建Get请求 HttpGet httpGet = new HttpGet(url); // 响应模型 CloseableHttpResponse response = null; String respString = ""; try { // 由客户端执行(发送)Get请求 response = httpClient.execute(httpGet); // 从响应模型中获取响应实体 HttpEntity responseEntity = response.getEntity(); // System.out.println("响应状态为:" + response.getStatusLine()); if (responseEntity != null) { // System.out.println("响应内容长度为:" + responseEntity.getContentLength()); // System.out.println("响应内容为:" + EntityUtils.toString(responseEntity)); respString = EntityUtils.toString(responseEntity); } } catch (IOException e) { e.printStackTrace(); } finally { try { // 释放资源 if (httpClient != null) httpClient.close(); if (response != null) response.close(); } catch (IOException e) { e.printStackTrace(); } } return respString; } public static String doGetHttps(String url, HttpHost proxy) { CloseableHttpClient httpClient = null; // 创建Get请求 HttpGet httpGet = new HttpGet(url); // 响应模型 HttpResponse response = null; String respString = ""; try { httpClient = getHttpsClient(proxy); // 由客户端执行(发送)Get请求 response = httpClient.execute(httpGet); // 从响应模型中获取响应实体 HttpEntity responseEntity = response.getEntity(); // System.out.println("响应状态为:" + response.getStatusLine()); if (responseEntity != null) { // System.out.println("响应内容长度为:" + responseEntity.getContentLength()); // System.out.println("响应内容为:" + EntityUtils.toString(responseEntity)); respString = EntityUtils.toString(responseEntity); } } catch (Exception e) { e.printStackTrace(); } finally { try { // 释放资源 if (httpClient != null) httpClient.close(); // if (response != null) response.close(); } catch (IOException e) { e.printStackTrace(); } } return respString; } public String doPost(String url, Map<String, String> map, String charset) { HttpClient httpClient = null; HttpPost httpPost = null; String result = null; try { httpClient = new SSLClient(); httpPost = new HttpPost(url); //设置参数 List<NameValuePair> list = new ArrayList<>(); map.forEach((key, value) -> list.add(new BasicNameValuePair(key, value))); if (list.size() > 0) { UrlEncodedFormEntity entity = new UrlEncodedFormEntity(list, charset); entity.setContentType("application/json"); httpPost.setHeader("Accept", "application/json"); httpPost.setHeader("Content-type", "application/json;charset=utf-8"); httpPost.setEntity(entity); } HttpResponse response = httpClient.execute(httpPost); if (response != null) { HttpEntity resEntity = response.getEntity(); if (resEntity != null) { result = EntityUtils.toString(resEntity, charset); } } } catch (Exception ex) { ex.printStackTrace(); } return result; } private static CloseableHttpClient getHttpsClient(HttpHost proxy) throws KeyManagementException, NoSuchAlgorithmException { //这里设置客户端不检测服务器ssl证书 X509TrustManager x509mgr = new X509TrustManager() { public void checkClientTrusted(X509Certificate[] xcs, String string) { } public void checkServerTrusted(X509Certificate[] xcs, String string) { } public X509Certificate[] getAcceptedIssuers() { return null; } }; SSLContext sslContext = SSLContext.getInstance("TLS"); sslContext.init(null, new TrustManager[]{x509mgr}, null); SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext, SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER); RequestConfig.Builder builder = RequestConfig.custom(); if (proxy != null) builder.setProxy(proxy); RequestConfig defaultRequestConfig = builder.build(); CloseableHttpClient httpclient = HttpClients.custom().setSSLSocketFactory(sslsf).setDefaultRequestConfig(defaultRequestConfig).build(); return httpclient; } static class SSLClient extends DefaultHttpClient { public SSLClient() throws Exception { super(); //传输协议需要根据自己的判断 SSLContext ctx = SSLContext.getInstance("TLS"); X509TrustManager tm = new X509TrustManager() { @Override public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { } @Override public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { } @Override publ