package com.miles.crawler.util;
import com.miles.crawler.SysConstant;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
/**
* Http发送请求工具类
*/
public class HttpClientUtils {
private final static Logger logger = LoggerFactory.getLogger(HttpClientUtils.class);
private final static String GET_METHOD = "GET";
private final static String POST_METHOD = "POST";
public static String sendGet(String url, Map<String, String> headers, Map<String, String> params) {
CloseableHttpClient client = HttpClients.createDefault();
StringBuilder reqUrl = new StringBuilder(url);
String result = "";
if (params != null && params.size() > 0) {
reqUrl.append("?");
for (Entry<String, String> param : params.entrySet()) {
reqUrl.append(param.getKey() + "=" + param.getValue() + "&");
}
url = reqUrl.subSequence(0, reqUrl.length() - 1).toString();
}
logger.debug("[url:" + url + ",method:" + GET_METHOD + "]");
HttpGet httpGet = new HttpGet(url);
logger.debug("Header\n");
if (headers != null && headers.size() > 0) {
for (Entry<String, String> header : headers.entrySet()) {
httpGet.addHeader(header.getKey(), header.getValue());
logger.debug(header.getKey() + " : " + header.getValue());
}
}
CloseableHttpResponse response = null;
try {
response = client.execute(httpGet);
if (response.getStatusLine().getStatusCode() == 200) {
HttpEntity entity = response.getEntity();
result = EntityUtils.toString(entity, SysConstant.DEFAULT_CHARSET);
}
} catch (IOException e) {
logger.error("网络请求出错,请检查原因");
} finally {
try {
if (response != null) {
response.close();
}
client.close();
} catch (IOException e) {
logger.error("网络关闭错误错,请检查原因");
}
}
return result;
}
public static String sendPost(String url, Map<String, String> headers, Map<String, String> params) {
CloseableHttpClient client = HttpClients.createDefault();
String result = "";
HttpPost httpPost = new HttpPost(url);
if (params != null && params.size() > 0) {
List<NameValuePair> paramList = new ArrayList<>();
for (Entry<String, String> param : params.entrySet()) {
paramList.add(new BasicNameValuePair(param.getKey(), param.getValue()));
}
logger.debug("[url: " + url + ",method: " + POST_METHOD + "]");
try {
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(paramList, SysConstant.DEFAULT_CHARSET);
httpPost.setEntity(entity);
} catch (UnsupportedEncodingException e) {
logger.error("不支持的编码");
}
if (headers != null && headers.size() > 0) {
logger.debug("Header\n");
if (headers != null && headers.size() > 0) {
for (Entry<String, String> header : headers.entrySet()) {
httpPost.addHeader(header.getKey(), header.getValue());
logger.debug(header.getKey() + " : " + header.getValue());
}
}
}
CloseableHttpResponse response = null;
try {
response = client.execute(httpPost);
HttpEntity entity = response.getEntity();
result = EntityUtils.toString(entity, SysConstant.DEFAULT_CHARSET);
} catch (IOException e) {
logger.error("网络请求出错,请检查原因");
} finally {
try {
if (response != null) {
response.close();
}
client.close();
} catch (IOException e) {
logger.error("网络关闭错误");
}
}
}
return result;
}
public static String senPostJson(String url, String json, Map<String, String> headers) {
CloseableHttpClient client = HttpClients.createDefault();
String result = "";
HttpPost httpPost = new HttpPost(url);
StringEntity stringEntity = new StringEntity(json, ContentType.APPLICATION_JSON);
httpPost.setEntity(stringEntity);
logger.debug("[url: " + url + ",method: " + POST_METHOD + ", json: " + json + "]");
if (headers != null && headers.size() > 0) {
logger.debug("Header\n");
if (headers != null && headers.size() > 0) {
for (Entry<String, String> header : headers.entrySet()) {
httpPost.addHeader(header.getKey(), header.getValue());
logger.debug(header.getKey() + " : " + header.getValue());
}
}
}
CloseableHttpResponse response = null;
try {
response = client.execute(httpPost);
HttpEntity entity = response.getEntity();
result = EntityUtils.toString(entity, SysConstant.DEFAULT_CHARSET);
} catch (IOException e) {
logger.error("网络请求出错,请检查原因");
} finally {
try {
if (response != null) {
response.close();
}
client.close();
} catch (IOException e) {
logger.error("网络关闭错误");
}
}
return result;
}
}
没有合适的资源?快使用搜索试试~ 我知道了~
给予SpringMVC的java爬虫Demo.zip
共71个文件
xml:60个
java:9个
yml:1个
1.该资源内容由用户上传,如若侵权请联系客服进行举报
2.虚拟产品一经售出概不退款(资源遇到问题,请及时私信上传者)
2.虚拟产品一经售出概不退款(资源遇到问题,请及时私信上传者)
版权申诉
0 下载量 111 浏览量
2024-03-08
11:43:17
上传
评论
收藏 35KB ZIP 举报
温馨提示
给予SpringMVC的java爬虫Demo.zip
资源推荐
资源详情
资源评论
收起资源包目录
给予SpringMVC的java爬虫Demo.zip (71个子文件)
SJT-code
crawler.iml 7KB
pom.xml 3KB
src
main
resources
application.yml 274B
java
com
miles
crawler
controller
CrawlerController.java 2KB
dao
Impl
GoodsInfoDaoImpl.java 824B
GoodsInfoDao.java 248B
beans
GoodsInfo.java 1KB
service
Impl
CrawlerServiceImpl.java 2KB
CrawlerService.java 327B
App.java 600B
SysConstant.java 733B
util
HttpClientUtils.java 6KB
.idea
dataSources
2e15f187-e5be-4d94-be22-460a86101226.xml 2KB
uiDesigner.xml 9KB
libraries
Maven__org_slf4j_jul_to_slf4j_1_7_25.xml 522B
Maven__org_ow2_asm_asm_5_0_3.xml 460B
Maven__org_hamcrest_hamcrest_library_1_3.xml 541B
Maven__org_hamcrest_hamcrest_core_1_3.xml 520B
Maven__ch_qos_logback_logback_core_1_1_11.xml 542B
Maven__com_alibaba_fastjson_1_2_34.xml 502B
Maven__com_google_code_findbugs_jsr305_1_3_9.xml 533B
Maven__org_apache_tomcat_tomcat_juli_8_5_15.xml 547B
Maven__org_slf4j_log4j_over_slf4j_1_7_25.xml 550B
Maven__org_springframework_spring_test_4_3_9_RELEASE.xml 604B
Maven__org_springframework_spring_context_4_3_9_RELEASE.xml 625B
Maven__org_apache_httpcomponents_httpclient_4_5_3.xml 565B
Maven__org_springframework_boot_spring_boot_test_autoconfigure_1_5_4_RELEASE.xml 757B
Maven__org_assertj_assertj_core_2_6_0.xml 523B
Maven__org_springframework_boot_spring_boot_test_1_5_4_RELEASE.xml 659B
Maven__org_springframework_spring_aop_4_3_9_RELEASE.xml 597B
Maven__org_springframework_spring_jdbc_4_3_9_RELEASE.xml 604B
Maven__com_google_errorprone_error_prone_annotations_2_0_18.xml 647B
Maven__com_google_j2objc_j2objc_annotations_1_1.xml 575B
Maven__org_codehaus_mojo_animal_sniffer_annotations_1_14.xml 638B
Maven__junit_junit_4_12.xml 443B
Maven__org_jsoup_jsoup_1_10_3.xml 473B
Maven__org_springframework_boot_spring_boot_starter_1_5_4_RELEASE.xml 680B
Maven__org_skyscreamer_jsonassert_1_4_0.xml 525B
Maven__org_slf4j_slf4j_api_1_7_25.xml 501B
Maven__commons_codec_commons_codec_1_10.xml 531B
Maven__org_apache_commons_commons_lang3_3_6.xml 544B
Maven__com_vaadin_external_google_android_json_0_0_20131108_vaadin1.xml 688B
Maven__com_alibaba_druid_spring_boot_starter_1_1_0.xml 614B
Maven__org_yaml_snakeyaml_1_17.xml 483B
Maven__org_apache_httpcomponents_httpcore_4_4_6.xml 551B
Maven__com_alibaba_druid_1_1_0.xml 474B
Maven__org_objenesis_objenesis_2_1.xml 496B
Maven__org_apache_tomcat_tomcat_jdbc_8_5_15.xml 547B
Maven__org_springframework_boot_spring_boot_starter_logging_1_5_4_RELEASE.xml 736B
Maven__org_springframework_spring_tx_4_3_9_RELEASE.xml 590B
Maven__org_springframework_boot_spring_boot_1_5_4_RELEASE.xml 624B
Maven__org_springframework_boot_spring_boot_starter_jdbc_1_5_4_RELEASE.xml 715B
Maven__net_minidev_json_smart_2_2_1.xml 509B
Maven__org_slf4j_jcl_over_slf4j_1_7_25.xml 536B
Maven__org_springframework_spring_beans_4_3_9_RELEASE.xml 611B
Maven__org_mockito_mockito_core_1_10_19.xml 537B
Maven__mysql_mysql_connector_java_5_1_42.xml 562B
Maven__net_minidev_accessors_smart_1_1.xml 530B
Maven__org_springframework_boot_spring_boot_starter_test_1_5_4_RELEASE.xml 715B
Maven__org_springframework_boot_spring_boot_autoconfigure_1_5_4_RELEASE.xml 722B
Maven__ch_qos_logback_logback_classic_1_1_11.xml 563B
Maven__org_springframework_spring_core_4_3_9_RELEASE.xml 604B
Maven__com_jayway_jsonpath_json_path_2_2_0.xml 534B
Maven__com_google_guava_guava_22_0.xml 487B
Maven__org_springframework_spring_expression_4_3_9_RELEASE.xml 646B
misc.xml 454B
dataSources.xml 891B
inspectionProfiles
Project_Default.xml 262B
compiler.xml 626B
modules.xml 254B
encodings.xml 172B
共 71 条
- 1
资源评论
JJJ69
- 粉丝: 6222
- 资源: 5780
下载权益
C知道特权
VIP文章
课程特权
开通VIP
上传资源 快速赚钱
- 我的内容管理 展开
- 我的资源 快来上传第一个资源
- 我的收益 登录查看自己的收益
- 我的积分 登录查看自己的积分
- 我的C币 登录后查看C币余额
- 我的收藏
- 我的下载
- 下载帮助
安全验证
文档复制为VIP权益,开通VIP直接复制
信息提交成功