很简易的java爬虫 可以爬取携程的航班数据.rar

package com.Interface; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import org.apache.http.HttpEntity; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.util.EntityUtils; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import net.sf.json.JSONArray; import net.sf.json.JSONObject; public class CrawlerHttpClient { // 创建一个客户端 类似打开一个浏览器 private static CloseableHttpClient httpClient = HttpClientBuilder.create().build(); // 创建一个get方法，类似在浏览器中输入一个地址，path则为URL的值 static HttpGet httpGet; static CloseableHttpResponse response; static HttpEntity httpEntity; private static Map<String, String> map; static { map = new HashMap<String, String>(); map.put("KY", "昆明航空"); } public static void getAirLineDetails(String flight) { String time = getDateString(-1, "yyyyMMdd");// new Date()为获取当前系统时间 String path = "https://flights.ctrip.com/actualtime/fno--" + flight + "-" + time + ".html"; // 创建get请求 httpGet = new HttpGet(path); httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36"); String airLineName = null; String airLineNo = null; String airLineDate = null; String airLineDay = ""; String timeTakeOff = ""; String planTakeOff = ""; String timeArrive = ""; String planArrive = ""; String airLineStartCity = ""; String airLineEndCity = ""; try { // 获取响应模型 response = httpClient.execute(httpGet); // 获取响应实体 httpEntity = response.getEntity(); if (httpEntity != null) { // System.out.println("响应内容长度为:" + httpEntity.getContentLength()); // System.out.println("响应内容为:" + EntityUtils.toString(httpEntity)); Document doc = Jsoup.parse(EntityUtils.toString(httpEntity)); airLineName = doc.select("div.detail-t").select("span.ml5").first().text(); // System.out.println("航空公司：" + airLineName); airLineNo = doc.select("div.detail-t").select("strong.ml5").text(); // System.out.println("航班号：" + airLineNo); airLineDate = doc.select("div.detail-t").select("span.ml10").first().text(); // System.out.println("航班日期：" + airLineDate); airLineDay = doc.select("div.detail-t").select("span.ml5").last().text(); // System.out.println("周几：" + airLineDay); Elements airLineDetails = doc.select("div.detail-m"); //System.out.println(doc.select("div.detail-m")); for (Element element : airLineDetails) { if(element.select("i.fsans").text().equals("取消")) { timeTakeOff="取消"; }else { timeTakeOff = airLineDate + " " + element.select("div.departure>p:eq(1)").text(); } airLineStartCity=""; airLineEndCity=""; //System.out.println("实际起飞:" + timeTakeOff); planTakeOff = airLineDate + " " + element.select("div.departure>p:eq(2)").text() .substring(element.select("div.departure>p:eq(2)").text().indexOf(" ") + 1); //System.out.println("计划起飞:" + planTakeOff); airLineStartCity =element.select("div.detail-route").select("div.departure>p:eq(0)").text(); airLineStartCity=get_city_code(airLineStartCity); //System.out.println("起飞机场:" + airLineStartCity); airLineEndCity = element.select("div.detail-route").select("div.arrive>p:eq(0)").text(); airLineEndCity=get_city_code(airLineEndCity); //System.out.println("到达机场:" + airLineEndCity); if (element.select("div.arrive>p:eq(1)").text().indexOf('+') != -1) {// 判断实际到达时间是否是第二天 String towDate = getDateString(0, "yyyy-MM-dd");// 获取第二天的时间 timeArrive = towDate + " " + element.select("div.arrive>p:eq(1)").text().substring(0, 5); }else if(element.select("i.fsans").text().equals("取消")) { timeArrive="取消"; } else { timeArrive = airLineDate + " " + element.select("div.arrive>p:eq(1)").text(); } if (element.select("div.arrive>p:eq(2)").text().indexOf('+') != -1) {// 判断计划到达时间是否是第二天 String towDate = getDateString(0, "yyyy-MM-dd");// 获取第二天的时间 planArrive = towDate + " " + element.select("div.arrive>p:eq(2)").text().substring( element.select("div.arrive>p:eq(2)").text().indexOf(" ") + 1, element.select("div.arrive>p:eq(2)").text().indexOf("+")); } else { planArrive = airLineDate + " " + element.select("div.arrive>p:eq(2)").text().substring(element.select("div.departure>p:eq(2)").text().indexOf(" ") + 1); } if(getDistanceTime(planTakeOff, timeTakeOff)>=20) { timeTakeOff=getDateString(0, "yyyy-MM-dd")+ " " + element.select("div.departure>p:eq(1)").text(); timeArrive=getDateString(0, "yyyy-MM-dd")+" "+element.select("div.arrive>p:eq(1)").text(); } //System.out.println("实际到达:" + timeArrive); //System.out.println("计划到达:" + planArrive); String para = "[\"" + airLineName + "\",\"" + airLineNo + "\",\"" + airLineDate + "\",\"" + airLineDay + "\",\"" + timeTakeOff + "\",\"" + planTakeOff + "\",\"" + timeArrive + "\",\"" + planArrive + "\",\"" + airLineStartCity + "\",\"" + airLineEndCity + "\"]"; //System.out.println(para); para = URLEncoder.encode(para, "UTF-8");// 设置编码格式 重要重要！！！！ //模拟http请求 去调service里面的方法 System.out.println(httpGet("addAirLines",para)); } } } catch (ClientProtocolException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (Exception e) { for (Map.Entry<String, String> entry : map.entrySet()) { if (flight.indexOf(entry.getKey()) != -1) { airLineName = entry.getValue(); } } airLineNo = flight; airLineDate = getDateString(-1, "yyyy-MM-dd"); //System.out.println(airLineDate); String para = "[\"" + airLineName + "\",\"" + airLineNo + "\",\"" + airLineDate + "\",\"" + airLineDay + "\",\"" + timeTakeOff + "\",\"" + planTakeOff + "\",\"" + timeArrive + "\",\"" + planArrive + "\",\"" + airLineStartCity + "\",\"" + airLineEndCity + "\"]"; //System.out.println(para); // 设置编码格式 重要重要！！！！ try { para = URLEncoder.encode(para, "UTF-8"); } catch (UnsupportedEncodingException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } System.out.println(httpGet("addAirLines",para)); // TODO Auto-generated catch block } } //计算时间差，以小时为单位。 public static int getDistanceTime(String startTime, String endTime) { SimpleDateFormat sDateFormat=new SimpleDateFormat("yyyy-MM-dd HH:mm"); int hours = 0; long time1 = 0; long time2 = 0; if(!endTime.equals("取消")&&!startTime.equals("--:--")) { try { time1 = sDateFormat.parse(startTime).getTime(); time2 = sDateFormat.parse(endTime).getTime(); } catch (ParseException e) { // TODO Auto-generated catch block e.printStackTrace(); } long diff; if (time1 < time2) { diff = time2 - time1; } else { diff = time1 - time2;