java爬虫抓取城市数据

package com.lcx.util; import java.awt.Dimension; import java.awt.Toolkit; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.InetAddress; import java.net.URL; import java.net.UnknownHostException; import java.util.ArrayList; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; public class Html { //根据url从网络获取网页文本 public static Document getHtmlTextByUrl(String url) { Document doc = null; try { //doc = Jsoup.connect(url).timeout(5000000).get(); int i = (int)(Math.random() * 1000); //做一个随机延时，防止网站屏蔽 while (i != 0) { i--; } doc = Jsoup.connect(url).data("query", "Java").userAgent("Mozilla").cookie("auth", "token").timeout(300000).post(); } catch (IOException e) { e.printStackTrace(); try { doc = Jsoup.connect(url).timeout(5000000).get(); } catch (IOException e1) { // TODO Auto-generated catch block e1.printStackTrace(); } } return doc; } //根据本地路径获取网页文本，如果不存在就通过url从网络获取并保存 public static Document getHtmlTextByPath(String name, String url) { String path = "D:/Html/" + name + ".html"; Document doc = null; File input = new File(path); // String urlcat = url; try { doc = Jsoup.parse(input, "GBK"); if (!doc.children().isEmpty()) { doc = null; System.out.println("已经存在"); } } catch (IOException e) { System.out.println("文件未找到，正在从网络获取......."); doc = getHtmlTextByUrl(url); //并且保存到本地 Save_Html(url, name); } return doc; } //此处为保存网页的函数 //将网页保存在本地（通过url,和保存的名字） public static void Save_Html(String url, String name) { try { name = name + ".html"; // System.out.print(name); File dest0 = new File("D:/Html/"); // D:\Html if (!dest0.exists()) { dest0.mkdirs(); } File dest = new File("D:/Html/" + name); // D:\Html // 创建文件 if (!dest.exists()) { try { dest.createNewFile(); System.out.println("创建成功-----D:/Html/" + name); } catch (IOException e) { e.printStackTrace(); } } //接收字节输入流 InputStream is; //字节输出流 FileOutputStream fos = new FileOutputStream(dest); URL temp = new URL(url); is = temp.openStream(); //为字节输入流加缓冲 BufferedInputStream bis = new BufferedInputStream(is); //为字节输出流加缓冲 BufferedOutputStream bos = new BufferedOutputStream(fos); int length; byte[] bytes = new byte[1024 * 20]; while ((length = bis.read(bytes, 0, bytes.length)) != -1) { fos.write(bytes, 0, length); } bos.close(); fos.close(); bis.close(); is.close(); } catch (IOException e) { e.printStackTrace(); } } //根据元素属性获取某个元素内的elements列表 public static Elements getEleByClass(Document doc, String className) { //<tr class="provincetr"> Elements elements = null; elements = doc.select(className); //这里把我们获取到的html文本doc，和工具class名，注意<tr class="provincetr"> return elements; //此处返回的就是所有的tr集合 } //获取电脑屏幕信息 public static void getScreen() { Dimension screensize = Toolkit.getDefaultToolkit().getScreenSize(); int width = (int) screensize.getWidth(); int height = (int) screensize.getHeight(); System.out.println("宽的像素：" + width + "高的像素：" + height); //获取屏幕的dpi int dpi = Toolkit.getDefaultToolkit().getScreenResolution(); System.out.println(dpi); //根据dpi和像素，可以计算物理尺寸 System.out.println("宽：" + width / dpi + "高：" + height / dpi); } //获取本机名称和IP public static void getIP(){ InetAddress ia = null; try { ia = ia.getLocalHost(); String localname = ia.getHostName(); String localip = ia.getHostAddress(); System.out.println("本机名称是：" + localname); System.out.println("本机的ip是 ：" + localip); } catch (UnknownHostException e) { e.printStackTrace(); } } //---------------------------------------------------------------------------------------- public static void main(String[] args) { } }