基于Socket和HttpURLConnection的爬虫实例

package com.gs.simple.impl.crawl; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.net.Socket; import com.gs.simple.iface.crawl.ICrawl; import com.gs.simple.pojos.CrawlResultPojo; import com.gs.simple.pojos.UrlPojo; public class SocketCrawlerImpl implements ICrawl { public CrawlResultPojo crawl(UrlPojo urlPojo) { CrawlResultPojo crawlResultPojo = new CrawlResultPojo(); if (urlPojo == null || urlPojo.getUrl() == null) { crawlResultPojo.setIsSuccess(false); crawlResultPojo.setPageContent(null); return crawlResultPojo; } String host = urlPojo.getHost(); if (host == null) { crawlResultPojo.setIsSuccess(false); crawlResultPojo.setPageContent(null); return crawlResultPojo; } BufferedWriter bw = null; BufferedReader br = null; try { Socket socket = new Socket(host, 80); bw = new BufferedWriter(new OutputStreamWriter( socket.getOutputStream())); bw.write("GET " + urlPojo.getUrl() + " HTTP 1.0\r\n"); bw.write("Host:" + host + "\r\n"); bw.write("\r\n"); bw.flush(); br = new BufferedReader(new InputStreamReader( socket.getInputStream(),"utf-8")); String line = null; StringBuilder stringBuilder = new StringBuilder(); while ((line = br.readLine()) != null) { // System.out.println(line); stringBuilder.append(line + "\n"); } crawlResultPojo.setIsSuccess(true); crawlResultPojo.setPageContent(stringBuilder.toString()); return crawlResultPojo; } catch (Exception e) { e.printStackTrace(); } finally { try { if (bw != null) { bw.close(); } if (br != null) { br.close(); } } catch (Exception e) { e.printStackTrace(); System.out.println("流未正确关闭"); } } return null; } public static void main(String[] args) { SocketCrawlerImpl socketCrawlImpl = new SocketCrawlerImpl(); UrlPojo urlPojo = new UrlPojo("http://www.baidu.com"); socketCrawlImpl.crawl(urlPojo); System.out.println("down!"); } }