利用HttpClient和HtmlParser实现的简单爬虫（Java）

package org.qing.searchengine.webcollector; import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.net.URI; import java.util.ArrayList; import java.util.List; import org.apache.http.Consts; import org.apache.http.HttpEntity; import org.apache.http.NameValuePair; import org.apache.http.client.HttpClient; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; /** * 网页下载类 * @author Qing * */ public class FileDownLoader { /** * 根据url和网页类型生成需要保存的网页中文名，去除url中非文件名字符 * @param url * @param contentType * @return */ public String getFileNameByUrl(String url,String contentType){ url = url.substring(7);//移除http:// if(contentType.indexOf("html") != -1){//html url = url.replaceAll("[\\?/:*|<>\"]","_");//去掉url中非文件名字符生成文件名 return url; } else{ return url.replaceAll("[\\?/:*|<>\"]","_")+"." + contentType.substring(contentType.lastIndexOf("/")+1); } } /** * 保存网页字节数组到本地文件 * @param data * @param filePath */ public void saveToLocal(String data,String filePath){ try{ DataOutputStream out = new DataOutputStream(new FileOutputStream(new File(filePath))); out.writeUTF(data);//write in utf-8 out.flush(); out.close(); }catch(Exception e){ e.printStackTrace(); } } /** * 下载url网页 * @param url * @return */ public String downloadFile(String url){ UrlEncodedFormEntity uefEntity; String filePath = null; CloseableHttpClient httpclient = HttpClients.createDefault(); try{ HttpGet httpget = new HttpGet(url); List<NameValuePair> params = new ArrayList<NameValuePair>(); String str = EntityUtils.toString(new UrlEncodedFormEntity(params, Consts.UTF_8)); httpget.setURI(new URI(httpget.getURI().toString() +"?" + str)); //执行get请求 CloseableHttpResponse response = httpclient.execute(httpget); //获取响应实体 HttpEntity entity = response.getEntity(); filePath = "temp/"+ getFileNameByUrl(url,response.getFirstHeader("Content-Type").getValue()); saveToLocal(EntityUtils.toString(entity),filePath); }catch(Exception e){ e.printStackTrace(); } return filePath; } public static void main(String[] args){ FileDownLoader downloader = new FileDownLoader(); downloader.downloadFile("http://club.xdnice.com/thread-1400344-1-1.html"); } }