Java抓取网络资源并插入数据库

package controller; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import webTools.GetBook; import dbTools.DBTools; public class TestTaoBao extends HttpServlet { /** * Constructor of the object. */ public TestTaoBao() { super(); } /** * Destruction of the servlet. <br> */ public void destroy() { super.destroy(); // Just puts "destroy" string in log // Put your code here } /** * The doGet method of the servlet. <br> * * This method is called when a form has its tag value method equals to get. * * @param request * the request send by the client to the server * @param response * the response send by the server to the client * @throws ServletException * if an error occurred * @throws IOException * if an error occurred */ int i=0; public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { // try { // GetBook getBook = new GetBook(); // getBook.getTaobaoHtml("http://shop62629085.taobao.com/?price1=&price2=&search=y&pageNum=1&scid=0&keyword=&orderType=hotsell&old_starts=&categoryp=&pidvid=&viewType=grid&isNew=&ends=&baobei_type=#anchor"); // } catch (Throwable e) { // // TODO Auto-generated catch block // e.printStackTrace(); // } try { GetBook getBook = new GetBook(); DBTools dbtools = new DBTools(); String sql = "SELECT * FROM test.vtao123 where id between 626 and 944"; ArrayList list_values = new ArrayList(); ArrayList list_vtao123 = dbtools.query(sql, list_values); // i=list_vtao123.size()-1; // System.out.println("i---------------------"); for (; i < list_vtao123.size(); i++) { String[] str_arr = (String[]) list_vtao123.get(i); String type = str_arr[1]; String dianpuName = str_arr[2]; String pageCount = "1"; String pageCount_String = getBook.outTag(getBook .getPageCount(getBook.getHtmlContent(str_arr[3]))); String dianpuDengji = getBook.getDianpuDengji(getBook .getHtmlContent(str_arr[3])); if(dianpuDengji.equals("s_red_2.gif")){ dianpuDengji="两心店"; } if (dianpuDengji.equals("")) { dianpuDengji = "商铺"; } Pattern pattern = Pattern.compile("[0-9]*"); Matcher isNum = pattern.matcher(pageCount); String isNumber =""; System.out.println("i=========================================="+i); if( !isNum.matches() ) { isNumber= "false"; }else{ isNumber="true"; } if (isNumber.equals("true")) { pageCount = pageCount_String.substring(pageCount_String .indexOf("/") + 1); if (Integer.parseInt(pageCount) > 5) { pageCount = "5"; } } for (int j = 1; j <= Integer.parseInt(pageCount); j++) { String href = str_arr[3]; href = href + "&price1=&price2=&pageNum=" + j + "&scid=0&keyword=&orderType=hotsell&old_starts=&categoryp=&pidvid=&viewType=grid&isNew=&ends=&baobei_type=#anchor"; String htmlContent = getBook.getHtmlContent(href); List list_BookSrc = getBook.getTaoBao(htmlContent); for (int m = 0; m < list_BookSrc.size(); m++) { String name = getBook.outTag(getBook .getTaoBaoName((String) list_BookSrc.get(m))); if (name.equals("") || name == null) { break; } String price = getBook.outTag(getBook .getTaoBaoPrice((String) list_BookSrc.get(m))).replaceAll("一口价", "").replaceAll("市场价：(.*?)元", "").replace("元", ""); String number = getBook.outTag(getBook .getTaoBaoNumber((String) list_BookSrc.get(m))).replaceAll("最近30天售出", "").replace("件", "").replace("已销售：", ""); System.out.println(dianpuDengji + " " + type + " " + dianpuName + " " + href + name + "\t" + price + "\t" + number); String sql1 = "insert into zhuyi_vtao(dianpuDengji,type,dianpuName,href,name,price,number) values(?,?,?,?,?,?,?)"; ArrayList list_values1 = new ArrayList(); list_values1.add(dianpuDengji); list_values1.add(type); list_values1.add(dianpuName); list_values1.add(href); list_values1.add(name); list_values1.add(price); list_values1.add(number); dbtools.update(sql1, list_values1); } } } // String htmlURL = "http://shop247483.taobao.com/?NUM=247483"; // GetBook getBook = new GetBook(); // getBook.getTaobaoHtml(htmlURL); } catch (Throwable e) { i=i+1; doGet(request,response); // e.printStackTrace(); } } /** * The doPost method of the servlet. <br> * * This method is called when a form has its tag value method equals to * post. * * @param request * the request send by the client to the server * @param response * the response send by the server to the client * @throws ServletException * if an error occurred * @throws IOException * if an error occurred */ public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { response.setContentType("text/html"); PrintWriter out = response.getWriter(); out .println("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\">"); out.println("<HTML>"); out.println(" <HEAD><TITLE>A Servlet</TITLE></HEAD>"); out.println(" <BODY>"); out.print(" This is "); out.print(this.getClass()); out.println(", using the POST method"); out.println(" </BODY>"); out.println("</HTML>"); out.flush(); out.close(); } /** * Initialization of the servlet. <br> * * @throws ServletException * if an error occurs */ public void init() throws ServletException { // Put your code here } }