java解析html抓取网站数据采集网站信息

package dd; import java.io.BufferedInputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; import java.util.HashSet; import java.util.Set; import org.htmlparser.NodeFilter; import org.htmlparser.Parser; import org.htmlparser.filters.TagNameFilter; import org.htmlparser.tags.ImageTag; import org.htmlparser.tags.LinkTag; import org.htmlparser.util.NodeList; import org.htmlparser.util.ParserException; import org.htmlparser.visitors.TextExtractingVisitor; public class ddd { private static int BUFFER_SIZE = 8096;// 缓冲区大小 /** * 保存图片 * 将HTTP资源另存为文件 * * @param destUrl * String * @param title * @param fileName * String * @throws IOException * @throws Exception */ public static void saveToFile(String destUrl, String text) { FileOutputStream fos = null; BufferedInputStream bis = null; HttpURLConnection httpUrl = null; URL url = null; byte[] buf = new byte[BUFFER_SIZE]; int size = 0; int pos = destUrl.lastIndexOf('/');//查找图片名称开始的位置 String fileName = ""; if (pos != -1) fileName = destUrl.substring(pos + 1, destUrl.length());//获得图片的名称 // else // fileName = destUrl.substring(destUrl.length()-10,destUrl.length()); // String path ="D:"+File.separator+"temp"+File.separator+"images"+File.separator;//并出本地路径D:\temp\images\ if (null != text && !"".equals(text)) { File file = new File(path + text + File.separator);//new文件夹对象，路径是D:\temp\images\+text\ if (!file.exists()) {//如果这个文件夹不存在 file.mkdirs();//就创建这个文件夹 } path = file.getPath();//获得创建的文件的路径 } path = path + File.separator + fileName;//并出图片文件的路径 // 下面是的将图片保存在本地路径中 try { url = new URL(destUrl); httpUrl = (HttpURLConnection) url.openConnection(); // 连接指定的资源 httpUrl.connect(); // 获取网络输入流 bis = new BufferedInputStream(httpUrl.getInputStream()); // 建立文件 fos = new FileOutputStream(path); // 保存文件 while ((size = bis.read(buf)) != -1) fos.write(buf, 0, size); fos.close(); bis.close(); httpUrl.disconnect(); } catch (MalformedURLException e) { e.printStackTrace(); } catch (IOException ex) { ex.printStackTrace(); } //System.out.println(" \tsave completely"); } /** * 主方法 * * @param argv * String[] */ public static void main(String argv[]) { String url = "http://www.whweishi.com/"; try { getPageLinks(url); } catch (Exception e) { e.printStackTrace(); } } /** * 得到链接和链接文本 * @param page * @throws ParserException */ private static void getPageLinks(String page) throws ParserException { Parser myParser = new Parser(page); // 设置编码 myParser.setEncoding("GBK"); String filterStr = "a"; NodeFilter filter = new TagNameFilter(filterStr); NodeList nodeList = myParser.extractAllNodesThatMatch(filter); for (int i = 0; i < nodeList.size(); i++) { LinkTag linkTag = (LinkTag) nodeList.elementAt(i); String link = linkTag.getLink(); //得到链接 // String text = linkTag.getLinkText();//得到链接的文本 TextExtractingVisitor visitor1=new TextExtractingVisitor(); Parser title_parser=new Parser(nodeList.elementAt(i).toHtml()); title_parser.visitAllNodesWith(visitor1); String text=visitor1.getExtractedText();//得到链接的文本，上面三行代码可以把取出来的值过滤掉html代码 //text = TextProcess(text);//过滤html代码 if (link.endsWith(".asp") && text.length() > 2) { try { getImages(link,text); } catch (Exception e) { e.printStackTrace(); } } } } /** * 过滤html代码 * @param text * @return */ // private static String TextProcess(String text) { // text = text.trim(); // text = text.replaceAll(">", ""); // text = text.replaceAll("<", ""); // text = text.replaceAll("/", ""); // text = text.replaceAll(">", ""); // text = text.replaceAll(" ", ""); // text = text.replaceAll("&nbsp;", ""); // text = text.replaceAll("&quot;", ""); // int pos = 0; // if ((pos = text.indexOf(":-")) != -1) // text = text.substring(pos + 2); // // pos = text.indexOf("－"); // if (pos != -1) // text = text.substring(0, pos); // pos = text.indexOf("-"); // if (pos != -1) // text = text.substring(0, pos); // text = text.replace(".", ""); // text = text.replaceAll(",", ""); // text = text.replaceAll("，", ""); // return text; // } // private static String getTitle(String url) throws ParserException { // // Parser myParser = new Parser(url); // // // 设置编码 // myParser.setEncoding("UTF-8"); // // String titleTag = "title"; // NodeFilter titleFilter = new TagNameFilter(titleTag); // NodeList titleList = myParser.extractAllNodesThatMatch(titleFilter); // int size = titleList.size(); // String title = null; // if (size == 1) { // TitleTag titleT = (TitleTag) titleList.elementAt(0); // title = titleT.getTitle(); // // } // return title; // // } /** * 获得图像 * @param resource * @param title * @throws Exception */ public static void getImages(String link, String text) throws Exception { // Set Set<String> imagesSet = new HashSet<String>(); Parser myParser = new Parser(link); // 设置编码 myParser.setEncoding("UTF-8"); String filterStr = "img"; NodeFilter filter = new TagNameFilter(filterStr); NodeList nodeList = myParser.extractAllNodesThatMatch(filter); for (int i = 0; i < nodeList.size(); i++) { ImageTag imageTag = (ImageTag) nodeList.elementAt(i);//把过滤的图片传给一个ImageTag String imageUrl = imageTag.getImageURL();//获得图片的地址 if (!imagesSet.contains(imageUrl)) { //System.out.print("\t saving ..."); saveToFile(imageUrl, text); } else { System.out.print("\t exist already,no need to save"); } } } }