使用HttpClient 和 jsoup 下载 Google logo

package com.googlelogo; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.json.JSONArray; import org.json.JSONObject; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; /** * google logo 下载程序 * * http://www.dh189.com/ * @author ZZJ * */ public class GoogleLogoCrawler extends Crawler { private static final String URL = "http://www.google.com.hk/logos/logos%y-%n.html"; private static final String LOGO_URL = "http://www.google.com.hk"; private static final String[] YEARS = new String[] { "98", "99", "00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "10" }; private static final String INDEX = "http://www.google.com.hk/logos/index.html"; private static final String DIR_PATH="D:\\googlelogos\\"; public void doStart() { JSONArray array = new JSONArray(); for (String year : YEARS) { for (int i = 1; i < 5; i++) { String url = URL.replaceAll("%y", year).replaceAll("%n", i + ""); String path = year + "_" + i; start(url, array, DIR_PATH + path + "\\", path); } } start(INDEX, array, DIR_PATH+"10_3\\", "10_3"); try { FileUtils.writeStringToFile(new File(DIR_PATH+"json"), array.toString(), "UTF-8"); } catch (IOException e) { e.printStackTrace(); } System.out.println(array); } public void start(String url, JSONArray array, String dir, String path) { try { String content = super.doGet(url); Document doc = Jsoup.parse(content); Elements dts = doc.select(".doodles dt"); Elements dds = doc.select(".doodles dd"); List<Map<String, Object>> list = new ArrayList<Map<String, Object>>(0); for (int i = 0; i < dts.size(); i++) { Element dt = dts.get(i); Element dd = dds.get(i); Map<String, Object> map = super.getMap(); list.add(map); map.put("date", dt.text()); map.put("url", LOGO_URL + dd.select("img").first().attr("src")); map.put("title", dd.text()); } JSONArray array2 = new JSONArray(); for (Map<String, Object> map : list) { JSONObject json = new JSONObject(); String proxy = StringUtils.substringAfterLast(map.get("url").toString(), "."); long date = new Date().getTime(); String name = date + "." + proxy; json.put("url", map.get("url").toString()); json.put("dir", name); json.put("title_en", map.get("title")); json.put("date", map.get("date")); //翻译 String dateZh = super.translateEnToCinese(map.get("date").toString()); String titleZh = super.translateEnToCinese(map.get("title").toString()); json.put("title_zh_cn", dateZh + " - " + titleZh); //下载图片 super.downloadFile(map.get("url").toString(), dir + name); array2.put(json); } array.put(new JSONObject().put(path, array2)); } catch (Exception e) { e.printStackTrace(); } } public static void main(String[] args) throws Exception { new GoogleLogoCrawler().doStart(); } }