MapReduce暑假大作业——基于紫荆的种子推荐.zip

import org.jsoup.Connection; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintWriter; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Scanner; /** * 获取紫荆种子的已完成下载列表 * Created by user36 on 6/25/14. */ public class GetTorrentDownloadList { static PrintWriter outputWriter; static PrintWriter failWriter; /** * 向文件写入一条下载记录 * 写入格式是“种子ID \t 用户1,用户2,用户3,... \n” * * @param tid 种子ID * @param uidList 用户列表 */ public static void writeRecord(String tid, List<String> uidList) { if(uidList == null) return; outputWriter.print(tid + "\t"); Iterator<String> it; StringBuilder builder = new StringBuilder(); for (it = uidList.iterator(); it.hasNext(); ) { builder.append(it.next()); if (it.hasNext()) builder.append(','); } outputWriter.println(builder.toString()); // System.out.println(builder.toString()); outputWriter.flush(); } public static void main(String[] args) throws FileNotFoundException, InterruptedException { outputWriter = new PrintWriter("torrentDownloadList.txt"); failWriter = new PrintWriter("fail.txt"); //从tID.unique.txt中读取种子ID列表 Scanner scanner = new Scanner(new File("tID.unique.txt")); while (scanner.hasNextLine()) { String tid = scanner.nextLine(); System.err.println("---- " + tid + " --->"); List<String> uidList = getDownloadUserList(tid); writeRecord(tid, uidList); Thread.sleep(500); } outputWriter.close(); scanner.close(); } /** * 获取某个种子的已完成下载用户列表 * * @param tid 种子ID * @return 用户ID列表 */ public static List<String> getDownloadUserList(String tid) { //获得种子已下载用户列表的信息页 String url = "http://zijingbt.njuftp.org/stats.html?id=" + tid + "&show=completes#completes"; Document document = getURLDocument(url); if(document == null){ failWriter.println(tid); failWriter.flush(); return null; } assert (document != null); //获得用户名单 Elements table = document.getElementsByAttributeValue("summary","completes"); Elements members = table.select("a.member"); members.addAll(table.select("a.friends")); members.addAll(table.select("a.uploader")); LinkedList<String> linkedList = new LinkedList<String>(); for(Element member :members){ //System.out.println(member.ownText()); linkedList.add(member.ownText()); } return linkedList; } //获取指定URL的页面 public static Document getURLDocument(String url) { Connection c = Jsoup.connect(url); c .userAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2017.2 Safari/537.36 OPR/24.0.1537.0 (Edition Developer)"); c.cookie("login", "bsidb"); c.cookie("uid", "3435"); c.cookie("md5", "%3D%0CjZ%82%96%D6%2C%01S%3FY%26%3F%B6B"); c.cookie("per_page", "50"); // "login="bsidb"; uid="3435"; md5="%3D%0CjZ%82%96%D6%2C%01S%3FY%26%3F%B6B"; per_page="50"; Connection.Response res = null; int tryCount = 0; final int MAX_TRY_COUNT = 3;//最多尝试三次 while (true && tryCount < MAX_TRY_COUNT) { try { c.timeout(10000); res = c.execute(); break; } catch (Exception e) { System.err.println("[GET URL ERROR]try to get " + url + ", " + tryCount + " time fails"); // e.printStackTrace(); System.err.println("we will try it later"); } try { Thread.sleep(5000);//5s后重试 } catch (InterruptedException e) { // TODO Auto-generated catch block e.printStackTrace(); } System.err.println("Try again."); tryCount++; } if (tryCount == MAX_TRY_COUNT) { System.err.println("[GET URL ERROR] url : " + url + " reaches max try times.Fails.\n"); return null;//返回null对象 } Document doc = null; try { doc = res.parse(); } catch (IOException e) { e.printStackTrace(); System.err.println("[GET URL ERROR]Parse fails for " + url); } //System.out.println("Got!"); return doc; } }