基于Python的爬取今日头条文章及视频+源代码+文档说明

今日头条爬虫技术说明 ========== 整体思路 -------- 1. 抓取今日头条app的数据包 2. 分析数据包，找出请求的数据（如文章列表，文章url等） 3. 根据文章url等信息，抓取文章内容 4. 若文章中包含视频，则取视频url，然后下载 具体实现 ------ ### 一、抓包 ### 1. 工具：`Fiddler`、 `android`手机、`google浏览器` 2. 步骤： （1）本示例采用[Fiddler](http://fiddler2.com/ "Fiddler")来抓包，安装到电脑，我的电脑是win10 （2）具体配置及使用请见[http://jingyan.baidu.com/article/03b2f78c7b6bb05ea237aed2.html](http://jingyan.baidu.com/article/03b2f78c7b6bb05ea237aed2.html "百度经验") （3）打开今日头条app，开始抓包，抓取到的包如下： （4）经分析得知左侧的json文件及为文章列表，如图 如右侧第一个content所指的json文件，文件内容为此时手机上的信息为 <img src = "http://i.imgur.com/LFSL1AA.png" width = "40%"> 可得上面的结论正确 ### 二、分析 ### 1. 分析所抓到的文章列表数据包：大致分为两类，一类是有视频的文章，一类则是没有视频的文章。 有视频的文章json内容里均有`video_id`这个key，如下图所示： 2. 没有视频的文章：json文件内容均包含`title`、 `abstract`、 `article_url`等信息，具体内容如下. <pre> { "read_count":7089, "media_name":"数码日记", "ban_comment":0, "abstract":"一般情况下，购买手机后我们都会买SIM卡，这是无可厚非的事情。但也有些网友买手机仅仅是作为备用机，用于日常上网使用，而在没有SIM卡的情况下，手机要想联网，只能通过连接Wi-Fi的方式实现。", "image_list":[ { "url":"http://p1.pstatp.com/list/ef6000f83af6113a252", "width":698, "url_list":[ { "url":"http://p1.pstatp.com/list/ef6000f83af6113a252" }, { "url":"http://p4.pstatp.com/list/ef6000f83af6113a252" }, { "url":"http://p.pstatp.com/list/ef6000f83af6113a252" } ], "uri":"list/ef6000f83af6113a252", "height":392 }, { "url":"http://p3.pstatp.com/list/ef6000f83f79a941883", "width":981, "url_list":[ { "url":"http://p3.pstatp.com/list/ef6000f83f79a941883" }, { "url":"http://p6.pstatp.com/list/ef6000f83f79a941883" }, { "url":"http://p.pstatp.com/list/ef6000f83f79a941883" } ], "uri":"list/ef6000f83f79a941883", "height":551 }, { "url":"http://p3.pstatp.com/list/ef6000f8405551c32f2", "width":943, "url_list":[ { "url":"http://p3.pstatp.com/list/ef6000f8405551c32f2" }, { "url":"http://p6.pstatp.com/list/ef6000f8405551c32f2" }, { "url":"http://p.pstatp.com/list/ef6000f8405551c32f2" } ], "uri":"list/ef6000f8405551c32f2", "height":530 } ], "has_video":false, "article_type":0, "tag":"digital", "has_m3u8_video":0, "keywords":"MIUI,SIM卡,Wi-Fi", "user_verified":1, "aggr_type":1, "cell_type":0, "article_sub_type":0, "bury_count":0, "title":"​不插SIM卡，不用Wi-Fi，小米手机也能上网", "ignore_web_transform":1, "source_icon_style":1, "tip":0, "hot":0, "share_url":"http://toutiao.com/a6351618096909779201/?iid=6181230843&app=news_article", "has_mp4_video":0, "source":"数码日记", "comment_count":22, "article_url":"http://toutiao.com/group/6351618096909779201/", "filter_words":[ { "id":"8:0", "name":"重复、旧闻", "is_selected":false }, { "id":"9:1", "name":"内容质量差", "is_selected":false }, { "id":"5:32370023", "name":"来源:数码日记", "is_selected":false }, { "id":"2:306461588", "name":"路由器", "is_selected":false }, { "id":"6:47522", "name":"小米手机", "is_selected":false } ], "publish_time":1478851309, "action_list":[ { "action":1, "extra":{ }, "desc":"" }, { "action":3, "extra":{ }, "desc":"" }, { "action":7, "extra":{ }, "desc":"" }, { "action":9, "extra":{ }, "desc":"" } ], "gallary_image_count":4, "cell_layout_style":1, "tag_id":6351618096909779000, "video_style":0, "verified_content":"头条问答数码专家，安卓领域老兵", "display_url":"http://toutiao.com/group/6351618096909779201/", "large_image_list":[ ], "item_id":6351618007961175000, "repin_count":117, "cell_flag":11, "user_info":{ "user_id":3378232303, "name":"数码日记", "verified_content":"头条问答数码专家，安卓领域老兵", "avatar_url":"http://p3.pstatp.com/thumb/1209/1834968385", "follow":false, "user_verified":true, "description":"" }, "source_open_url":"sslocal://media_account?media_id=3392972202", "level":0, "digg_count":0, "behot_time":1478855948, "cursor":1478855948999, "url":"http://toutiao.com/group/6351618096909779201/", "preload_web":1, "user_repin":0, "has_image":true, "item_version":0, "media_info":{ "user_id":3378232303, "verified_content":"", "avatar_url":"http://p2.pstatp.com/large/1209/1834968385", "media_id":3392972202, "name":"数码日记", "follow":false, "recommend_reason":"", "is_star_user":false, "user_verified":true }, "group_id":6351618096909779000, "middle_image":{ "url":"http://p1.pstatp.com/list/ef6000f83af6113a252", "width":698, "url_list":[ { "url":"http://p1.pstatp.com/list/ef6000f83af6113a252" }, { "url":"http://p4.pstatp.com/list/ef6000f83af6113a252" }, { "url":"http://p.pstatp.com/list/ef6000f83af6113a252" } ], "uri":"list/ef6000f83af6113a252", "height":392 } } </pre> 3. 有视频的文�