Python爬虫开发 基于Python实现的批量抓取采集新浪博客页面的所有文章 含源代码及案例数据集.rar

#coding:utf-8 import urllib import time import os ''' 抓取新浪博客 博客页面的所有文章 http://blog.sina.com.cn/s/articlelist_1191258123_0_1.html ''' page=1 # 起始页 while page <= 7: # 一共爬取7页 url = [''] * 50 # 预分配大小，每页有50篇文章 base_url = 'http://blog.sina.com.cn/s/articlelist_1191258123_0_' temp = base_url + str(page) + '.html' # 网页的完全路径 content = urllib.urlopen(temp).read() # 读取HTML文本内容，打开首页 i = 0 title = content.find(r'<a title=') # title href = content.find(r'href=', title) # href html = content.find(r'.html', href) # html,找到网页 while title != -1 and href != -1 and html != -1 and i < 50: url[i] = content[href + 6 : html + 5] print url[i] title = content.find(r'<a title=',html) href = content.find(r'href=',title) html = content.find(r'.html',href) i = i + 1 else: print 'end page=',page '''''写入本地文件''' j = 0 while(j < i): content = urllib.urlopen(url[j]).read() path = 'hanhan/' + str(page) + "/" if os.path.isdir(path): # 已经有路径 open(path + url[j][-26:], 'w+').write(content) else: os.makedirs(path) # 没有则创建路径 open(path + url[j][-26:], 'w+').write(content) j = j + 1 time.sleep(1) # 延时 else: print 'download' page = page + 1 else: print 'all find end'