# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
class Douban250SpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class Douban250DownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
没有合适的资源?快使用搜索试试~ 我知道了~
Python 爬豆瓣电影 Top 250,Scrapy框架
共19个文件
py:8个
pyc:5个
xml:3个
需积分: 12 7 下载量 149 浏览量
2018-12-20
11:30:37
上传
评论 1
收藏 24KB 7Z 举报
温馨提示
Python 爬豆瓣电影 Top 250,Scrapy框架,https://blog.csdn.net/qq_31939617/article/details/85115242
资源推荐
资源详情
资源评论
收起资源包目录
douban250.7z (19个子文件)
douban250
scrapy.cfg 261B
douban250.csv 37KB
.idea
misc.xml 294B
modules.xml 277B
douban250.iml 478B
workspace.xml 20KB
inspectionProfiles
douban250
main.py 79B
middlewares.py 4KB
pipelines.py 290B
spiders
DouBan250.py 1KB
__pycache__
__init__.cpython-37.pyc 148B
DouBan250.cpython-37.pyc 1KB
__init__.py 161B
__pycache__
__init__.cpython-37.pyc 140B
settings.cpython-37.pyc 429B
items.cpython-37.pyc 398B
items.py 311B
__init__.py 0B
settings.py 3KB
共 19 条
- 1
资源评论
码点
- 粉丝: 2115
- 资源: 60
上传资源 快速赚钱
- 我的内容管理 展开
- 我的资源 快来上传第一个资源
- 我的收益 登录查看自己的收益
- 我的积分 登录查看自己的积分
- 我的C币 登录后查看C币余额
- 我的收藏
- 我的下载
- 下载帮助
安全验证
文档复制为VIP权益,开通VIP直接复制
信息提交成功