# -*- coding: utf-8 -*-
# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
class ImmocSpiderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, dict or Item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Response, dict
# or Item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class ImmocDownloaderMiddleware(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
没有合适的资源?快使用搜索试试~ 我知道了~
scrapy简单案例--爬取慕课网所有课程信息
共18个文件
py:10个
pyc:7个
cfg:1个
需积分: 48 36 下载量 46 浏览量
2018-04-04
12:59:20
上传
评论 6
收藏 11KB ZIP 举报
温馨提示
这是一个简单的scrapy入门案例,scrapy是1.5python是3
资源推荐
资源详情
资源评论
收起资源包目录
immoc.zip (18个子文件)
immoc
images
scrapy.cfg 253B
immoc
middlewares.py 4KB
pipelines.py 286B
courseitems.py 146B
spiders
myspider.py 1KB
__pycache__
myspider.cpython-36.pyc 1KB
__init__.cpython-36.pyc 152B
__init__.py 161B
__pycache__
mypipelines.cpython-36.pyc 1KB
imagepipelines.cpython-36.pyc 1KB
courseitems.cpython-36.pyc 443B
__init__.cpython-36.pyc 144B
settings.cpython-36.pyc 339B
items.py 285B
__init__.py 0B
settings.py 3KB
imagepipelines.py 1KB
mypipelines.py 577B
共 18 条
- 1
资源评论
三斤和他的朋友们
- 粉丝: 25
- 资源: 4
上传资源 快速赚钱
- 我的内容管理 展开
- 我的资源 快来上传第一个资源
- 我的收益 登录查看自己的收益
- 我的积分 登录查看自己的积分
- 我的C币 登录后查看C币余额
- 我的收藏
- 我的下载
- 下载帮助
最新资源
- 基于景观生态风险评价的流域景观格局优化,教学视频和资料,喜欢的就下载吧,保证受用
- java设计模式-建造者模式(Builder Pattern)
- C语言刷题-lesson5_1731564764305.pdf
- JavaScript开发指南PDG版最新版本
- JavaScript程序员参考(JavaScriptProgrammer'sReference)pdf文字版最新版本
- jQuery1.4参考指南的实例源代码实例代码最新版本
- CUMCM-2018-D.pdf
- jQueryapi技术文档chm含jQuery选择器使用最新版本
- DWIN_SET.rar
- transformer-transformer
资源上传下载、课程学习等过程中有任何疑问或建议,欢迎提出宝贵意见哦~我们会及时处理!
点击此处反馈
安全验证
文档复制为VIP权益,开通VIP直接复制
信息提交成功