Python scrapy.signals 模块，item_scraped() 实例源码
我们从Python开源项目中，提取了以下14个代码示例，用于说明如何使用scrapy.signals.item_scraped()。
def from_crawler(cls, crawler):
instance = cls(crawler.stats)
crawler.signals.connect(instance.item_dropped,
signal=signals.item_dropped)
crawler.signals.connect(instance.item_scraped,
signal=signals.item_scraped)
crawler.signals.connect(instance.response_received,
signal=signals.response_received)
crawler.signals.connect(instance.response_downloaded,
signal=signals.response_downloaded)
crawler.signals.connect(instance.item_saved,
signal=mysignals.item_saved)
crawler.signals.connect(instance.item_saved_Failed,
signal=mysignals.item_saved_Failed)
crawler.signals.connect(instance.html_saved,
signal=mysignals.html_saved)
crawler.signals.connect(instance.html_saved_Failed,
signal=mysignals.html_saved_Failed)
crawler.signals.connect(instance.timeouterror,
signal=mysignals.timeouterror)
crawler.signals.connect(instance.dnslookuperror,
signal=mysignals.dnslookuperror)
return instance
def __init__(self, crawler):
self.crawler = crawler
self.initiatives = 0
self.amendments = 0
self.finishtext = 0
self.responses = 0
self.members = 0
# connect the extension object to signals
crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
def item_scraped(self, item, spider):
if isinstance(item, InitiativeItem):
self.initiatives += 1
elif isinstance(item, AmendmentItem):
self.amendments += 1
elif isinstance(item, FinishTextItem):
self.finishtext += 1
elif isinstance(item, ResponseItem):
self.responses += 1
elif isinstance(item, MemberItem):
self.responses += 1
def setup_redis(self):
"""Setup redis connection and idle signal.
This should be called after the spider has set its crawler object.
"""
if not self.redis_key:
self.redis_key = '%s:start_urls' % self.name
self.server = connection.from_settings(self.crawler.settings)
# idle signal is called when the spider has no requests left,
# that's when we will schedule new requests from redis queue
self.crawler.signals.connect(self.spider_idle, signal=signals.spider_idle)
self.crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
self.log("Reading URLs from redis list '%s'" % self.redis_key)
def item_scraped(self, *args, **kwargs):
"""Avoids waiting for the spider to idle before scheduling the next request"""
self.schedule_next_request()
def from_crawler(cls, crawler):
ext = cls(crawler.stats)
crawler.signals.connect(ext.spider_opened,
signal=signals.spider_opened)
crawler.signals.connect(ext.spider_closed,
signal=signals.spider_closed)
crawler.signals.connect(ext.item_scraped,
signal=signals.item_scraped)
return ext
def from_crawler(cls,
signal=signals.item_scraped)
crawler.signals.connect(instance.response_received,
signal=signals.response_downloaded)
return instance
def item_scraped(self, spider):
#??item??????itempipeline??????
self.stats.inc_value('item/scraped', spider=spider)
def setup_redis(self):
"""Setup redis connection and idle signal.
This should be called after the spider has set its crawler object.
"""
if not self.redis_key:
self.redis_key = '%s:start_urls' % self.name
self.server = connection.from_settings(self.crawler.settings)
# idle signal is called when the spider has no requests left, signal=signals.item_scraped)
self.log("Reading URLs from redis list '%s'" % self.redis_key)
def item_scraped(self, **kwargs):
"""Avoids waiting for the spider to idle before scheduling the next request"""
self.schedule_next_request()
def from_crawler(cls, crawler):
pipeline = cls()
crawler.signals.connect(pipeline.engine_started, signals.engine_started) # ????
crawler.signals.connect(pipeline.engine_stopped, signals.engine_stopped) # ????
crawler.signals.connect(pipeline.item_scraped, signals.item_scraped) # ??????????
crawler.signals.connect(pipeline.item_dropped, signals.item_dropped) # ??????????
crawler.signals.connect(pipeline.spider_opened, signals.spider_opened) # ????????????
crawler.signals.connect(pipeline.spider_closed, signals.spider_closed) # ????????????
crawler.signals.connect(pipeline.spider_idle, signals.spider_idle) # ????
crawler.signals.connect(pipeline.spider_error, signals.spider_error) # ????
crawler.signals.connect(pipeline.request_scheduled, signals.request_scheduled) # ??????
crawler.signals.connect(pipeline.request_dropped, signals.request_dropped) # ??????
crawler.signals.connect(pipeline.response_received, signals.response_received) # ????
crawler.signals.connect(pipeline.response_downloaded, signals.response_downloaded) # ????
return pipeline
版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 [email protected] 举报，一经查实，本站将立刻删除。