python爬虫(八) 分布式爬虫

分布式原理

将多台主机组合起来，共同完成一个爬取任务，大大提高爬取的效率

分布式爬虫的架构

Redis服务器：

1. 管理Spider服务器请求的url并去重
2. 存储Spider服务器爬下来的数据
   Spider服务器：
3. 从Redis获取请求
4. 把爬取下来的数据发送给Redis服务器

分布式爬虫的优点

1. 可以充分利用多台机器的带宽
2. 可以充分利用多台机器的IP地址
3. 爬取效率高

分布式爬虫必须要解决的问题

1. 分布式爬虫是好几台机器在同时运行，如何保证不同的机器爬取页面的时候不会出现重复数据
2. 分布式爬虫在不同的机器上运行，在把数据爬完后，如何保存在同一个地方

Redis介绍

redis是一种支持分布式的nosql数据库，他的数据是保存在内存中，同时redis可以定时把内存数据同步到磁盘，即可将数据持久化，有丰富的数据结构等

Scrapy-Redis介绍

Scrapy-redis是一个基于redis的分布式爬虫框架。配合scrapy使用，让爬虫具有了分布式爬取的功能

将豆瓣top250保存到redis中

settings

BOT_NAME = 'douban'
SPIDER_MODULES = ['douban.spiders']
NEWSPIDER_MODULE = 'douban.spiders'
USER_AGENT = 'scrapy-redis (+https://github.com/rolando/scrapy-redis)'
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
SCHEDULER_PERSIST = True
ROBOTSTXT_OBEY = False
DOWNLOAD_DELAY = 1
DEFAULT_REQUEST_HEADERS = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language': 'en',# 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'
}
ITEM_PIPELINES = {# 'douban.pipelines.DoubanPipeline': 300,'scrapy_redis.pipelines.RedisPipeline': 400,
}

spider

import scrapy
from bs4 import BeautifulSoup
from ..items import DoubanItem
class DoubanSpider(scrapy.Spider):name = 'douban' #定义爬虫的名称，启动程序时使用addowed_domains=['book.douban.com/top250'] # 定义允许爬虫爬取的域名start_urls = ['https://book.douban.com/top250'] # 定义的起始的网址,告诉爬虫应该从哪个网址开始爬取# parse是scrapy里默认的处理response（响应）的一个方法def parse(self, response, **kwargs):bs = BeautifulSoup(response.text,'html.parser')tr_tags = bs.find_all('tr',class_='item')for tr in tr_tags:item = DoubanItem() # 这个类的对象title = tr.find_all('a')[1]['title'] # 提取书的名称publish = tr.find('p',class_='pl').text # 提取出版信息score = tr.find('span',class_='rating_nums').text # 提取评分信息item['title'] = titleitem['publish'] = publishitem['score'] = score# print([title,publish,score])# 数据封装完毕后，需要提交给引擎yield item

将链家租房信息保存到redis和mongodb中

settings

BOT_NAME = 'lianjia'SPIDER_MODULES = ['lianjia.spiders']
NEWSPIDER_MODULE = 'lianjia.spiders'USER_AGENT = 'scrapy-redis (+https://github.com/rolando/scrapy-redis)'DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
SCHEDULER_PERSIST = TrueROBOTSTXT_OBEY = FalseDOWNLOAD_DELAY = 1DOWNLOADER_MIDDLEWARES = {'lianjia.middlewares.LianjiaDownloaderMiddleware': 300,
}
ITEM_PIPELINES = {'scrapy_redis.pipelines.RedisPipeline': 400,
}

spider

import scrapy
from scrapy_redis.spiders import RedisSpider
class ZufangSpider(RedisSpider):name = 'zufang'allowed_domains = ['lianjia.com']# start_urls = [f'https://bj.lianjia.com/zufang/pg{i}/#contentList' for i in range(1,3)]redis_key = 'myspider:start_urls'index = 2def parse(self, response):base_url = f'https://bj.lianjia.com/zufang/pg{self.index}/#contentList'full_urls=[f'https://bj.lianjia.com{i}' for i in response.xpath('//div[@class="content__list--item--main"]/p[@class="content__list--item--title"]/a/@href').getall()]for item in full_urls:yield scrapy.Request(url=item,callback=self.parse_info,dont_filter=True)yield scrapy.Request(url=base_url,callback=self.parse,dont_filter=True)self.index += 1def parse_info(self,response):title = response.xpath('//p[@class="content__title"]/text()').get() # 标题total_price = response.xpath('//div[@class="content__aside--title"]/span/text()|//div[@class ="content__aside--title"]/text()').getall()price = ''.join(total_price).replace('\n            ','').strip() # 价格mode = response.xpath('//ul[@class="content__aside__list"]/li[1]/text()').get() # 租赁方式type = response.xpath('//ul[@class="content__aside__list"]/li[2]/text()').get() # 房屋类型direction = response.xpath('//ul[@class="content__aside__list"]/li[3]/span[2]/text()').get() # 朝向楼层elevator = response.xpath('//div[@class="content__article__info"]/ul/li[9]/text()').get() # 电梯parking = response.xpath('//div[@class="content__article__info"]/ul/li[11]/text()').get() # 车位water = response.xpath('//div[@class="content__article__info"]/ul/li[12]/text()').get() # 用水electric = response.xpath('//div[@class="content__article__info"]/ul/li[14]/text()').get() # 用电gas = response.xpath('//div[@class="content__article__info"]/ul/li[15]/text()').get() # 燃气heating = response.xpath('//div[@class="content__article__info"]/ul/li[17]/text()').get() # 采暖# print(title)yield {'title':title,'price':price,'mode':mode,'type':type,'direction':direction,'elevator':elevator,'parking':parking,'water':water,'electric':electric,'gas':gas,'heating':heating}

下载器中间件

from fake_useragent import UserAgent
from scrapy import signals
from itemadapter import is_item, ItemAdapter
class LianjiaDownloaderMiddleware:def process_request(self, request, spider):request.headers['User-Agent'] = UserAgent().randomreturn Nonedef process_response(self, request, response, spider):return responsedef process_exception(self, request, exception, spider):passdef spider_opened(self, spider):spider.logger.info('Spider opened: %s' % spider.name)

存到MongoDB的文件

from redis import Redis
from pymongo import MongoClient
import json
# 创建Redis的连接
redis_client = Redis(host='127.0.0.1',port='6379')
# 创建MongoDB的链接
mongo_client= MongoClient()
while True:# 从Redis中取数据try:source,data = redis_client.blpop(['zufang:items'])except:break# print(data)# print(source)lianjia = mongo_client.zufang.lianjia# print(type(data)) # byteschange_data = json.loads(data)# print(change_data)# print(type(change_data)) # dictlianjia.insert_one(change_data)process_exception(self, request, exception, spider):passdef spider_opened(self, spider):spider.logger.info('Spider opened: %s' % spider.name)

存到MongoDB的文件

from redis import Redis
from pymongo import MongoClient
import json
# 创建Redis的连接
redis_client = Redis(host='127.0.0.1',port='6379')
# 创建MongoDB的链接
mongo_client= MongoClient()
while True:# 从Redis中取数据try:source,data = redis_client.blpop(['zufang:items'])except:break# print(data)# print(source)lianjia = mongo_client.zufang.lianjia# print(type(data)) # byteschange_data = json.loads(data)# print(change_data)# print(type(change_data)) # dictlianjia.insert_one(change_data)