利用python编写爬虫程序，从招聘网站上爬取数据，将数据存入到MongoDB数据库中，将存入的数据作一定的数据清洗后做数据分析，最后将分析的结果做数据可视化。
1.分析网页，我选择全部在详情页爬取
爬取字段：岗位名称，薪资水平，招聘单位，工作地点，工作经验，学历要求
编写items.py

import scrapy
class Lagou01Item(scrapy.Item):# define the fields for your item here like:# name = scrapy.Field()name = scrapy.Field()money = scrapy.Field()danwei = scrapy.Field()didian = scrapy.Field()jinyan = scrapy.Field()xueli = scrapy.Field()zhizhe = scrapy.Field()

编写spiders.py
限制爬取页数

start_urls = ['https://www.lagou.com/zhaopin/Python/' + str(x) for x in range(1, 22)]

爬取各字段

item['name'] = job.xpath('div[1]/div[1]/div[1]/a/h3/text()').extract_first()  # 职位item['money'] = job.xpath('div[1]/div[1]/div[2]/div/span/text()').extract_first()  # 工资item['danwei'] = job.xpath('div[1]/div[2]/div[1]/a/text()').extract_first()  # 公司名字item['didian'] = job.xpath('div[1]/div[1]/div[1]/a/span/em/text()').extract_first()  # 地点

进入详情页爬取剩余字段

# 详情页的url
url = job.xpath('div[1]/div[1]/div[1]/a/@href').extract_first()

完整的spiders.py代码

# -*- coding: utf-8 -*-
import scrapy
from lagou01.items import Lagou01Item
from scrapy.spiders import CrawlSpider
from scrapy import Requestclass LagouSpider(CrawlSpider):name = 'lagou'allowed_domains = ['lagou.com']start_urls = ['https://www.lagou.com/zhaopin/Python/' + str(x) for x in range(1, 22)]def parse(self, response):joblist = response.xpath('//*[@class="con_list_item default_list"]')for job in joblist:item = Lagou01Item()item['name'] = job.xpath('div[1]/div[1]/div[1]/a/h3/text()').extract_first()  # 职位item['money'] = job.xpath('div[1]/div[1]/div[2]/div/span/text()').extract_first()  # 工资item['danwei'] = job.xpath('div[1]/div[2]/div[1]/a/text()').extract_first()  # 公司名字item['didian'] = job.xpath('div[1]/div[1]/div[1]/a/span/em/text()').extract_first()  # 地点print("职务名称：", item['name'])print("薪资水平：", item['money'])print("招聘单位：", item['danwei'])print("工作地点：", item['didian'])# 详情页的urlurl = job.xpath('div[1]/div[1]/div[1]/a/@href').extract_first()# 继续请求详情页URL，用用meta传递已经爬取到的部分数据# 使用callback指定回调函数yield Request(url=url, meta={'item': item}, callback=self.parse_detail)def parse_detail(self, response):# 接受传递过来的数据item = response.meta['item']# 继续向Item添加信息item['jinyan'] = response.xpath('//*[@class="job_request"]/h3/span[3]/text()').extract_first()  # 工作经验item['xueli'] = response.xpath('//*[@class="job_request"]/h3/span[4]/text()').extract_first()  # 学历要求# 任职要求zhizhes = response.xpath('//*[@class="job_bt"]/div[1]/p/text()').getall()zhizhe_str = ""for zhizhe in zhizhes:zhizhe_str += zhizhe.strip()item["zhizhe"] = zhizhe_strprint('工作经验:', item['jinyan'])print('学历要求:', item['xueli'])print('任职要求:', item['zhizhe'])yield item

编写pipelines.py

import pymongo
from pymongo import MongoClient
class Lagou01Pipeline(object):#在open_spider方法中连接MongoDB，创建数据库和集合，也可以在__init__初始化方法中处理这些操作def open_spider(self,spider):self.db = MongoClient('localhost',27017).lagou_dbself.collection = self.db.lagou_collectiondef process_item(self, item, spider):#把Item转化成字典方式，然后添加数据self.collection.insert_one(dict(item))return item

编写settings.py

USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.14 Safari/537.36'
ROBOTSTXT_OBEY = False
COOKIES_ENABLED = False
#设置爬取时间间隔
DOWNLOAD_DELAY = 2

运行结果

(base) D:\requests1>cd lagou01
(base) D:\requests1\lagou01>scrapy crawl lagou

MongoDB中保存的数据