1.抓取首页

def get_one_page(url):try:headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36'}response =requests.get(url,headers=headers)if response.status_code == 200:return response.textreturn Noneexcept  RequestException:return None

2.正则提取

所以这里的正则表达式为：<div class="name">(.*?)</div>

使用迭代器

def parse_one_page(html):pattern =re.compile('<div class="name">(.*?)</div>',re.S)items=re.findall(pattern,html)for item in items:yield item

3.写入文件

def write_to_file(content):with open('D:/result.txt','a',encoding='utf-8')as f:f.write(json.dumps(content,ensure_ascii=False)+'\n')

4.整合代码

def main():offset=1url = 'https://read.douban.com/provider/all'html = get_one_page(url)for item in parse_one_page(html):print(item)write_to_file(item)

完整代码

import json
import requests
from requests.exceptions import RequestException
import re
import time
def get_one_page(url):try:headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36'}response =requests.get(url,headers=headers)if response.status_code == 200:return response.textreturn Noneexcept  RequestException:return Nonedef parse_one_page(html):pattern =re.compile('<div class="name">(.*?)</div>',re.S)items=re.findall(pattern,html)for item in items:yield item
def write_to_file(content):with open('D:/result.txt','a',encoding='utf-8')as f:f.write(json.dumps(content,ensure_ascii=False)+'\n')def main():offset=1url = 'https://read.douban.com/provider/all'html = get_one_page(url)for item in parse_one_page(html):print(item)write_to_file(item)if __name__ == "__main__":main()

运行结果