效果图：

多线程，代码实现：

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2020/11/14 18:33
# @Author  : huni
# @File    : dianyingtaici.py
# @Software: PyCharmfrom threading import Thread        #多线程的包
from queue import Queue             #队列
from fake_useragent import UserAgent    #模拟请求头的包，可用可不用，我这里没用，自己写的headers
import requests
from lxml import etree
import osclass CrawlInfo(Thread):#重写构造函数def __init__(self,url_queue,html_queue):Thread.__init__(self)#声明两个类属性self.url_queue = url_queueself.html_queue = html_queue#重写run方法def run(self):#爬虫代码headers = {'referer': 'https://www.mingyantong.com/allarticle/jingdiantaici','User-Agent': 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36'}while self.url_queue.empty() == False:          #当url队列中不是空的就继续爬url = self.url_queue.get()                  #从队列中获取一个urlreponse = requests.get(url=url,headers=headers)if reponse.status_code == 200:self.html_queue.put(reponse.text)       #访问成功把html文件放进html队列中class ParseInfo(Thread):def __init__(self,html_queue):Thread.__init__(self)self.html_queue = html_queue#重写run方法def run(self):head = {'referer': 'https://www.mingyantong.com/allarticle/jingdiantaici','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36'}while self.html_queue.empty() == False:e = etree.HTML(self.html_queue.get())           #从html队列中获取一个html使用etree解析#下面的内容可以修改为自己需要解析的内容，比如爬取段子，音频，视频等，只需要在运行前把base_url做更改就可以了movie_list = e.xpath('//*[@id="block-views-xqtermspagearticletype-block_1"]/div/div/div/div/div[2]/div')url_dic = {}for movie in movie_list:movie_name = movie.xpath('./div[2]/span/a/text()')[0]word_num = eval(movie.xpath('./div[2]/div/div[2]/text()')[0])# print(movie_name, word_num)url_dic[movie_name] = word_numpage_url_list = [f'https://www.mingyantong.com/article/{name}?page={page}'for name, num in url_dic.items()for page in range(0, (int(num) // 10) + 1)]# print(page_url_list)for page_url in page_url_list:page_text = requests.get(url=page_url, headers=head).texttree1 = etree.HTML(page_text)div_list = tree1.xpath('//div[@class="view-content"]/div')with open('./电影台词.txt', 'a', encoding='utf-8') as fp:for div in div_list:author = div.xpath('./div/div[2]/a//text()')movie = div.xpath('./div/div[2]/span/a//text()')content = div.xpath('./div/div[1]/a//text()')if (len(author) != 0 or len(movie) != 0) and '简介' not in author:# print(author, movie, content)author = ''.join(author)movie = ''.join(movie)for i in content:if len(content) == 1:fp.write(i + '\n')print('文本写入成功')else:fp.write(i)print('文本写入成功')if len(content) == 1:fp.write(' ' * 65 + '-' * 3 + author + '《' + movie + '》' + '\n')print('作者写入成功')else:fp.write('\n' + ' ' * 65 + '-' * 3 + author + '《' + movie + '》' + '\n')print('作者写入成功')if __name__ == '__main__':# if not os.path.exists('./4Ktupian'):#     os.mkdir('./4Ktupian')#创建一个存储有url和html的容器：队列url_queue = Queue()html_queue = Queue()base_url = 'https://www.mingyantong.com/allarticle/jingdiantaici?page={}'      #占位符填充for i in range(1,6):new_url = base_url.format(i)        #占位符填充url_queue.put(new_url)crawl_list = []             #创建三个线程，加到线程列表中for i in range(10):Crawl = CrawlInfo(url_queue,html_queue)crawl_list.append(Crawl)Crawl.start()for crawl in crawl_list:#等待操作，可以理解成url队列解析完了之后，需要等待一会，再交给html队列解析内容crawl.join()parse_list = []for i in range(10):parse = ParseInfo(html_queue)parse_list.append(parse)parse.start()for parse in parse_list:parse.join()