学了一段时间python了,网上的公开课也看了不少,课程都听得懂,但是一想写点什么就不知道从何下手、觉得太麻烦,现在终于狠下心走出舒适区,动手写了第一个爬虫。

爬取目标

人民日报、上观新闻、观察者网、澎湃新闻
主要爬取了涉及时事、政治方面的几个板块。
因为新闻页面 html 比较好解析，而且不需要反爬。规避了很多困难。

代码

我把代码封装了一下，想试试的朋友可以直接下载，然后import 来使用。
毕竟还是初学者，觉得不同类之间的代码重复率很高，可能写了很多无用代码。 还请大家多多提出修改意见。

不多BB,上代码了。

爬取过程主要用到了requests和BeautifulSoup 库
数据最终存储在sqlite3 数据库里

# #!/usr/bin/env python
# # coding: utf-8
import requests
from bs4 import BeautifulSoup as bs
import re
import os
import time
import sqlite3
import datetime
def how():how =  '''(1)使用方法: update_info(['人民日报-时政','上观新闻-政情','观察者网-国际','澎湃新闻-时事'],3)\n-> 爬取这四个网站对应版块共[3]页的新闻信息\n(2)使用方法: update_content(['人民日报','澎湃新闻'])\n-> 更新这两个网站对应已爬取信息的新闻内容\n'''print(how)info = {'人民日报':['时政', '国际', '台湾', '军事','社会', '法治'],'上观新闻':['要闻', '政情', '天下'],'观察者网':['国际', '军事', '科技'],'澎湃新闻':['时事']
}def update_info(goal, page_num):'''如果 传入 update_info(['人民日报','上观新闻'])-> 爬取 这两个网站的新闻'''DB_Name = 'datasource.db'table_name = 'news_info'conn = sqlite3.connect(DB_Name)# print('连接数据库%s成功' % (DB_Name))cursor = conn.cursor()conn.execute('Pragma foreign_keys = ON;')# 创建表create = f'''Create Table if not exists news_info (info_ID Integer Primary key Autoincrement,Source Varchar(20), Title Varchar(100),Date char(8), Author Varchar(20), Url Varchar(100), updated varchar(8), Site Varchar(8))'''# 建立news_info 表 6列: info_ID(主) , 来源, 标题, 日期, 作者, URL, 是否有内容conn.execute(create)cursor.close()conn.close()for each in goal:page_info = each.split('-')[1]page_key = each.split('-')[0]if page_key =='人民日报':s = Rm_Spider(page_info,page_num)s.crawl_info()elif page_key == '上观新闻':s = Sg_Spider(page_info,page_num)s.crawl_info()elif page_key == '观察者网':s = Gc_Spider(page_info,page_num)s.crawl_info()elif page_key == '澎湃新闻':s = Pp_Spider(page_info,page_num)s.crawl_info()def update_content(goal):DB_Name = 'datasource.db'table_name = 'news_info'conn = sqlite3.connect(DB_Name)cursor = conn.cursor()conn.execute('Pragma foreign_keys = ON;')# 创建表conn.execute('''Create Table if not exists news_content (content_ID Integer Primary key Autoincrement,info_ID Integer,title Text, body Text, content Text,Foreign Key (info_ID) References news_info(info_ID));''')# 建立 news_content 表 4列 : content_ID(主),info_ID(外), 正文, 文本cursor.close()conn.close()for each in goal:if each == '人民日报':s = Rm_Spider('时政',1)s.crawl_content()s.updated()elif each == '上观新闻':s = Sg_Spider('政情', 1)s.crawl_content()s.updated()elif each == '观察者网':s = Gc_Spider('国际', 1)s.crawl_content()s.updated()elif each == '澎湃新闻':s = Pp_Spider('时事', 1)s.crawl_content()s.updated()
#人民日报虫
class Rm_Spider():def __init__(self,page_info,page_num):self.info = {'时政': 'politics', '国际': 'world', '台湾': 'tw', '军事': 'military','社会': 'society', '法治': 'legal'}self.page_info = self.info[page_info]self.page_num = page_numdef get_pagelist(self):page_info = self.page_infonum = self.page_numpagelist = []for index in range(1,num+1):if index == 1:page = f'http://{page_info}.people.com.cn/'else:page = f'http://{page_info}.people.com.cn/index{index}.html'pagelist.append(page)return pagelistdef get_html(self,url):response = requests.get(url)response.encoding = 'GBK'if response.status_code == 200:return response.textelse:return Nonedef get_linklist(self, page_html, page_info):#传入 get_html return的结果, self.page_infopage_info = self.page_infopattern = re.compile(r'^/n1(.*?)html$')soup = bs(page_html, 'lxml')  # 这是一个 bs实例对象link_list = soup.find_all('a', href=pattern)linklist = []for link in link_list:link = f'http://{page_info}.people.com.cn/{link.get("href")}'if link not in linklist:linklist.append(link)return linklistdef get_info(self, html, link):'''返回得到的新闻 来源,标题,日期,作者,url,有木有内容参数是get_html得到的html解析'''bsobj = bs(html, 'lxml')try:source = bsobj.find('div', class_="box01").div.a.text.replace('\u200b','').replace('\u3000', '').replace('\xa0', '').replace('\t', '').replace('\r', '')title = bsobj.h1.text.replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '')author = bsobj.find('div', class_="edit clearfix").text.strip('(责编：)').replace('\u200b','').replace('\u3000', '').replace('\xa0', '').replace('\t', '').replace('\r', '')url = linkdate_str = url.split('/')[5] + url.split('/')[6]  # date 是 发布日期  link 是 urldate = datetime.datetime.strptime(date_str, "%Y%m%d").strftime('%Y-%m-%d')return source, title, author, date, urlexcept AttributeError:return None, None, None, None, Nonedef news_info_save(self, source, title, date, author, url):'''存入sqlite数据库中的news_info表'''news_info = 'Insert Into news_info(source, title, date, author, url, Site) Values (?,?,?,?,?,"人民日报");'self.cursor.execute(news_info, (source, title, date, author, url))self.conn.commit()def crawl_info(self):DB_Name = 'datasource.db'self.conn = sqlite3.connect(DB_Name)
#         print('Rm_Spider连接数据库%s成功' % (DB_Name))self.cursor = self.conn.cursor()self.conn.execute('Pragma foreign_keys = ON;')page_info = self.page_infonum = self.page_numpage_list = self.get_pagelist()  # page_info , numcount = 0print('人民日报--信息爬取开始')for each in page_list:html_code = self.get_html(each)  # 把每一个新闻版面的html代码拿到linklist = self.get_linklist(html_code,page_info)  # 拿到所有的链接for link in linklist:html_content = self.get_html(link)  # 拿到每一篇新闻的html代码news_source, news_title, news_date, news_author, news_url = self.get_info(html_content,link)  # 拿到写入文件的内容, 标题if news_source:count += 1self.news_info_save(news_source, news_title, news_date, news_author, news_url)print(f'{count}-{news_title}  信息爬取完成!'.ljust(100), end='\r')time.sleep(1)self.cursor.close()self.conn.close()print('\n结束')def get_content(self, html):'''返回得到的新闻正文,文本参数是get_html得到的html解析'''bsobj = bs(html, 'lxml')try:body = str(bsobj.find('div', id='rwb_zw')).replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '')  # 正文title = bsobj.h1.text.replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '')plist = bsobj.find('div', id='rwb_zw').find_all('p')  #result = ''for each in plist:clean_each = each.text.strip('\r\n\t\u3000')result += f'{clean_each}\n'content = f'{title}\n{result}'.replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '')  # 文本return title, body, contentexcept AttributeError:return None, None, Nonedef news_content_save(self, info_ID, title, body, content):'''存入sqlite数据库中的news_content表'''news_content = 'Insert Into news_content(info_ID,title, body,content ) Values (?,?,?,?);'self.cursor.execute(news_content, (info_ID, title, body, content))self.conn.commit()def updated(self):DB_Name = 'datasource.db'conn1 = sqlite3.connect(DB_Name)cursor1 = conn1.cursor()cursor1.execute('Update news_info Set updated = "已更新" Where updated is null and Site = "人民日报"')conn1.commit()cursor1.close()conn1.close()def crawl_content(self):DB_Name = 'datasource.db'self.conn = sqlite3.connect(DB_Name)
#         print('Rm_Spider连接数据库%s成功' % (DB_Name))self.cursor = self.conn.cursor()self.conn.execute('Pragma foreign_keys = ON;')count = 0print('人民日报--内容爬取开始')sql = 'Select url, Info_ID From news_info Where site = "人民日报" and updated is null'url_list = self.cursor.execute(sql).fetchall()for each in url_list:link = each[0]info_ID = each[1]html_content = self.get_html(link)news_title, news_body, news_content = self.get_content(html_content)if news_title:count += 1self.news_content_save(info_ID, news_title,news_body, news_content)print(f'{count}-{news_title}  内容爬取完成!'.ljust(100), end='\r')time.sleep(1)self.cursor.close()self.conn.close()print('\n结束')
#上观新闻虫
class Sg_Spider():def __init__(self, page_info, page_num):self.info = {'要闻': '42', '政情': '1', '天下': '21'}self.page_info = self.info[page_info]self.page_num = page_numdef get_pagelist(self):page_info = self.page_infonum = self.page_numpagelist = []for index in range(1, num + 1):page = f'https://www.jfdaily.com/news/list?section={page_info}&page={index}'pagelist.append(page)return pagelistdef get_html(self, url):'''获取各版面的html代码参数:待爬取版面url'''response = requests.get(url)response.encoding = 'utf-8'if response.status_code == 200:return response.textelse:return Nonedef get_linklist(self, page_html):'''获取版面下各个新闻的链接参数为版面的html代码'''bsobj = bs(page_html, 'lxml')link_list = bsobj.find_all('div', class_="chengshi_img")#     print(each.a.get('href'))linklist = []for each in link_list:tail = each.a.get('href')link = f'https://www.jfdaily.com{tail}'if link not in linklist:linklist.append(link)return linklistdef get_info(self, html, link):bsobj = bs(html, 'lxml')date_pattern = re.compile(r'\d{4}-\d{2}-\d{2}')try:source = bsobj.find('span', class_="max-words").text.replace('\u200b','').replace('\u3000', '').replace('\xa0', '').replace('\t', '').replace('\r', '')title = bsobj.title.text.replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '')  # 标题author = bsobj.find('div', class_="fenxiang_zz").find_all('span')[1].text.replace('\u200b','').replace('\u3000', '').replace('\xa0', '').replace('\t', '').replace('\r', '')url = linkdate = date_pattern.findall(bsobj.find('div', class_="fenxiang_zz").text)[0]return source, title, author, date, urlexcept AttributeError:return None, None, None, None, Nonedef get_content(self, html):bsobj = bs(html, 'lxml')try:title = bsobj.title.text.replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '')   # 标题body = str(bsobj.find('div', id="newscontents")).replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '')  # 正文plist = bsobj.find('div', id="newscontents").find_all('p')  # 文本列表pre_content = ''for p in plist:clean_p = p.text.strip("\r\n\t")pre_content += f'{clean_p}\n'content = f'{title}\n{pre_content}'.replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '')  # 文本结果return title, body, contentexcept AttributeError:return None, None, Nonedef news_info_save(self, source, title, date, author, url):'''存入sqlite数据库中的news_info表'''news_info = 'Insert Into news_info(source, title, date, author, url,Site) Values (?,?,?,?,?,"上观新闻");'self.cursor.execute(news_info, (source, title, date, author, url))self.conn.commit()def news_content_save(self, info_ID, title, body, content):'''存入sqlite数据库中的news_content表'''news_content = 'Insert Into news_content(info_ID,title, body,content ) Values (?,?,?,?);'self.cursor.execute(news_content, (info_ID, title, body, content))self.conn.commit()def updated(self):DB_Name = 'datasource.db'conn1 = sqlite3.connect(DB_Name)cursor1 = conn1.cursor()cursor1.execute('Update news_info Set updated = "已更新" Where updated is null and Site = "上观新闻"')conn1.commit()cursor1.close()conn1.close()def crawl_info(self):DB_Name = 'datasource.db'self.conn = sqlite3.connect(DB_Name)
#         print('Sg_Spider连接数据库%s成功' % (DB_Name))self.cursor = self.conn.cursor()self.conn.execute('Pragma foreign_keys = ON;')page_info = self.page_infonum = self.page_numpage_list = self.get_pagelist()count = 0print('上观新闻--信息爬取开始')for each in page_list:html_code = self.get_html(each)  # 把每一个新闻版面的html代码拿到linklist = self.get_linklist(html_code)  # 拿到所有的链接for link in linklist:html_content = self.get_html(link)  # 拿到每一篇新闻的html代码news_source, news_title, news_date, news_author, news_url = self.get_info(html_content,link)  # 拿到写入文件的内容, 标题if news_source:count += 1self.news_info_save(news_source, news_title, news_date, news_author, news_url)print(f'{count}-{news_title}  信息爬取完成!'.ljust(100), end='\r')time.sleep(1)#关闭数据库self.cursor.close()self.conn.close()print('\n结束')def crawl_content(self):DB_Name = 'datasource.db'self.conn = sqlite3.connect(DB_Name)
#         print('Sg_Spider连接数据库%s成功' % (DB_Name))self.cursor = self.conn.cursor()self.conn.execute('Pragma foreign_keys = ON;')count = 0print('上观新闻--内容爬取开始')sql = 'Select url, Info_ID From news_info Where site = "上观新闻" and updated is null'url_list = self.cursor.execute(sql).fetchall()for each in url_list:link = each[0]info_ID = each[1]html_content = self.get_html(link)news_title, news_body, news_content = self.get_content(html_content)if news_title:count += 1self.news_content_save(info_ID, news_title,news_body, news_content)print(f'{count}-{news_title}  内容爬取完成!'.ljust(100), end='\r')time.sleep(1)self.cursor.close()self.conn.close()print('\n结束')
#观察者网虫
class Gc_Spider():def __init__(self, page_info, page_num):self.info = {'国际': 'GuoJi·ZhanLue', '军事': 'JunShi', '科技': 'GongYe·KeJi'}self.page_info = self.info[page_info]self.page_num = page_numdef get_pagelist(self):page_info = self.page_infonum = self.page_numpagelist = []for index in range(1, num + 1):page = f'https://www.guancha.cn/{page_info}/list_{index}.shtml'pagelist.append(page)return pagelistdef get_html(self, url):response = requests.get(url)response.encoding = 'utf-8'if response.status_code == 200:return response.textelse:return Nonedef get_linklist(self, page_html):bsobj = bs(page_html, 'lxml')linklist = []h4s = bsobj.find('ul', class_="column-list fix").find_all('h4')for h4 in h4s:link = f'https://www.guancha.cn{h4.a.get("href")}'linklist.append(link)return linklistdef get_info(self, html, link):bsobj = bs(html, 'lxml')try:title = bsobj.title.text.replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '') # 标题url = linksource = bsobj.find('div', class_="time fix").find_all('span')[-1].text.strip('来源：').replace('\u200b','').replace('\u3000', '').replace('\xa0', '').replace('\t', '').replace('\r', '')author = bsobj.find('ul', class_="article-other").li.text.strip('责任编辑:\r\n\t\t\t&nbsp').replace('\u200b','').replace('\u3000', '').replace('\xa0', '').replace('\t', '').replace('\r', '')date_list = link.split('/')[-1].split('_')date = f'{date_list[0]}-{date_list[1]}-{date_list[2]}'return source, title, author, date, urlexcept AttributeError:return None, None, None, None, Nonedef get_content(self, html):bsobj = bs(html, 'lxml')try:title = bsobj.title.text.replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ')  # 标题body = str(bsobj.find('div', class_="content all-txt")).replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '') # 正文plist = bsobj.find('div', class_='content all-txt').find_all('p')  # 文本列表pre_content = ''for p in plist:clean_p = p.text.strip("\r\n\t")pre_content += f'{clean_p}\n'content = f'{title}\n{pre_content}'.replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '')  # 文本结果return title, body, contentexcept AttributeError:return None, None, Nonedef news_info_save(self, source, title, date, author, url):'''存入sqlite数据库中的news_info表'''news_info = 'Insert Into news_info(source, title, date, author, url,Site) Values (?,?,?,?,?,"观察者网");'self.cursor.execute(news_info, (source, title, date, author, url))self.conn.commit()def news_content_save(self, info_ID, title, body, content):'''存入sqlite数据库中的news_content表'''news_content = 'Insert Into news_content(info_ID,title, body,content ) Values (?,?,?,?);'self.cursor.execute(news_content, (info_ID, title, body, content))self.conn.commit()def updated(self):DB_Name = 'datasource.db'conn1 = sqlite3.connect(DB_Name)cursor1 = conn1.cursor()cursor1.execute('Update news_info Set updated = "已更新" Where updated is null and Site = "观察者网"')conn1.commit()cursor1.close()conn1.close()def crawl_info(self):DB_Name = 'datasource.db'table_name = 'news_info'self.conn = sqlite3.connect(DB_Name)
#         print('Gc_Spider连接数据库%s成功' % (DB_Name))self.cursor = self.conn.cursor()self.conn.execute('Pragma foreign_keys = ON;')page_info = self.page_infonum = self.page_numpage_list = self.get_pagelist()count = 0print('观察者网--信息爬取开始')for each in page_list:html_code = self.get_html(each)  # 把每一个新闻版面的html代码拿到linklist = self.get_linklist(html_code)  # 拿到所有的链接for link in linklist:html_content = self.get_html(link)  # 拿到每一篇新闻的html代码news_source, news_title, news_date, news_author, news_url = self.get_info(html_content,link)  # 拿到写入文件的内容, 标题if news_source:count += 1self.news_info_save(news_source, news_title, news_date, news_author, news_url)print(f'{count}-{news_title}  信息爬取完成!'.ljust(100), end='\r')time.sleep(1)self.cursor.close()self.conn.close()print('\n结束')def crawl_content(self):DB_Name = 'datasource.db'self.conn = sqlite3.connect(DB_Name)
#         print('Gc_Spider连接数据库%s成功' % (DB_Name))self.cursor = self.conn.cursor()self.conn.execute('Pragma foreign_keys = ON;')count = 0print('观察者网--内容爬取开始')sql = 'Select url, Info_ID From news_info Where site = "观察者网" and updated is null'url_list = self.cursor.execute(sql).fetchall()for each in url_list:link = each[0]info_ID = each[1]html_content = self.get_html(link)news_title, news_body, news_content = self.get_content(html_content)if news_title:count += 1self.news_content_save(info_ID, news_title,news_body, news_content)print(f'{count}-{news_title}  内容爬取完成!'.ljust(100), end='\r')time.sleep(1)self.cursor.close()self.conn.close()print('\n结束')
#澎湃新闻虫
class Pp_Spider():def __init__(self, page_info, page_num):self.info = {'时事': '''/load_index.jsp?nodeids=25462,25488,25489,25490,25423,25426,25424,25463,25491,25428,68750,27604,25464,25425,25429,25481,25430,25678,25427,25422,25487,25634,25635,25600,&channelID=25950&topCids=,8173176,8173073,8172980,8173179,8173070,8173170,8173132'''}self.page_info = self.info[page_info]self.page_num = page_numdef get_pagelist(self):page_info = self.page_infonum = self.page_numpagelist = []for index in range(1, num + 1):page = f'https://www.thepaper.cn/{page_info}&pageidx={index}'pagelist.append(page)return pagelistdef get_html(self, url):response = requests.get(url)response.encoding = 'utf-8'if response.status_code == 200:return response.textelse:return Nonedef get_linklist(self, page_html):bsobj = bs(page_html, 'lxml')link_list = bsobj.find_all('div', class_="news_li")#     print(each.a.get('href'))linklist = []for each in link_list:tail = each.a.get('href')link = f'https://www.thepaper.cn/{tail}'if link not in linklist:linklist.append(link)return linklistdef get_info(self, html, link):bsobj = bs(html, 'lxml')date_pattern = re.compile(r'\d{4}-\d{2}-\d{2}')try:title = bsobj.title.text.replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '')url = linksource = bsobj.find('div', class_="news_about").find_all('p')[0].text.replace('\u200b','').replace('\u3000', '').replace('\xa0', '').replace('\t', '').replace('\r', '')author = bsobj.find('div', class_="infor_item").text.strip('责任编辑：').replace('\u200b','').replace('\u3000', '').replace('\xa0', '').replace('\t', '').replace('\r', '')date = date_pattern.findall(bsobj.find('div', class_="news_about").find_all('p')[1].text)[0]return source, title, author, date, urlexcept AttributeError:return None, None, None, None, Nonedef get_content(self, html):def extract_imgtext(body):'''去除掉图片下面的文字的算法'''soup = bodyif soup('span'):for each in soup('span'):each.extract()if soup('p'):for each in soup('p'):each.extract()if soup('font'):for each in soup('font'):each.extract()if soup('div'):for each in soup('div'):each.extract()if soup('img'):for each in soup('img'):each.extract()return soupbsobj = bs(html, 'lxml')title = bsobj.title.text.replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '')body = str(bsobj.find('div', class_="news_txt")).replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '')body_soup = bsobj.find('div', class_="news_txt")soup = extract_imgtext(body_soup)#     try:# print(sou.contents)for index in range(len(soup.contents)):#         print(soup.contents[index])soup.contents[index] = str(soup.contents[index]).strip('<strong></strong>')#         print(soup.contents[index])if soup.contents[index] == 'b':soup.contents[index] = '\n'#         print(soup.contents[index])soup.contents[index].strip('<strong>')#     print(soup.contents)content_dirty = f'{title}\n'for each in soup.contents:content_dirty += eachcontent = content_dirty.replace('\u200b','').replace('\u3000', '').replace('\xa0', ' ').replace('\t', '').replace('\r', '')return title, body, contentdef crawl_info(self):DB_Name = 'datasource.db'table_name = 'news_info'self.conn = sqlite3.connect(DB_Name)
#         print('Pp_Spider连接数据库%s成功' % (DB_Name))self.cursor = self.conn.cursor()self.conn.execute('Pragma foreign_keys = ON;')page_info = self.page_infonum = self.page_numpage_list = self.get_pagelist()count = 0print('澎湃新闻--信息爬取开始')for each in page_list:html_code = self.get_html(each)  # 把每一个新闻版面的html代码拿到linklist = self.get_linklist(html_code)  # 拿到所有的链接for link in linklist:html_content = self.get_html(link)  # 拿到每一篇新闻的html代码news_source, news_title, news_date, news_author, news_url = self.get_info(html_content, link)if news_source:count += 1self.news_info_save(news_source, news_title, news_date, news_author, news_url)print(f'{count}-{news_title}  信息爬取完成!'.ljust(100), end='\r')time.sleep(1)self.cursor.close()self.conn.close()print('\n结束')def news_info_save(self, source, title, date, author, url):'''存入sqlite数据库中的news_info表'''news_info = 'Insert Into news_info(source, title, date, author, url,Site) Values (?,?,?,?,?,"澎湃新闻");'self.cursor.execute(news_info, (source, title, date, author, url))self.conn.commit()def news_content_save(self, info_ID, title, body, content):'''存入sqlite数据库中的news_content表'''news_content = 'Insert Into news_content(info_ID,title, body,content ) Values (?,?,?,?);'self.cursor.execute(news_content, (info_ID, title, body, content))self.conn.commit()def updated(self):DB_Name = 'datasource.db'conn1 = sqlite3.connect(DB_Name)cursor1 = conn1.cursor()cursor1.execute('Update news_info Set updated = "已更新" Where updated is null and Site = "澎湃新闻"')conn1.commit()cursor1.close()conn1.close()def crawl_content(self):DB_Name = 'datasource.db'self.conn = sqlite3.connect(DB_Name)
#         print('Pp_Spider连接数据库%s成功' % (DB_Name))self.cursor = self.conn.cursor()self.conn.execute('Pragma foreign_keys = ON;')count = 0print('澎湃新闻--内容爬取开始')sql = 'Select url, Info_ID From news_info Where site = "澎湃新闻" and updated is null'url_list = self.cursor.execute(sql).fetchall()for each in url_list:link = each[0]info_ID = each[1]html_content = self.get_html(link)news_title, news_body, news_content = self.get_content(html_content)if news_title:count += 1self.news_content_save(info_ID, news_title,news_body, news_content)print(f'{count}-{news_title}  内容爬取完成!'.ljust(100), end='\r')time.sleep(1)self.cursor.close()self.conn.close()print('\n结束')

创建一个表格

一个简单的表格是这么创建的：

项目	Value
电脑	$1600
手机	$12
导管	$1