前言

由于ScienceDirect网站一次性只能导出100份文献的索引，有使用者会一页一页的导出，总共60页的导出效率极低。如果需要多关键词的大批量导出文献索引，则需要用到python来提高效率。

相关库

from selenium import webdriver
import time
import re
from urllib import parse
import os

分解代码

爬取文献

    def downloadThisPage(self,page,keyword):global driverdriver.get('https://www.sciencedirect.com/search?qs={}&show=100&offset={}'.format(keyword,page))time.sleep(1)if self.new == 1:self.pages = driver.find_element_by_id('srp-pagination')self.pages = self.pages.textself.pages = re.findall('Page 1 of (.*?)next', self.pages)[0]self.pages = int(self.pages) * 100		#获取总共有多少页if self.ifLogin == 0:input('请在登录完成后在本界面按回车键')		#第一次运行代码时需要在chrome界面登录网页selectAllResult = driver.find_element_by_css_selector('div.u-show-from-sm>div>div>label.checkbox-label>span.checkbox-check')selectAllResult.click()time.sleep(1)exportAllItems = driver.find_element_by_css_selector('div.ExportAllLink>button.export-all-link-button>span.button-link-text>span.export-all-link-text')exportAllItems.click()time.sleep(1)exportRIS = driver.find_elements_by_css_selector('div.export-options>button.button-link-primary>span.button-link-text')[1]exportRIS.click()time.sleep(1)self.ifLogin = 1self.new = 0

编码关键词

    def getKeyWord(self):for item in keywordList:self.currentKeyWord = itemself.new = 1keyWord = itemkeyWordTranslate = parse.quote(keyWord)		#将字符串编码为URL编码格式self.main(keyWordTranslate)

合并并删除所有RIS文件

    def emergeAllFile(self):allFiles = []chosenFiles = []for root, dirs, files in os.walk(r"C:\Users\t\Downloads"):for file in files:allFiles.append(os.path.join(root, file))for item in allFiles:item = item.split('.')if item[-1] == 'ris':chosenFiles.append('.'.join(item))		#判断是否为RIS文件并将RIS文件路径加入列表originalFile = open('{}.ris'.format(self.currentKeyWord), 'ab')		#在代码本地新建RIS文件for info in chosenFiles:singleFile = open(info, 'rb', )fileContent = singleFile.read()originalFile.write(fileContent)singleFile.close()fileContent = ''originalFile.close()		#将所有RIS文件内容写入新建的文件中filepath = r'C:\Users\t\Downloads'del_list = os.listdir(filepath)for f in del_list:		file_path = os.path.join(filepath, f)if os.path.isfile(file_path):os.remove(file_path)		#删除所有文件

完整代码

from selenium import webdriver
import time
import re
from urllib import parse
import oskeywordList = ['class structure China','social structure china','occupational structure china','Social Economic Index china'\,'International Standard Classification of Occupations china ','occupational categories china','occupational hierarchies china'\,'occupational classification china','occupational stratification china','occupational class china','social class china','occupational position china'\,'socioeconomic position china','socioeconomic status china','occupational status china','occupational strata china','social stratum china','social status china'\,'occupational prestige china','prestige hierarchies china','occupational mobility china','social mobility china','occupational segregation china']class search():def __init__(self):self.ifLogin = 0self.new = 0def getKeyWord(self):for item in keywordList:self.currentKeyWord = itemself.new = 1keyWord = itemkeyWordTranslate = parse.quote(keyWord)self.main(keyWordTranslate)def main(self,keyword):page = 0self.pages = 1000while page < self.pages:self.downloadThisPage(page,keyword)page += 100print('关键词{}共{}页爬取完毕'.format(self.currentKeyWord,self.pages))print('正在整合并删除文件')self.emergeAllFile()def emergeAllFile(self):allFiles = []chosenFiles = []for root, dirs, files in os.walk(r"C:\Users\t\Downloads"):for file in files:allFiles.append(os.path.join(root, file))for item in allFiles:item = item.split('.')if item[-1] == 'ris':chosenFiles.append('.'.join(item))originalFile = open('{}.ris'.format(self.currentKeyWord), 'ab')for info in chosenFiles:singleFile = open(info, 'rb', )fileContent = singleFile.read()originalFile.write(fileContent)singleFile.close()fileContent = ''originalFile.close()filepath = r'C:\Users\t\Downloads'del_list = os.listdir(filepath)for f in del_list:file_path = os.path.join(filepath, f)if os.path.isfile(file_path):os.remove(file_path)def downloadThisPage(self,page,keyword):global driverdriver.get('https://www.sciencedirect.com/search?qs={}&show=100&offset={}'.format(keyword,page))time.sleep(1)if self.new == 1:self.pages = driver.find_element_by_id('srp-pagination')self.pages = self.pages.textself.pages = re.findall('Page 1 of (.*?)next', self.pages)[0]self.pages = int(self.pages) * 100if self.ifLogin == 0:input('请在登录完成后在本界面按回车键')selectAllResult = driver.find_element_by_css_selector('div.u-show-from-sm>div>div>label.checkbox-label>span.checkbox-check')selectAllResult.click()time.sleep(1)exportAllItems = driver.find_element_by_css_selector('div.ExportAllLink>button.export-all-link-button>span.button-link-text>span.export-all-link-text')exportAllItems.click()time.sleep(1)exportRIS = driver.find_elements_by_css_selector('div.export-options>button.button-link-primary>span.button-link-text')[1]exportRIS.click()time.sleep(1)self.ifLogin = 1self.new = 0driver = webdriver.Chrome()
cnki = search()
cnki.getKeyWord()
input()

代码使用前的注意事项

请修改代码开头keywordList列表中的内容为使用者需要搜索的关键词
请将代码中的文件路径‘C:\Users\t\Downloads’修改为使用者Chrome浏览器的默认下载路径