抓取的网站是 吉林水文信息网，都是公开数据。本文主要讲的是如何使用selenium提取页面表格，下面***号间的代码就是提取表格的代码，只需要在开头的地方获取tr和td两个元素就可以了。原理是通过tr元素个数获知表格行数，td元素中的colspan、rowspan属性获知单元格总个数，从而计算出表格的列数。然后构建值为0的对应行列数表格，逐一将td元素中的文本填入表格，即可得到页面的表格。

from selenium.webdriver.common.keys import Keys
from selenium import webdriver
import time
import os
from selenium.webdriver.common.action_chains import ActionChains
import random
import sys
import urllib.request
import urllib.parse
import socket
import xlsxwriter

def output_excel(file_name, report_data1): #输出Excel

today = time.strftime('%Y%m%d%H%M',time.localtime(time.time())); #time.struct_time(tm_year=2018, tm_mon=7, tm_mday=23, tm_hour=12, tm_min=50, tm_sec=30, tm_wday=0, tm_yday=204, tm_isdst=0)
#新建文件
filename = (os.getcwd()+'\\'+file_name+today+'.xlsx')
test_book = xlsxwriter.Workbook(filename)#添加sheet1
worksheet = test_book.add_worksheet(file_name) #往文件里面添加sheet
#worksheet.freeze_panes(1, 4)#设置窗口冻结区域  距离顶部1行，距离左边2列
bold0 = worksheet.set_column(0, 0, 10.7) #列宽
x=-1;
for report in report_data1:if len(report)==0:continuey=-1;x=x+1;for rep in report:if rep=='nan':rep=''y=y+1;try:worksheet.write(x,y,rep)except:worksheet.write(x,y,rep);#关闭文件
test_book.close()

def get_data():
os.system(‘taskkill /F /IM chrome.exe’);os.system(‘taskkill /F /IM chromedriver.exe’);

#url = r'http://httpbin.org/get?show_env=1'
#url = r'http://httpbin.org/post?show_env=1'
url = r'http://www.hydrojl.net/pages/fb/W01FirstPageNew.aspx'while(True):try:obj = webdriver.Chrome()#obj = webdriver.Firefox()obj.set_page_load_timeout(60)time.sleep(6)obj.get(url)time.sleep(10)obj.maximize_window()breakexcept:print('启动浏览器异常')os.system('taskkill /F /IM chrome.exe');os.system('taskkill /F /IM chromedriver.exe');time.sleep(20)
try:#滚动页面obj.execute_script("window.scrollBy(0,300)") #向下滚动time.sleep(3)#obj.execute_script("window.scrollBy(0,document.body.scrollHeight/5*2)")#time.sleep(2)#切换框架obj.switch_to.frame("indexFrame")  #用name来定位time.sleep(2)#print(obj.page_source)#单击 跳转页面btn = obj.find_elements_by_xpath('//form[@id="form1"]/div/div/a/img')[0]ActionChains(obj).move_to_element(btn).perform() #窗口移动到元素time.sleep(2)btn.click() #单击#obj.implicitly_wait(60)time.sleep(10)#切换框架obj.switch_to.frame("iframeContent")  #用name来定位time.sleep(2)#抓数据#**************************************************************************trlist = obj.find_elements_by_xpath('//div[@id="chg"]//tbody/tr') #行tdlist = obj.find_elements_by_xpath('//div[@id="chg"]//tbody/tr/td') #列#行：rowspan="2"#列：colspan="14"trn = len(trlist) #总行数temlist = []for td in tdlist:try:temlist.append(int(td.get_attribute('colspan')))except:temlist.append(1)try:temlist.append(int(td.get_attribute('rowspan'))-1)except:passtdn = sum(temlist)tdn = int(tdn/trn) #总列数#构建表格table = []for j in range(0,trn):table.append([0]*tdn)#tdlist 所有元素for i in range(0,trn): #行for j in range(0,tdn): #列if table[i][j]==0:td = tdlist.pop(0) #获取数据table[i][j] = td.texttry:aa = int(td.get_attribute('colspan'))#列for a in range(1,aa):table[i][j+a] = ''except:passtry:aa = int(td.get_attribute('rowspan'))#行for a in range(1,aa):table[i+a][j] = ''except:passif len(tdlist)==0:breakif len(tdlist)==0:breakfor j in table:print(j)#**************************************************************************datalist = tabletime.sleep(6)#退出程序obj.close()obj.quit()print('ok')return datalist
except:os.system('taskkill /F /IM chrome.exe');os.system('taskkill /F /IM chromedriver.exe');print('no')return []

def main():

datalist = get_data()output_excel('雨情信息', datalist)

if name==‘main’:

main()time.sleep(6)os.system('taskkill /F /IM chrome.exe');os.system('taskkill /F /IM chromedriver.exe')
#os.system('pause')