作业要求：https://edu.cnblogs.com/campus/gzcc/GZCC-16SE1/homework/3159

前言：在猫眼电影上看到《何以为家》电影的评分比较高，于是爬取用户的部分评论进行分析。

 

 

一、获取数据的url接口

1、在电脑网页版上可以看到只有看到10条的热门评论，数据过于少无法进行分析。

 

2、使用手机网页版进行获取url接口，但是发现只能加载到1000条评论。1000条后

      的评论无法加载，也返回不了数据，于是只能爬取1000条数据进行分析。

 

     根据url的规律，和返回的json数据，可知每个url返回15条评论的数据，

     offset的值是指从第几条评论开始返回。

 

 

 

3、在网上找到了一个旧的url接口，上面的返回的json数据还有城市，而新的url没有，

      于是就使用旧的url。

  

http://m.maoyan.com/mmdb/comments/movie/1218727.json?_v_=yes&offset=？&startTime=0

二、设置合理的user-agent，模拟成真实的浏览器去提取内容。

#设置合理的user-agent，爬取数据函数
def getData(url):headers =[{'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36','Cookie': '_lxsdk_cuid=16a8d7b1613c8-0a2b4d109e58f-b781636-144000-16a8d7b1613c8; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; uuid_n_v=v1; iuuid=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; webp=true; ci=20%2C%E5%B9%BF%E5%B7%9E; selectci=; __mta=45946523.1557151818494.1557367174996.1557368154367.23; _lxsdk=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; __mta=45946523.1557151818494.1557368154367.1557368240554.24; from=canary; _lxsdk_s=16a9a2807fa-ea7-e79-c55%7C%7C199'},{ 'User-Agent': 'Mozilla / 5.0(Linux;Android 6.0;  Nexus 5 Build / MRA58N) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 73.0 .3683.103Mobile  Safari / 537.36','Cookie':'_lxsdk_cuid=16a8d7b1613c8-0a2b4d109e58f-b781636-144000-16a8d7b1613c8; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; uuid_n_v=v1; iuuid=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; webp=true; ci=20%2C%E5%B9%BF%E5%B7%9E; selectci=; __mta=45946523.1557151818494.1557367174996.1557368154367.23; _lxsdk=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; __mta=45946523.1557151818494.1557368154367.1557368240554.24; from=canary; _lxsdk_s=16a9a2807fa-ea7-e79-c55%7C%7C199'},{'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10','Cookie':'_lxsdk_cuid=16a8d7b1613c8-0a2b4d109e58f-b781636-144000-16a8d7b1613c8; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; uuid_n_v=v1; iuuid=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; webp=true; ci=20%2C%E5%B9%BF%E5%B7%9E; selectci=; __mta=45946523.1557151818494.1557367174996.1557368154367.23; _lxsdk=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; __mta=45946523.1557151818494.1557368154367.1557368240554.24; from=canary; _lxsdk_s=16a9a2807fa-ea7-e79-c55%7C%7C199'}]#   proxies = [{'https': 'https://120.83.111.194:9999','http':'http://14.20.235.120:808'},{"http": "http://119.131.90.115:9797",# "https": "https://14.20.235.96:9797"}]get=requests.get(url, headers=headers[random.randint(0,2)]);get.encoding = 'utf-8'return get

　　

 

 

三、对爬取的数据进行处理，生成。

#数据处理函数
def dataProcess(data):data = json.loads(data.text)['cmts']allData = []for i in data:dataList = {}dataList['id'] = i['id']dataList['nickName'] = i['nickName']dataList['cityName'] = i['cityName'] if 'cityName' in i else ''  # 处理cityName不存在的情况dataList['content'] = i['content'].replace('\n', ' ', 10)  # 处理评论内容换行的情况dataList['score'] = i['score']dataList['startTime'] = i['startTime']if "gender" in i:dataList['gendar'] = i["gender"]else:dataList['gendar'] = i["gender"] = 0allData.append(dataList)return allData

　　

 

 

 

四、把爬取的数据生成csv文件和保存到数据库。

代码：

#处理后的数据保存为csv文件
pd.Series(allData)
newsdf=pd.DataFrame(allData)
newsdf.to_csv('news.csv',encoding='utf-8')#把csv文件保存到sqlite
newsdf = pd.read_csv('news.csv')
with sqlite3.connect('sqlitetest.sqlite') as db:newsdf.to_sql('data',con = db)

　　

截图：

 

 

 

  最后只爬取到了1004条的数据，不知道是不是猫眼电影对评论数据的获取进行了限制，加载

  到一定数据量就无法加载了。

 

 

四、数据可视化分析。

 

4.1、评论者性别分析

 代码：

# 评论者性别分布可视化
def sex(gender):from pyecharts import Pielist_num = []print(gendar)list_num.append(gender.count(0)) # 未知print(gender.count(0))list_num.append(gender.count(1)) # 男list_num.append(gender.count(2)) # 女attr = ["未知","男","女"]pie = Pie("性别饼图")pie.add("", attr, list_num,is_label_show=True)pie.render("sex_pie.html")

　

截图：　

    这部电影除去未知性别的，在已知性别的评论者女性的比例比较多，说明这部电影女性的

    爱好者比较多。

 

 

4.2、评论者评分等级分析

 

代码：

# 评论者评分等级环状饼图
def scoreProcess(score):from pyecharts import Pielist_num = []list_num.append(scores.count(0))list_num.append(scores.count(0.5))list_num.append(scores.count(1))list_num.append(scores.count(1.5))list_num.append(scores.count(2))list_num.append(scores.count(2.5))list_num.append(scores.count(3))list_num.append(scores.count(3.5))list_num.append(scores.count(4))list_num.append(scores.count(4.5))list_num.append(scores.count(5))attr = ["0", "0.5", "1","1.5","2","2.5", "3", "3.5","4","4.5","5"]pie = Pie("评分等级环状饼图",title_pos="center")pie.add("", attr, list_num, is_label_show=True,label_text_color=None,radius=[40, 75],legend_orient="vertical",legend_pos="left",legend_top="100px",center=[50,60])pie.render("score_pie.html")

　　

 

 截图。

      根据上面分饼图可得满分的占了67%左右，4.5分以上占了82%左右，可知这部电影的

      评价十分高，应该是非常好看的，值得去观看。

 

 4.2、观众分布地图分析

       根据网上资料自从 v0.3.2 开始，pyecharts 将不再自带地图 js 文件。根据需要可以安装对应的地图包。

       全球国家地图: echarts-countries-pypkg : 世界地图和 213 个国家，包括中国地图
       中国省级地图: echarts-china-provinces-pypkg：23 个省，5 个自治区
       中国市级地图: echarts-china-cities-pypkg ：370 个中国城市
       中国县区级地图: echarts-china-counties-pypkg ：2882 个中国县·区
       中国区域地图: echarts-china-misc-pypkg：11 个中国区域地图，比如华南、华北

代码：

# 观众分布图
def cityProcess(citysTotal):from pyecharts import Geogeo =Geo("《何以为家》观众分布", title_color='#fff', title_pos='center',width=1200,height = 600, background_color = '#404a95')attr, value = geo.cast(citysTotal)geo.add("", attr, value, is_visualmap=True, visual_range=[0, 100], visual_text_color='#fff',legend_pos = 'right', is_geo_effect_show = True, maptype='china',symbol_size=10)geo.render("city_geo.html")

　　

截图：

  可以看出观众都是集中在沿海附近的城市，这也说这些城市相对于中国西北地区更为发达

   一些。尤其是北京、上海、广州、深圳的观众是最多的。这些地区的消费水平上也相对更

   高一些。人口也会计较的聚集。

 

  

四、完整代码。

import requests
from bs4 import BeautifulSoup
from datetime import datetime
import re
import sqlite3
import pandas as pd
import time
import pandas
import random
import json#设置合理的user-agent，爬取数据函数
def getData(url):headers =[{'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36','Cookie': '_lxsdk_cuid=16a8d7b1613c8-0a2b4d109e58f-b781636-144000-16a8d7b1613c8; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; uuid_n_v=v1; iuuid=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; webp=true; ci=20%2C%E5%B9%BF%E5%B7%9E; selectci=; __mta=45946523.1557151818494.1557367174996.1557368154367.23; _lxsdk=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; __mta=45946523.1557151818494.1557368154367.1557368240554.24; from=canary; _lxsdk_s=16a9a2807fa-ea7-e79-c55%7C%7C199'},{ 'User-Agent': 'Mozilla / 5.0(Linux;Android 6.0;  Nexus 5 Build / MRA58N) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 73.0 .3683.103Mobile  Safari / 537.36','Cookie':'_lxsdk_cuid=16a8d7b1613c8-0a2b4d109e58f-b781636-144000-16a8d7b1613c8; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; uuid_n_v=v1; iuuid=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; webp=true; ci=20%2C%E5%B9%BF%E5%B7%9E; selectci=; __mta=45946523.1557151818494.1557367174996.1557368154367.23; _lxsdk=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; __mta=45946523.1557151818494.1557368154367.1557368240554.24; from=canary; _lxsdk_s=16a9a2807fa-ea7-e79-c55%7C%7C199'},{'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10','Cookie':'_lxsdk_cuid=16a8d7b1613c8-0a2b4d109e58f-b781636-144000-16a8d7b1613c8; _lx_utm=utm_source%3DBaidu%26utm_medium%3Dorganic; uuid_n_v=v1; iuuid=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; webp=true; ci=20%2C%E5%B9%BF%E5%B7%9E; selectci=; __mta=45946523.1557151818494.1557367174996.1557368154367.23; _lxsdk=1BB9A320700C11E995DE7D45B75E59C6FC50A50D996543D0819E9EB2E6507E92; __mta=45946523.1557151818494.1557368154367.1557368240554.24; from=canary; _lxsdk_s=16a9a2807fa-ea7-e79-c55%7C%7C199'}]#   proxies = [{'https': 'https://120.83.111.194:9999','http':'http://14.20.235.120:808'},{"http": "http://119.131.90.115:9797",# "https": "https://14.20.235.96:9797"}]get=requests.get(url, headers=headers[random.randint(0,2)]);get.encoding = 'utf-8'return get#数据处理函数
def dataProcess(data):data = json.loads(data.text)['cmts']allData = []for i in data:dataList = {}dataList['id'] = i['id']dataList['nickName'] = i['nickName']dataList['cityName'] = i['cityName'] if 'cityName' in i else ''  # 处理cityName不存在的情况dataList['content'] = i['content'].replace('\n', ' ', 10)  # 处理评论内容换行的情况dataList['score'] = i['score']dataList['startTime'] = i['startTime']if "gender" in i:dataList['gendar'] = i["gender"]else:dataList['gendar'] = i["gender"] = 0allData.append(dataList)return allDataallData=[]
for i in range(67):get=getData('http://m.maoyan.com/mmdb/comments/movie/1218727.json?_v_=yes&offset={}&startTime=0'.format(i*15))allData.extend(dataProcess(get))#处理后的数据保存为csv文件
pd.Series(allData)
newsdf=pd.DataFrame(allData)
newsdf.to_csv('news.csv',encoding='utf-8')# #把csv文件保存到sqlite
# newsdf = pd.read_csv('news.csv')
# with sqlite3.connect('sqlitetest.sqlite') as db:
#  newsdf.to_sql('data',con = db)# 评论者性别分布可视化
def sexProcess(gender):from pyecharts import Pielist_num = []list_num.append(gender.count(0)) # 未知list_num.append(gender.count(1)) # 男list_num.append(gender.count(2)) # 女attr = ["未知","男","女"]pie = Pie("性别饼图",title_pos="center")pie.add("", attr, list_num,is_label_show=True)pie.render("sex_pie.html")gendar=[]
for i in allData:gendar.append(i['gendar'])
sexProcess(gendar)# 评论者评分等级环状饼图
def scoreProcess(scores):from pyecharts import Pielist_num = []list_num.append(scores.count(0))list_num.append(scores.count(0.5))list_num.append(scores.count(1))list_num.append(scores.count(1.5))list_num.append(scores.count(2))list_num.append(scores.count(2.5))list_num.append(scores.count(3))list_num.append(scores.count(3.5))list_num.append(scores.count(4))list_num.append(scores.count(4.5))list_num.append(scores.count(5))attr = ["0", "0.5", "1","1.5","2","2.5", "3", "3.5","4","4.5","5"]pie = Pie("评分等级环状饼图",title_pos="center")pie.add("", attr, list_num, is_label_show=True,label_text_color=None,radius=[40, 75],legend_orient="vertical",legend_pos="left",legend_top="100px",center=[50,60])pie.render("score_pie.html")scores=[]
for i in allData:scores.append(i['score'])
scoreProcess(scores)# 观众分布图
def cityProcess(citysTotal):from pyecharts import Geogeo =Geo("《何以为家》观众分布", title_color='#fff', title_pos='center',width=1200,height = 600, background_color = '#404a95')attr, value = geo.cast(citysTotal)geo.add("", attr, value, is_visualmap=True, visual_range=[0, 100], visual_text_color='#fff',legend_pos = 'right', is_geo_effect_show = True, maptype='china',symbol_size=10)geo.render("city_geo.html")# 城市名称的处理
citysTotal={}
coordinatesJson = pd.read_json('city_coordinates.json',encoding='utf-8')
for i in allData:for j in coordinatesJson:if str(i['cityName']) in str(j) :if str(j) not in citysTotal:citysTotal[str(j)]=1else:citysTotal[str(j)]=citysTotal[str(j)]+1breakcityProcess(citysTotal)