## 中文情感分类--关于疫情、微博、中文、文本

本次中文情感分析源于数据挖掘与分析课大作业，主要内容为:对疫情期间的微博文本进行情感分类，进而分析情感变化。

 1. 数据集：训练集和待预测数据集，其中训练集为打好标签的微博疫情相关文本，待预测训练集为情感趋势来源。
 2. python库：主要使用 jieba、pandas，其余详见import
 3. 主要涉及内容有：分词，去停用词，构建词向量模型，分词文本向量化，模型训练，预测等部分。

【文件路径\\、/没有修改成一致。部分代码不够简洁流畅，仅提供步骤参考，相关文件、代码(同组成员的微博爬虫、清洗、以及数据集链接)会考虑需要上传】

---

part 1：训练集文本

    --分词，去停，构建词向量（这里没有用pandas，十分后悔）

1.import部分及main方法：

import jieba
import numpy as np
import pandas as pd
import osimport gensim
from gensim.test.utils import common_texts,get_tmpfile
from gensim.models import Word2Vecimport mathimport csvif __name__=='__main__':data = pd.read_csv('D:\\documents\\data mining\\数据集\\情感分类-疫情微博\\nCoV_100k_train.labled.csv',engine="python")#data = pd.read_csv('D:\\documents\\data mining\\数据集\\普通情感分类-7\\情感训练集.csv')#print(data.head())#提取目标列，第2列data1 = list(data.iloc[:,3]) #根据数据集修改 100k-3，情感训练集-0#print(data1[0])label = list(data.iloc[:,6]) #根据数据集修改 100k-6，情感训练集-1#分词size = 100 #词向量模型(data2,label) = word_cut(data1,label,size) #返回分词后列表，以字符串为元素，字符串用'，'隔开字符print('分词成功')print(len(data2),len(label))

2.分词，去停，词向量

def word_cut(data1,label,size):filelist = []for i in data1:i=str(i)i = i.replace('展开全文c','')s=jieba.cut(i,cut_all=False)cutstr = '$$$'.join(s)'''s1 = iter(s)cutstr=''for i in s1:if cutstr =='':cutstr+=ielse:cutstr+='$$$'cutstr+=i'''textlist = cutstr.split('$$$')#print(textlist)filelist.append(textlist)filelist = removesw(filelist) #去停用词后的list，可能有空j=0for i in range(len(filelist)):#删除空值if len(filelist[i-j])== 0:del filelist[i-j]del label[i-j]j+=1#print(len(filelist),len(label))#print(filelist[0],label[0])#print(filelist[1],label[1])#print(filelist[-2],label[-2])#print(filelist[-1],label[-1])#打开txttxtfile = open('D:/documents/data mining/数据集/代码/data_cut.txt',mode = 'w')for i in range(len(filelist)):string=''for j in filelist[i]:if j != '':if string == '':string += jelse:string += ','string += j##写入txt文件  #分词+labeltxtfile.write(string.encode("gbk", 'ignore').decode("gbk", "ignore")+' '+str(label[i])+'\n')txtfile.close()print('cut_word写入txt')model = Word2Vec(filelist,size=size,window=5,min_count=1,workers=4)model.save("D:/documents/data mining/数据集/代码/word2vec.bin")print('cut_word加入词向量模型')return (filelist,label)

本段主要为 利用结巴分词进行分词，分词结果使用$$$分隔，使用下方去停方法。

将去停后的分词文本加入词向量模型，其中word2vec中的filelist只要为可循环的变量均可，后续往词向量模型加入，以及获得文本向量的语句见part2.

def removesw(filelist):  #filelist:由分词构成的liststop_word = Nonewith open('D:/documents/data mining/数据集/stopwords-master/cn_stopwords.txt','r',encoding = 'utf-8') as f:stop_words = f.readlines()stop_words = [word.replace('\n','') for word in stop_words]# stop word 替换#i=0for i in range(len(filelist)):filelist[i]=[x for x in filelist[i] if x not in stop_words]return filelist

本段去停用词，txt为网络找的停用词表，中途会根据微博语境增删改。for循环里的代码比较核心。

part 2：预测集数据

  --本部分主要使用pandas库，对预测集分词、去停，结果加入part1中构建的词向量模型。然后利用词向量模型、训练集&预测集分析结果，构建文本向量并写入.csv文件。

1.import部分+数据清洗、分词、去停

（清洗部分希望去掉部分无意义词段，防止分词后无法去除。）

import os
import pandas as pd
import jiebaimport gensim
from gensim.test.utils import common_texts,get_tmpfile
from gensim.models import Word2Vecimport numpy as np
import csv#----数据清洗，分词----
with open('D:/documents/data mining/数据集/stopwords-master/cn_stopwords.txt','r',encoding = 'utf-8') as f:stop_words = f.readlines()stop_words = [word.replace('\n','') for word in stop_words]stop_words.append('\u200b')origin_dir='D:\\documents\\data mining\\数据集\\代码\\cleaned_text\\'
files=os.listdir(origin_dir)
after_clean_dir='D:\\documents\\data mining\\数据集\\代码\\after_clean\\'def clean_mix(s):#print(type(s))return s.replace('收起全文d','').replace('展开全文d','').replace('的秒拍视频','').replace('的微博视频','').replace('的快手视频','').replace('\n','').replace('O网页链接','')def after_jieba_stopword(s):a=jieba.cut(str(s),cut_all=False)b = '$$$'.join(a)c=[x for x in b.split('$$$') if x not in stop_words]return ' '.join(c)N_origin=0
N_filter=0
for file in files:data=pd.read_table(origin_dir+file,sep=',',encoding='utf-8')N_origin+=len(data)#分词data['cleaned_text']=data['cleaned_text'].map(lambda x:clean_mix(str(x)) if type(x)==type('') else '') #去词data['cleaned_text']=data['cleaned_text'].map(lambda x:after_jieba_stopword(x)) #分词，去停用词data['removeWellSign']=data['removeWellSign'].map(lambda x:clean_mix(str(x)) if type(x)==type('') else '')data['removeWellSign']=data['removeWellSign'].map(lambda x:after_jieba_stopword(x))data_filter=data.loc[data['cleaned_text']!='',:] data_filter['id']=np.arange(0,len(data_filter),1)N_filter+=len(data_filter)data_filter[['id','original_text','cleaned_text','removeWellSign']].to_csv(after_clean_dir+file,sep=',',index=None,encoding='utf-8')print(file,'over')print(N_origin)
print(N_filter)

2.词向量模型训练

  --待预测数据集分词结果加入词向量模型

#训练模型，向量化
after_clean_dir='D:\\documents\\data mining\\数据集\\代码\\after_clean\\'
files=os.listdir(after_clean_dir)
model = Word2Vec.load("D:/documents/data mining/数据集/代码/word2vec.bin")for file in files:data=pd.read_table(after_clean_dir+file,sep=',',encoding='utf-8')filelist=list(data['cleaned_text'].map(lambda x:x.split(' ')) )model.train(filelist,total_examples=model.corpus_count,epochs= model.iter)print(file,'train over')model.save("D:/documents/data mining/数据集/代码/word2vec.bin")
print('预测文本加入词向量模型-成功')

3.文本向量化

利用分词后的文本，分别从词向量模型中获得词语对应向量（向量中不包含所有词），加总（权重为1）、平均，得到句子对应文本向量。

#模型106万条文本的向量化
after_clean_dir='D:\\documents\\data mining\\数据集\\代码\\after_clean\\'
vectors_dir='D:\\documents\\data mining\\数据集\\代码\\vectors\\'
files=os.listdir(after_clean_dir)
model = Word2Vec.load("D:/documents/data mining/数据集/代码/word2vec.bin")for file in files:data=pd.read_table(after_clean_dir+file,sep=',',encoding='utf-8')filelist=list(data['cleaned_text'].map(lambda x:x.split(' ')))df=pd.DataFrame()for text in filelist:text_vector = np.zeros(100).reshape((1,100))count = 0for word in text:try:text_vector += model[word].reshape((1,100))#print(word,model[word])count += 1except KeyError:continueif count !=0:text_vector /= count #count个单词，所以除以countvector_list= list(list(text_vector)[0])df=df.append(pd.Series(vector_list),ignore_index=True)df.to_csv(vectors_dir+file,sep=',',index=None,header=None)print(file,'train over')#---训练集文本向量化---
model = Word2Vec.load("D:/documents/data mining/数据集/代码/word2vec.bin")
txtfile = open('D:\\documents\\data mining\\数据集\\代码\\data_cut.txt','r')data=[]
for i in txtfile.readlines():a=i.split(' ')a = [word.replace('\n','') for word in a]#print(a)data.append(a) #[[cut_word,label],[cut_word,label]]for i in data:text = i[0].split(',')text_vector = np.zeros(100).reshape((1,100))count = 0for word in text:try:text_vector += model[word].reshape((1,100))count += 1except KeyError:continueif count !=0:text_vector /= count #count个单词，所以除以countvector_list= list(list(text_vector)[0])#print(i[0],vector_list)i=i.append(vector_list) #print(data[0])with open('D:\\documents\\data mining\\数据集\\代码\\trainText_vector.csv','w',newline='') as tf:writer = csv.writer(tf,delimiter = ',')#writer.writerow(file_columns)for row in data:#print(row)row1 = row[2]row1.append(int(row[1]))#print(row1)writer.writerow(row1)tf.close()
print('训练文本向量化完成')

4.模型训练

--这里的模型为决策树模型，使用OneVsOne分类方式，是经过挑选的。训练过程中，将训练集向量9:1分为训练集和测试集，正确率较高，且在预测分类中效果较好。

from sklearn.multiclass import OneVsOneClassifier
from sklearn.tree import DecisionTreeRegressorfrom sklearn.model_selection import train_test_split
from sklearn.preprocessing import label_binarizefrom joblib import dump, load#---模型训练及预测---
after_clean_dir='D:\\documents\\data mining\\数据集\\代码\\after_clean\\'
vectors_dir='D:\\documents\\data mining\\数据集\\代码\\vectors\\'
label_dir='D:\\documents\\data mining\\数据集\\代码\\text_label\\'
files=os.listdir(after_clean_dir)#模型训练
labeled_path = 'D:\\documents\\data mining\\数据集\\代码\\trainText_vector.csv'labeled=pd.read_table(labeled_path,sep=',')
n=len(labeled)#11281vectors=labeled.iloc[:,:-1]
labels=labeled.iloc[:,-1]X_train, X_test, y_train, y_test = train_test_split(vectors, labels, test_size=0.2)y_test_list=list(y_test)y_train_list2=np.array(list(y_train.map(lambda x:[x])))
X_train_list=np.array(X_train)
X_test_list=np.array(X_test)n_train=len(y_train)#10152
n_test=len(y_test)#1129def accuracy(a,b):c=[]for i in range(len(a)):if a[i]==b[i]:c.append(1)else:c.append(0)return sum(c)/len(c)model_tree_one=OneVsOneClassifier(DecisionTreeRegressor()) #2v2
model_tree_one.fit(X_train,y_train)
predict_tree_one=model_tree_one.predict(X_test)
print(predict_tree_one)
accuracy_tree_one=accuracy(predict_tree_one,y_test_list) #0.7478753541076487
print("accuracy_tree_one:"+str(accuracy_tree_one))dump(model_tree_one,'model_tree_one.joblib')
print('预测模型建立并存储完成')

5.情感分类预测

#预测
#model_tree_one=load('D:\\documents\\data mining\\数据集\\代码\\model_tree_one.joblib')
model_tree_one=load('D:\\documents\\data mining\\数据集\\代码\\svc.joblib')
for file in files:vectors_file=pd.read_table(vectors_dir+file,sep=',',header=None)text_file=pd.read_table(after_clean_dir+file,sep=',')result=model_tree_one.predict(vectors_file)text_file['label']=resulttext_file.to_csv(label_dir+file,sep=',',index=None)print(file,'predict over')

6.随便输出到.csv的分类结果（积极，消极，总数等）

# 预测结果统计
from pandas import DataFrame
analysis_dir = 'D:\\documents\\data mining\\数据集\\代码\\text_label\\'
analysis_files = os.listdir(analysis_dir)
#analysis_data = {'date':[],'neg':[],'pos':[],'total':[]}
analysis_df = DataFrame(data=[],index=[],columns=['deta','neg','pos','total'])for file in analysis_files:analysis_file = pd.read_table(analysis_dir+file,sep=',')#pos = analysis_file.loc[analysis_file['label'] == '1',:].count()#neg = analysis_file.loc[analysis_file['label'] == '-1',:].count()vc=analysis_file['label'].value_counts(normalize = False, dropna = False)pos = vc[1]neg = vc[-1]total = analysis_file['label'].count()print(file,neg,pos,total) #analysis_df=analysis_df.append(pd.DataFrame([[file.replace('.csv','').replace('.','-'),neg,pos,total]],columns=['deta','neg','pos','total']))analysis_df.to_csv('D:\\documents\\data mining\\数据集\\代码\\结果图.csv',sep=',',index=None)