【人工智能项目】Bert实现泰迪杯论坛文本分类任务

本次又是bert任务，用bert来实现论文文本分类的任务。那么接着走起走起走起！！！

运行环境

本次运行环境在google colab白嫖的gpu进行加速训练。

!nvidia-smi

导包

#! -*- coding:utf-8 -*-
import re, os, json, codecs, gc
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import os
import time
import datetimefrom sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.utils import class_weight as cwfrom keras import Sequentialfrom keras.models import Modelfrom keras.layers import LSTM,Activation,Dense,Dropout,Input,Embedding,BatchNormalization,Add,concatenate,Flatten
from keras.layers import Conv1D,Conv2D,Convolution1D,MaxPool1D,SeparableConv1D,SpatialDropout1D,GlobalAvgPool1D,GlobalMaxPool1D,GlobalMaxPooling1D
from keras.layers.pooling import _GlobalPooling1D
from keras.layers import MaxPooling2D,GlobalMaxPooling2D,GlobalAveragePooling2Dfrom keras.optimizers import RMSprop,Adamfrom keras.preprocessing.text import Tokenizer
from keras.preprocessing import sequencefrom keras.utils import to_categoricalfrom keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint
from keras.callbacks import ReduceLROnPlateau%matplotlib inlineimport warnings
warnings.filterwarnings("ignore")

读取文件

import xlrdtrain_dataset_path = "train.xlsx"
test_dataset_path = "test.xlsx"train_content=xlrd.open_workbook(filename=train_dataset_path,encoding_override='gbk')
test_content=xlrd.open_workbook(filename=test_dataset_path,encoding_override='gbk')train_df=pd.read_excel(train_content,engine='xlrd',header=0,names=["id","user_id","target","time","text","labels"])
test_df=pd.read_excel(test_content,engine='xlrd',header=0,names=["id","user_id","target","time","text","labels"])

train_df

test_df

合并

train_df["ocr"] = train_df["target"] + "  " + train_df["text"]
test_df["ocr"] = test_df["target"] + "  " + test_df["text"]

train_df

标签统计

sns.countplot(train_df["labels"])
plt.xlabel("Label")
plt.title("News sentiment analysis")

train_df["labels"].unique()

array([’__label__城乡建设’, ‘__label__环境保护’, ‘__label__交通运输’, ‘__label__教育文体’,
‘__label__劳动和社会保障’, ‘__label__商贸旅游’, ‘__label__卫生计生’], dtype=object)

# 统计新闻数据中情感，正面情绪对应0 中性情绪对应1 负面情绪对应2
dit = {'__label__城乡建设':0,'__label__环境保护':1,"__label__交通运输":2,"__label__教育文体":3,"__label__劳动和社会保障":4,"__label__商贸旅游":5,"__label__卫生计生":6}
train_df['label'] = train_df['labels'].map(dit)

sns.countplot(train_df["label"])
plt.xlabel("Label")
plt.title("News sentiment analysis")

最终数据集

train_df = train_df[["ocr","label"]]

from sklearn.utils import shuffle
train_df = shuffle(train_df).reset_index(drop=True)

安装bert

!pip install keras_bert

下载预训练的模型权重

!wget -c https://storage.googleapis.com/chineseglue/pretrain_models/roeberta_zh_L-24_H-1024_A-16.zip

解压缩文件

!unzip roeberta_zh_L-24_H-1024_A-16.zip

训练

#! -*- coding:utf-8 -*-
import re, os, json, codecs, gc
import numpy as np
import pandas as pd
from random import choice
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import KFold
from keras_bert import load_trained_model_from_checkpoint, Tokenizerfrom keras.layers import *
from keras.callbacks import *
from keras.models import Model
import keras.backend as K
from keras.optimizers import Adammaxlen = 256
config_path = './bert_config_large.json'
# checkpoint_path = '/export/home/liuyuzhong/kaggle/bert/chinese_L-12_H-768_A-12/bert_model.ckpt'
checkpoint_path = './roberta_zh_large_model.ckpt'
dict_path = './vocab.txt'token_dict = {}
with codecs.open(dict_path, 'r', 'utf8') as reader:for line in reader:token = line.strip()token_dict[token] = len(token_dict)class OurTokenizer(Tokenizer):def _tokenize(self, text):R = []for c in text:if c in self._token_dict:R.append(c)elif self._is_space(c):R.append('[unused1]') # space类用未经训练的[unused1]表示else:R.append('[UNK]') # 剩余的字符是[UNK]return Rtokenizer = OurTokenizer(token_dict)def seq_padding(X, padding=0):L = [len(x) for x in X]ML = max(L)return np.array([np.concatenate([x, [padding] * (ML - len(x))]) if len(x) < ML else x for x in X])class data_generator:def __init__(self, data, batch_size=4, shuffle=True):self.data = dataself.batch_size = batch_sizeself.shuffle = shuffleself.steps = len(self.data) // self.batch_sizeif len(self.data) % self.batch_size != 0:self.steps += 1def __len__(self):return self.stepsdef __iter__(self):while True:idxs = list(range(len(self.data)))if self.shuffle:np.random.shuffle(idxs)X1, X2, Y = [], [], []for i in idxs:d = self.data[i]text = d[0][:maxlen]x1, x2 = tokenizer.encode(first=text)y = d[1]X1.append(x1)X2.append(x2)Y.append([y])if len(X1) == self.batch_size or i == idxs[-1]:X1 = seq_padding(X1)X2 = seq_padding(X2)Y = seq_padding(Y)yield [X1, X2], Y[:, 0, :][X1, X2, Y] = [], [], []from keras.metrics import top_k_categorical_accuracy
# 
def acc_top5(y_true, y_pred):return top_k_categorical_accuracy(y_true, y_pred, k=5)
# 模型           
def build_bert(nclass):bert_model = load_trained_model_from_checkpoint(config_path, checkpoint_path, seq_len=None)for l in bert_model.layers:l.trainable = Truex1_in = Input(shape=(None,))x2_in = Input(shape=(None,))x = bert_model([x1_in, x2_in])x = Lambda(lambda x: x[:, 0])(x)p = Dense(nclass, activation='softmax')(x)model = Model([x1_in, x2_in], p)model.compile(loss='categorical_crossentropy', optimizer=Adam(1e-5),metrics=['accuracy', acc_top5])print(model.summary())return model

from keras.utils import to_categoricalDATA_LIST = []
# 改
for data_row in train_df.iloc[:].itertuples():DATA_LIST.append((data_row.ocr, to_categorical(data_row.label, 7)))
DATA_LIST = np.array(DATA_LIST)DATA_LIST_TEST = []
for data_row in test_df.iloc[:].itertuples():DATA_LIST_TEST.append((data_row.ocr, to_categorical(0, 7)))
DATA_LIST_TEST = np.array(DATA_LIST_TEST)

# k者交叉验证
def run_cv(nfold, data, data_label, data_test):kf = KFold(n_splits=nfold, shuffle=True, random_state=520).split(data)# 改train_model_pred = np.zeros((len(data), 7))test_model_pred = np.zeros((len(data_test), 7))k = 0for i, (train_fold, test_fold) in enumerate(kf):X_train, X_valid, = data[train_fold, :], data[test_fold, :]# 改model = build_bert(7)# early_stopping = EarlyStopping(monitor='val_acc', patience=3)plateau = ReduceLROnPlateau(monitor="val_acc", verbose=1, mode='max', factor=0.5, patience=2)checkpoint = ModelCheckpoint('./bert_dump/' + str(i) + '.hdf5', monitor='val_acc', verbose=2, save_best_only=True, mode='max',save_weights_only=True)train_D = data_generator(X_train, shuffle=True)valid_D = data_generator(X_valid, shuffle=True)test_D = data_generator(data_test, shuffle=False)history = model.fit_generator(train_D.__iter__(),steps_per_epoch=len(train_D),epochs=2,validation_data=valid_D.__iter__(),validation_steps=len(valid_D),callbacks=[early_stopping, plateau, checkpoint],)# return modeltrain_model_pred[test_fold, :] =  model.predict_generator(valid_D.__iter__(), steps=len(valid_D),verbose=1)test_model_pred += model.predict_generator(test_D.__iter__(), steps=len(test_D),verbose=1)if k == 0:breakdel model; gc.collect()K.clear_session()# breakreturn train_model_pred, test_model_pred,history

train_model_pred, test_model_pred,history = run_cv(2, DATA_LIST, None, DATA_LIST_TEST)

绘制训练过程中的loss和acc

# 绘制训练过程中的 loss 和 acc 变化曲线
import matplotlib.pyplot as plt
%matplotlib inlinedef history_plot(history_fit):plt.figure(figsize=(12,6))# summarize history for accuracyplt.subplot(121)plt.plot(history_fit.history["accuracy"])plt.plot(history_fit.history["val_accuracy"])plt.title("model accuracy")plt.ylabel("accuracy")plt.xlabel("epoch")plt.legend(["train", "valid"], loc="upper left")# summarize history for lossplt.subplot(122)plt.plot(history_fit.history["loss"])plt.plot(history_fit.history["val_loss"])plt.title("model loss")plt.ylabel("loss")plt.xlabel("epoch")plt.legend(["train", "valid"], loc="upper left")plt.show()

history_plot(history)

test_pred = [np.argmax(x) for x in test_model_pred]
test_df['labels'] = test_pred

test_df['labels']

test_df

# 统计新闻数据中情感，正面情绪对应0 中性情绪对应1 负面情绪对应2
dit = {0:'__label__城乡建设',1:'__label__环境保护',2:"__label__交通运输",3:"__label__教育文体",4:"__label__劳动和社会保障",5:"__label__商贸旅游",6:"__label__卫生计生"}
test_df['label'] = test_df['labels'].map(dit)

test_df

predict_df = test_df[["id","label"]]

predict_df

predict_df.to_csv('res.csv',index=None,encoding='utf_8_sig')

那么本次到此结束，回头见！！！