1、本文实现的是一个简易版的xgboost回归例子，只是为了帮助理解xgboost底层原理，像一些抽样的参数比如subsample或者colsample_bytree以及min_child_weight等就没考虑了，同时也假定特征都是连续变量

2、与gbdt不同的是xgboost中的cart树分裂是依据信息增益最大的方向进行分裂，同时Gain要大于0，Gain的公式：

3、对于分裂得到的叶子节点利用节点的更新公式更新

代码

import numpy as np
from collections import Counter
import copyn_estimators =10#树的棵数
MAX_DEPTH = 2
LR = 0.3# 回归例子
data = np.array([[5,20,1.1],[7,30,1.3],[21,70,1.7],[30,60,1.8],[26,40,1.6],])class XGBoostRegression:def __init__(self,n_estimators,lr,max_depth,reg_lambda,reg_alpha):''':param n_estimators: cart树的棵树:param lr: 学习率:param max_depth: 树的最大深度:param reg_lambda: l2正则:param reg_alpha: l1正则'''self.n_estimators = n_estimatorsself.lr = lrself.max_depth = max_depthself.reg_lambda = reg_lambdaself.reg_alpha = reg_alphaself.tree_list = []@staticmethod# 切分分类数据def split_data(data,feat,val,data_type='classifier'):if data_type == 'classifier':arr1 = data[np.nonzero(data[:,feat] == val)]arr2 = data[np.nonzero(data[:,feat] != val)]else:arr1 = data[np.nonzero(data[:,feat].astype(float) < val)]arr2 = data[np.nonzero(data[:,feat].astype(float) >= val)]return arr1,arr2,np.nonzero(data[:,feat].astype(float) < val)[0],np.nonzero(data[:,feat].astype(float) >= val)[0]@staticmethod# 连续变量的切分点处理def continuity_params_process(arr,feat):c = arr[:,feat].astype(float)c_sort = sorted(set(c))new_c = []for i in range(len(c_sort)-1):val = (c_sort[i] + c_sort[i+1]) / 2new_c.append(val)return new_c# 选择最好的切分点# 满足Gain最大且大于0才分裂def select_split(self,data,Y):min_gini = -1best_feat = Nonebest_val = Noneleft = Noneright = Noneleft_y = Noneright_y = Noneg_left = Noneh_left = Noneg_right = Noneh_right = Nonedata_type = 'continuity'for i in range(data.shape[1]-1):# c_set = set(data[:, i])c_set = self.continuity_params_process(data,i)for val in c_set:arr1,arr2,arr1_index,arr2_index = self.split_data(data,i,val,data_type)gain, G_left, H_left, G_right, H_right = self.calc_gain(arr1,Y[arr1_index],arr2,Y[arr2_index])if min_gini < gain and gain > 0:min_gini = gainbest_feat = ibest_val = valleft = arr1right = arr2left_y = Y[arr1_index]right_y = Y[arr2_index]g_left = G_lefth_left = H_leftg_right = G_righth_right = H_rightif best_feat is None:g = np.sum(data[:,-1] - Y)h = len(data)return best_feat,best_val,left,right,left_y,right_y,g,h,g,hreturn best_feat,best_val,left,right,left_y,right_y,g_left,h_left,g_right,h_rightdef calc_gain(self,left,left_y,right,right_y):G_left = np.sum(left[:,-1] - left_y)H_left = len(left)G_right = np.sum(right[:,-1] - right_y)H_right = len(right)Gain = (G_left ** 2 / (H_left + self.reg_lambda) + G_right ** 2 / (H_right+self.reg_lambda) -(G_left + G_right) ** 2/ (H_left + H_right + self.reg_lambda)) - self.reg_alphareturn Gain,G_left,H_left,G_right,H_right# 构建递归树def create_tree(self,data,Y,n=0):tree = {}dd = data[:,:-1].tolist()ddd = list(map(tuple,dd))cc = Counter(ddd)if len(cc) == 1:g = np.sum(data - Y)h = len(data)return -g / (h + self.reg_lambda)best_feat,best_val,left,right,left_y,right_y,g_left,h_left,g_right,h_right = self.select_split(data,Y)if best_feat is None:return -g_left / (h_left + self.reg_lambda)n += 1if n >= self.max_depth:tree[(best_feat,best_val,'left')] = -g_left / (h_left + self.reg_lambda)tree[(best_feat,best_val,'right')] = -g_right / (h_right + self.reg_lambda)else:tree[(best_feat,best_val,'left')] = self.create_tree(left,left_y,n)tree[(best_feat,best_val,'right')] = self.create_tree(right,right_y,n)return treedef fit(self,dataset):''':param data: 输入数据:param n_estimators: 弱分类器的个数:param lr: 学习率，也叫残差学习率:return:'''data = copy.copy(dataset)self.tree_list.append(np.mean(data[:, -1]))for i in range(self.n_estimators):for j in range(len(data)):data[j,-1] = self.predict(data[j,:-1])self.tree_list.append(self.create_tree(data,dataset[:,-1]))# 预测单颗树def predict_one(self,tree,X):if type(tree) != dict:return treefor key in tree:if X[key[0]] < key[1]:r = tree[(key[0],key[1],'left')]else:r = tree[(key[0], key[1], 'right')]return self.predict_one(r, X)# 预测def predict(self,X):result = self.tree_list[0]for tree in self.tree_list[1:]:result += self.lr * self.predict_one(tree,X)return resultfrom xgboost.sklearn import XGBRegressor
xgb = XGBRegressor(n_estimators=n_estimators,learning_rate=0.3,max_depth=MAX_DEPTH,base_score=np.mean(data[:, -1]))
xgb.fit(data[:,:-1],data[:,-1])
print("xgboost:",xgb.predict(data[0,:-1].reshape(1,-1)))my_xgb_tree = XGBoostRegression(n_estimators=n_estimators,lr=LR,max_depth=MAX_DEPTH,reg_lambda=1,reg_alpha=0)
my_xgb_tree.fit(data)
print("my xgb tree:",my_xgb_tree.predict(data[0,:-1]))