构建的集成分类器，它通常比任何单个成员的预测性能都要好。
把不同的分类算法及其各自相应的权重组合起来。目标是建立一个更强大的超级分类器，以平衡单个分类器在特定数据集上的弱点。

import numpy as np
import matplotlib.pyplot as plt
from sklearn.base import BaseEstimator
from sklearn.base import ClassifierMixin
from sklearn.preprocessing import LabelEncoder
# from sklearn.externals import six
import six
import sys
sys.modules['sklearn.externals.six'] = sixfrom sklearn.base import clone
from sklearn.pipeline import _name_estimators
import operator
from sklearn import datasets
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score
from sklearn.metrics import roc_curve
from sklearn.metrics import auc
from itertools import product
from sklearn.model_selection import GridSearchCV# 用父类 BaseEstimator和ClassifierMixin轻而易举地获得了一些基本功能，
# 这包括设置和返回分类器参数的get_params和set_params方法，
# 以及计算预测准确度的score方法。
class MajorityVoteClassifier(BaseEstimator, ClassifierMixin):""" A majority vote ensemble classifierParameters----------classifiers : array-like, shape = [n_classifiers]Different classifiers for the ensemblevote : str, {'classlabel', 'probability'} (default='label')If 'classlabel' the prediction is based on the argmax ofclass labels. Else if 'probability', the argmax ofthe sum of probabilities is used to predict the class label(recommended for calibrated classifiers).weights : array-like, shape = [n_classifiers], optional (default=None)If a list of `int` or `float` values are provided, the classifiersare weighted by importance; Uses uniform weights if `weights=None`."""def __init__(self, classifiers, vote='classlabel', weights=None):self.classifiers = classifiers# 使用_name_estimators函数访问集成分类器中每个成员分类器的参数self.named_classifiers = {key: value for key, valuein _name_estimators(classifiers)}# print("_name_estimators(classifiers):", _name_estimators(classifiers))# print(self.named_classifiers)self.vote = voteself.weights = weightsdef fit(self, X, y):""" Fit classifiers.Parameters----------X : {array-like, sparse matrix}, shape = [n_samples, n_features]Matrix of training samples.y : array-like, shape = [n_samples]Vector of target class labels.Returns-------self : object"""if self.vote not in ('probability', 'classlabel'):raise ValueError("vote must be 'probability' or 'classlabel'""; got (vote=%r)"% self.vote)if self.weights and len(self.weights) != len(self.classifiers):raise ValueError('Number of classifiers and weights must be equal''; got %d weights, %d classifiers'% (len(self.weights), len(self.classifiers)))# Use LabelEncoder to ensure class labels start with 0, which# is important for np.argmax call in self.predictself.lablenc_ = LabelEncoder()self.lablenc_.fit(y)self.classes_ = self.lablenc_.classes_self.classifiers_ = []for clf in self.classifiers:fitted_clf = clone(clf).fit(X, self.lablenc_.transform(y))self.classifiers_.append(fitted_clf)return selfdef predict(self, X):""" Predict class labels for X.Parameters----------X : {array-like, sparse matrix}, shape = [n_samples, n_features]Matrix of training samples.Returns----------maj_vote : array-like, shape = [n_samples]Predicted class labels."""if self.vote == 'probability':maj_vote = np.argmax(self.predict_proba(X), axis=1)# print("maj_vote:", maj_vote)else:  # 'classlabel' vote#  Collect results from clf.predict calls# np.array （默认情况下）将会copy该对象，而 np.asarray 除非必要，否则不会copy该对象。predictions = np.asarray([clf.predict(X) for clf in self.classifiers_]).T# print("predictions:", predictions)# 将arr数组的每一个元素经过func函数变换形成的一个新数组maj_vote = np.apply_along_axis(lambda x: np.argmax(np.bincount(x, weights=self.weights)),axis=1,arr=predictions)# print("转化前maj_vote:", maj_vote)maj_vote = self.lablenc_.inverse_transform(maj_vote)# print("转化后的maj_vote:", maj_vote)return maj_votedef predict_proba(self, X):""" Predict class probabilities for X.Parameters----------X : {array-like, sparse matrix}, shape = [n_samples, n_features]Training vectors, where n_samples is the number of samples andn_features is the number of features.Returns----------avg_proba : array-like, shape = [n_samples, n_classes]Weighted average probability for each class per sample."""probas = np.asarray([clf.predict_proba(X) for clf in self.classifiers_])avg_proba = np.average(probas, axis=0, weights=self.weights)# print("probas:", probas)# print("avg_proba:", avg_proba)return avg_probadef get_params(self, deep=True):""" Get classifier parameter names for GridSearch"""if not deep:return super(MajorityVoteClassifier, self).get_params(deep=False)else:out = self.named_classifiers.copy()for name, step in six.iteritems(self.named_classifiers):for key, value in six.iteritems(step.get_params(deep=True)):out['%s__%s' % (name, key)] = valuereturn outiris = datasets.load_iris()
X, y = iris.data[50:, [1, 2]], iris.target[50:]
# print("y:", y)
le = LabelEncoder()
y = le.fit_transform(y)
# print("y:", y)X_train, X_test, y_train, y_test = train_test_split(X, y,test_size=0.5,random_state=1,stratify=y)clf1 = LogisticRegression(penalty='l2', C=0.001, random_state=1)
'''
criterion这个参数正是用来决定不纯度的计算方法。sklearn提供了两种选择：
1）输入”entropy“，使用信息熵
2）输入”gini“，使用基尼系数
'''
clf2 = DecisionTreeClassifier(max_depth=1, criterion='entropy', random_state=0)# metric:用于树的距离度量。默认'minkowski与P = 2（即欧氏度量）
clf3 = KNeighborsClassifier(n_neighbors=1, p=2, metric='minkowski')pipe1 = Pipeline([['sc', StandardScaler()],['clf', clf1]])
pipe3 = Pipeline([['sc', StandardScaler()],['clf', clf3]])clf_labels = ['Logistic regression', 'Decision tree', 'KNN']# 在将它们构建成集成分类器之前，先通过对训练集进行10次交叉验证以评估每个分类器模型的性能
# print('10-fold cross validation:\n')
# for clf, label in zip([pipe1, clf2, pipe3], clf_labels):
#     scores = cross_val_score(estimator=clf,
#                              X=X_train,
#                              y=y_train,
#                              cv=10,
#                              scoring='roc_auc')
#     print("ROC AUC: %0.2f (+/- %0.2f) [%s]" % (scores.mean(), scores.std(), label))# Majority Rule (hard) Votingmv_clf = MajorityVoteClassifier(classifiers=[pipe1, clf2, pipe3])
y_pred_ = mv_clf.fit(X_train, y_train).predict(X_test)
print("y_pred_:", y_pred_)
print("get_params():", mv_clf.get_params())
print("get_params(deep=False):", mv_clf.get_params(deep=False))clf_labels += ['Majority voting']
all_clf = [pipe1, clf2, pipe3, mv_clf]# 可以看到各个分类器的MajorityVotingClassifier的性能已经在有10个分区的交叉验证评估上得到改善
# for clf, label in zip(all_clf, clf_labels):
#     scores = cross_val_score(estimator=clf,
#                              X=X_train,
#                              y=y_train,
#                              cv=10,
#                              scoring='roc_auc')
#     print("ROC AUC: %0.2f (+/- %0.2f) [%s]"
#           % (scores.mean(), scores.std(), label))# # Evaluating and tuning the ensemble classifier
# ROC结果图
# 基于测试数据来计算ROC曲线，以检查MajorityVoteClassifier对未见过的数据是否泛化良好。
colors = ['black', 'orange', 'blue', 'green']
linestyles = [':', '--', '-.', '-']
for clf, label, clr, ls in zip(all_clf, clf_labels, colors, linestyles):# assuming the label of the positive class is 1y_pred = clf.fit(X_train, y_train).predict_proba(X_test)[:, 1]fpr, tpr, thresholds = roc_curve(y_true=y_test,y_score=y_pred)roc_auc = auc(x=fpr, y=tpr)  # 计算曲线下面积plt.plot(fpr, tpr,color=clr,linestyle=ls,label='%s (auc = %0.2f)' % (label, roc_auc))plt.legend(loc='lower right')
plt.plot([0, 1], [0, 1],linestyle='--',color='gray',linewidth=2)plt.xlim([-0.1, 1.1])
plt.ylim([-0.1, 1.1])
plt.grid(alpha=0.5)
plt.xlabel('False positive rate (FPR)')
plt.ylabel('True positive rate (TPR)')#plt.savefig('images/07_04', dpi=300)
plt.show()# 看看集成分类器决策区域
# 集成分类器决策区域似乎是单个分类器决 策区域的混合体
sc = StandardScaler()
X_train_std = sc.fit_transform(X_train)all_clf = [pipe1, clf2, pipe3, mv_clf]x_min = X_train_std[:, 0].min() - 1
x_max = X_train_std[:, 0].max() + 1
y_min = X_train_std[:, 1].min() - 1
y_max = X_train_std[:, 1].max() + 1xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),np.arange(y_min, y_max, 0.1))f, axarr = plt.subplots(nrows=2, ncols=2,sharex='col',sharey='row',figsize=(7, 5))for idx, clf, tt in zip(product([0, 1], [0, 1]),all_clf, clf_labels):clf.fit(X_train_std, y_train)Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])Z = Z.reshape(xx.shape)axarr[idx[0], idx[1]].contourf(xx, yy, Z, alpha=0.3)axarr[idx[0], idx[1]].scatter(X_train_std[y_train == 0, 0],X_train_std[y_train == 0, 1],c='blue',marker='^',s=50)axarr[idx[0], idx[1]].scatter(X_train_std[y_train == 1, 0],X_train_std[y_train == 1, 1],c='green',marker='o',s=50)axarr[idx[0], idx[1]].set_title(tt)# plt.savefig('images/07_05', dpi=300)
plt.show()
# 访问单个分类器的属性
print(mv_clf.get_params())# 通过网格搜索来优化逻辑回归分类器的逆正则化 参数C和决策树深度
params = {'decisiontreeclassifier__max_depth': [1, 2],'pipeline-1__clf__C': [0.001, 0.1, 100.0]}grid = GridSearchCV(estimator=mv_clf,param_grid=params,cv=10,scoring='roc_auc')
grid.fit(X_train, y_train)# 网格搜索完成后，可以列出不同的超参数值组合，以及通过10个分区的交叉验证计算所获得ROC AUC的平均评分
for r, _ in enumerate(grid.cv_results_['mean_test_score']):print("%0.3f +/- %0.2f %r"% (grid.cv_results_['mean_test_score'][r],grid.cv_results_['std_test_score'][r] / 2.0,grid.cv_results_['params'][r]))print('Best parameters: %s' % grid.best_params_)
print('Accuracy: %.2f' % grid.best_score_)

运行结果：
y_pred_: [0 0 0 0 0 1 1 1 1 0 0 1 0 1 1 1 1 0 1 0 1 0 0 0 1 0 1 0 0 1 1 0 0 1 0 1 0
1 1 0 1 0 0 1 0 0 1 0 0 0]
get_params(): {‘pipeline-1’: Pipeline(steps=[[‘sc’, StandardScaler()],
[‘clf’, LogisticRegression(C=0.001, random_state=1)]]), ‘decisiontreeclassifier’: DecisionTreeClassifier(criterion=‘entropy’, max_depth=1, random_state=0), ‘pipeline-2’: Pipeline(steps=[[‘sc’, StandardScaler()],
[‘clf’, KNeighborsClassifier(n_neighbors=1)]]), ‘pipeline-1__memory’: None, ‘pipeline-1__steps’: [[‘sc’, StandardScaler()], [‘clf’, LogisticRegression(C=0.001, random_state=1)]], ‘pipeline-1__verbose’: False, ‘pipeline-1__sc’: StandardScaler(), ‘pipeline-1__clf’: LogisticRegression(C=0.001, random_state=1), ‘pipeline-1__sc__copy’: True, ‘pipeline-1__sc__with_mean’: True, ‘pipeline-1__sc__with_std’: True, ‘pipeline-1__clf__C’: 0.001, ‘pipeline-1__clf__class_weight’: None, ‘pipeline-1__clf__dual’: False, ‘pipeline-1__clf__fit_intercept’: True, ‘pipeline-1__clf__intercept_scaling’: 1, ‘pipeline-1__clf__l1_ratio’: None, ‘pipeline-1__clf__max_iter’: 100, ‘pipeline-1__clf__multi_class’: ‘auto’, ‘pipeline-1__clf__n_jobs’: None, ‘pipeline-1__clf__penalty’: ‘l2’, ‘pipeline-1__clf__random_state’: 1, ‘pipeline-1__clf__solver’: ‘lbfgs’, ‘pipeline-1__clf__tol’: 0.0001, ‘pipeline-1__clf__verbose’: 0, ‘pipeline-1__clf__warm_start’: False, ‘decisiontreeclassifier__ccp_alpha’: 0.0, ‘decisiontreeclassifier__class_weight’: None, ‘decisiontreeclassifier__criterion’: ‘entropy’, ‘decisiontreeclassifier__max_depth’: 1, ‘decisiontreeclassifier__max_features’: None, ‘decisiontreeclassifier__max_leaf_nodes’: None, ‘decisiontreeclassifier__min_impurity_decrease’: 0.0, ‘decisiontreeclassifier__min_impurity_split’: None, ‘decisiontreeclassifier__min_samples_leaf’: 1, ‘decisiontreeclassifier__min_samples_split’: 2, ‘decisiontreeclassifier__min_weight_fraction_leaf’: 0.0, ‘decisiontreeclassifier__presort’: ‘deprecated’, ‘decisiontreeclassifier__random_state’: 0, ‘decisiontreeclassifier__splitter’: ‘best’, ‘pipeline-2__memory’: None, ‘pipeline-2__steps’: [[‘sc’, StandardScaler()], [‘clf’, KNeighborsClassifier(n_neighbors=1)]], ‘pipeline-2__verbose’: False, ‘pipeline-2__sc’: StandardScaler(), ‘pipeline-2__clf’: KNeighborsClassifier(n_neighbors=1), ‘pipeline-2__sc__copy’: True, ‘pipeline-2__sc__with_mean’: True, ‘pipeline-2__sc__with_std’: True, ‘pipeline-2__clf__algorithm’: ‘auto’, ‘pipeline-2__clf__leaf_size’: 30, ‘pipeline-2__clf__metric’: ‘minkowski’, ‘pipeline-2__clf__metric_params’: None, ‘pipeline-2__clf__n_jobs’: None, ‘pipeline-2__clf__n_neighbors’: 1, ‘pipeline-2__clf__p’: 2, ‘pipeline-2__clf__weights’: ‘uniform’}
get_params(deep=False): {‘classifiers’: [Pipeline(steps=[[‘sc’, StandardScaler()],
[‘clf’, LogisticRegression(C=0.001, random_state=1)]]), DecisionTreeClassifier(criterion=‘entropy’, max_depth=1, random_state=0), Pipeline(steps=[[‘sc’, StandardScaler()],
[‘clf’, KNeighborsClassifier(n_neighbors=1)]])], ‘vote’: ‘classlabel’, ‘weights’: None}
{‘pipeline-1’: Pipeline(steps=[(‘sc’, StandardScaler()),
[‘clf’, LogisticRegression(C=0.001, random_state=1)]]), ‘decisiontreeclassifier’: DecisionTreeClassifier(criterion=‘entropy’, max_depth=1, random_state=0), ‘pipeline-2’: Pipeline(steps=[(‘sc’, StandardScaler()),
[‘clf’, KNeighborsClassifier(n_neighbors=1)]]), ‘pipeline-1__memory’: None, ‘pipeline-1__steps’: [(‘sc’, StandardScaler()), [‘clf’, LogisticRegression(C=0.001, random_state=1)]], ‘pipeline-1__verbose’: False, ‘pipeline-1__sc’: StandardScaler(), ‘pipeline-1__clf’: LogisticRegression(C=0.001, random_state=1), ‘pipeline-1__sc__copy’: True, ‘pipeline-1__sc__with_mean’: True, ‘pipeline-1__sc__with_std’: True, ‘pipeline-1__clf__C’: 0.001, ‘pipeline-1__clf__class_weight’: None, ‘pipeline-1__clf__dual’: False, ‘pipeline-1__clf__fit_intercept’: True, ‘pipeline-1__clf__intercept_scaling’: 1, ‘pipeline-1__clf__l1_ratio’: None, ‘pipeline-1__clf__max_iter’: 100, ‘pipeline-1__clf__multi_class’: ‘auto’, ‘pipeline-1__clf__n_jobs’: None, ‘pipeline-1__clf__penalty’: ‘l2’, ‘pipeline-1__clf__random_state’: 1, ‘pipeline-1__clf__solver’: ‘lbfgs’, ‘pipeline-1__clf__tol’: 0.0001, ‘pipeline-1__clf__verbose’: 0, ‘pipeline-1__clf__warm_start’: False, ‘decisiontreeclassifier__ccp_alpha’: 0.0, ‘decisiontreeclassifier__class_weight’: None, ‘decisiontreeclassifier__criterion’: ‘entropy’, ‘decisiontreeclassifier__max_depth’: 1, ‘decisiontreeclassifier__max_features’: None, ‘decisiontreeclassifier__max_leaf_nodes’: None, ‘decisiontreeclassifier__min_impurity_decrease’: 0.0, ‘decisiontreeclassifier__min_impurity_split’: None, ‘decisiontreeclassifier__min_samples_leaf’: 1, ‘decisiontreeclassifier__min_samples_split’: 2, ‘decisiontreeclassifier__min_weight_fraction_leaf’: 0.0, ‘decisiontreeclassifier__presort’: ‘deprecated’, ‘decisiontreeclassifier__random_state’: 0, ‘decisiontreeclassifier__splitter’: ‘best’, ‘pipeline-2__memory’: None, ‘pipeline-2__steps’: [(‘sc’, StandardScaler()), [‘clf’, KNeighborsClassifier(n_neighbors=1)]], ‘pipeline-2__verbose’: False, ‘pipeline-2__sc’: StandardScaler(), ‘pipeline-2__clf’: KNeighborsClassifier(n_neighbors=1), ‘pipeline-2__sc__copy’: True, ‘pipeline-2__sc__with_mean’: True, ‘pipeline-2__sc__with_std’: True, ‘pipeline-2__clf__algorithm’: ‘auto’, ‘pipeline-2__clf__leaf_size’: 30, ‘pipeline-2__clf__metric’: ‘minkowski’, ‘pipeline-2__clf__metric_params’: None, ‘pipeline-2__clf__n_jobs’: None, ‘pipeline-2__clf__n_neighbors’: 1, ‘pipeline-2__clf__p’: 2, ‘pipeline-2__clf__weights’: ‘uniform’}
0.983 +/- 0.02 {‘decisiontreeclassifier__max_depth’: 1, ‘pipeline-1__clf__C’: 0.001}
0.983 +/- 0.02 {‘decisiontreeclassifier__max_depth’: 1, ‘pipeline-1__clf__C’: 0.1}
0.967 +/- 0.05 {‘decisiontreeclassifier__max_depth’: 1, ‘pipeline-1__clf__C’: 100.0}
0.983 +/- 0.02 {‘decisiontreeclassifier__max_depth’: 2, ‘pipeline-1__clf__C’: 0.001}
0.983 +/- 0.02 {‘decisiontreeclassifier__max_depth’: 2, ‘pipeline-1__clf__C’: 0.1}
0.967 +/- 0.05 {‘decisiontreeclassifier__max_depth’: 2, ‘pipeline-1__clf__C’: 100.0}
Best parameters: {‘decisiontreeclassifier__max_depth’: 1, ‘pipeline-1__clf__C’: 0.001}
Accuracy: 0.98

备注：scikit-learn实现了更复杂的多数票分类器。可以从scikit-learn 0.17 或者更新版本中找到sklearn.ensemble.VotingClassifier。