集成学习

Hard Voting硬投票法

import numpy as np
from sklearn import datasets
import  warnings
from sklearn.model_selection import train_test_split
from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
warnings.filterwarnings("ignore")


X,y = datasets.make_moons(n_samples=500, noise =0.3,random_state=42)
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=42)

voting_clf = VotingClassifier(estimators=[
    ('log_clf',LogisticRegression()),
    ('svm_clf',SVC()),
    ('dt_clf',DecisionTreeClassifier(random_state=666))
],voting = 'hard')

voting_clf.fit(X_train,y_train)
voting_clf.score(X_test,y_test)

输出准确率：0.896

Soft Voting软投票法

import numpy as np
from sklearn import datasets
import  warnings
from sklearn.model_selection import train_test_split
from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
warnings.filterwarnings("ignore")

X,y = datasets.make_moons(n_samples=500, noise =0.3,random_state=42)
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=42)

voting_clf2 = VotingClassifier(estimators=[
    ('log_clf',LogisticRegression()),
    ('svm_clf',SVC(probability=True)),
    ('dt_clf',DecisionTreeClassifier(random_state=666))
],voting = 'soft')

voting_clf2.fit(X_train,y_train)
voting_clf2.score(X_test,y_test)

输出准确率：0.912

Bagging放回取样

n_estimators=500基学习器的数量为500个
max_samples=100训练每个基学习器的样本数量为100个

import numpy as np
from sklearn import datasets
import  warnings
from sklearn.model_selection import train_test_split
from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
warnings.filterwarnings("ignore")


X,y = datasets.make_moons(n_samples=500, noise =0.3,random_state=42)
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=42)

from sklearn.ensemble import BaggingClassifier
bagging_clf = BaggingClassifier(DecisionTreeClassifier(random_state=666),n_estimators=500,max_samples=100,bootstrap=True)
bagging_clf.fit(X_train,y_train)
bagging_clf.score(X_test,y_test)

输出准确率：0.916

Random Subspaces针对特征进行随机采样

max_features : The number of features to draw from X to train each base estimator.

import numpy as np
from sklearn import datasets
import  warnings
from sklearn.model_selection import train_test_split
from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.ensemble import BaggingClassifier

warnings.filterwarnings("ignore")
X,y = datasets.make_moons(n_samples=500, noise =0.3,random_state=42)

ramdom_subspaces_clf = BaggingClassifier(DecisionTreeClassifier(random_state=666),n_estimators=500,max_samples=1.0,bootstrap=True,oob_score=True,max_features=1,bootstrap_features=True)
ramdom_subspaces_clf.fit(X,y)
ramdom_subspaces_clf.oob_score_

输出准确率：0.824

Random Patches针对样本和特征进行随机采样

from sklearn.ensemble import BaggingClassifier
import numpy as np
from sklearn import datasets
import  warnings
from sklearn.model_selection import train_test_split
from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC

warnings.filterwarnings("ignore")
X,y = datasets.make_moons(n_samples=500, noise =0.3,random_state=42)

ramdom_patches_clf = BaggingClassifier(DecisionTreeClassifier(random_state=666),n_estimators=500,max_samples=100,bootstrap=True,oob_score=True,max_features=1,bootstrap_features=True)
ramdom_patches_clf.fit(X,y)
ramdom_patches_clf.oob_score_

输出准确率：0.86

Random Forest随机森林

import numpy as np
from sklearn import datasets
import  warnings
from sklearn.model_selection import train_test_split
from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier

warnings.filterwarnings("ignore")
X,y = datasets.make_moons(n_samples=500, noise =0.3,random_state=42)

rf_clf = RandomForestClassifier(n_estimators=500,random_state=666,oob_score=True)
rf_clf.fit(X,y)
rf_clf.oob_score_

输出准确率：0.896

Extra-Trees极其随机森林

import numpy as np
from sklearn import datasets
import  warnings
from sklearn.model_selection import train_test_split
from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier

warnings.filterwarnings("ignore")
X,y = datasets.make_moons(n_samples=500, noise =0.3,random_state=42)

from sklearn.ensemble import ExtraTreesClassifier
et_clf = ExtraTreesClassifier(n_estimators=500,bootstrap=True,oob_score=True,random_state=666)
et_clf.fit(X,y)
et_clf.oob_score_

输出准确率：0.892

AdaBoosting

import numpy as np
from sklearn import datasets
import  warnings
from sklearn.model_selection import train_test_split
from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.ensemble import AdaBoostClassifier

warnings.filterwarnings("ignore")
X,y = datasets.make_moons(n_samples=500, noise =0.3,random_state=42)
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=42)

ada_clf = AdaBoostClassifier(DecisionTreeClassifier(max_depth=2),n_estimators=500,random_state=666)
ada_clf.fit(X_train,y_train)
ada_clf.score(X_test,y_test)

输出准确率：0.872

Gradient Boosting Decision Tree(GBDT)梯度提升树

import numpy as np
from sklearn import datasets
import  warnings
from sklearn.model_selection import train_test_split
from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.ensemble import GradientBoostingClassifier

warnings.filterwarnings("ignore")
X,y = datasets.make_moons(n_samples=500, noise =0.3,random_state=42)
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=42)

gb_clf = GradientBoostingClassifier(max_depth=2, n_estimators=30)
gb_clf.fit(X_train,y_train)
gb_clf.score(X_test,y_test)

输出准确率：0.912