Алгоритм Градієнтного бустингу
На цьому уроці розглянемо градієнтний бустінг: для чого він використовується та як його можна застосувати для побудови високоточних моделей прогнозування. Детально розберемо принцип послідовного навчання слабких моделей і побудуємо алгоритм градієнтного бустінгу з нуля, щоб зрозуміти його внутрішню логіку та роботу.
Code and results
Published version after Merge. Results were saved by the author; code does not run on this page.
Python · 1
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import train_test_split
from sklearn.datasets import fetch_california_housing, load_breast_cancer
from sklearn.ensemble import GradientBoostingRegressor, GradientBoostingClassifier
from sklearn.metrics import r2_score, accuracy_score, classification_report
import numpy as npPython · 2
X, y = fetch_california_housing(return_X_y=True, as_frame=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
gbm = GradientBoostingRegressor(
n_estimators=200,
learning_rate=0.1,
max_depth=3,
random_state=42,
)
gbm.fit(X_train, y_train)
pred = gbm.predict(X_test)
print(r2_score(y_test, pred))Results saved by the author in Colab
0.8004451261281281
Python · 3
class GBRegression:
def __init__(self, n=100, max_depth=3, lr=0.1, random_state=42):
self.n = n
self.max_depth = max_depth
self.lr = lr
self.models = []
self.random_state = random_state
def fit(self, X, y):
self.models = []
self.init_pred = np.mean(y)
total_pred = np.full(len(y), self.init_pred)
residual = y - total_pred
for i in range(self.n):
model = DecisionTreeRegressor(max_depth=self.max_depth, random_state=self.random_state)
model.fit(X, residual)
self.models.append(model)
total_pred += self.lr * model.predict(X)
residual = y - total_pred
return self
def predict(self, X):
pred = np.full(len(X), self.init_pred)
for model in self.models:
pred += self.lr * model.predict(X)
return pred
gbm = GBRegression(n=200)
gbm.fit(X_train, y_train)
pred = gbm.predict(X_test)
print(r2_score(y_test, pred))
Results saved by the author in Colab
0.800599655795802
Python · 4
X, y = load_breast_cancer(return_X_y=True, as_frame=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
gbm_clf = GradientBoostingClassifier(
n_estimators=200,
learning_rate=0.1,
max_depth=3,
random_state=42,
)
gbm_clf.fit(X_train, y_train)
pred = gbm_clf.predict(X_test)
print(accuracy_score(y_test, pred))
print(classification_report(y_test, pred))
Results saved by the author in Colab
0.956140350877193
precision recall f1-score support
0 0.95 0.93 0.94 43
1 0.96 0.97 0.97 71
accuracy 0.96 114
macro avg 0.96 0.95 0.95 114
weighted avg 0.96 0.96 0.96 114
Python · 5
class GBClassification:
def __init__(self, n=100, max_depth=3, lr=0.1, random_state=42):
self.n = n
self.max_depth = max_depth
self.lr = lr
self.random_state = random_state
self.models = []
@staticmethod
def sigmoid(z):
return 1 / (1 + np.exp(-z))
def fit(self, X, y):
self.models = []
p0 = np.mean(y)
self.init_pred = np.log(p0 / (1 - p0))
total_pred = np.full(len(y), self.init_pred)
residual = y - self.sigmoid(total_pred)
for i in range(self.n):
model = DecisionTreeRegressor(max_depth=self.max_depth, random_state=self.random_state)
model.fit(X, residual)
self.models.append(model)
total_pred += self.lr * model.predict(X)
residual = y - self.sigmoid(total_pred)
return self
def predict(self, X, threshold=0.5):
pred = np.full(len(X), self.init_pred)
for model in self.models:
pred += self.lr * model.predict(X)
proba = self.sigmoid(pred)
return (proba >= threshold).astype(int)Python · 6
gbm_clf = GBClassification()
gbm_clf.fit(X_train, y_train)
pred = gbm_clf.predict(X_test)
print(accuracy_score(y_test, pred))
print(classification_report(y_test, pred))Results saved by the author in Colab
0.956140350877193
precision recall f1-score support
0 0.95 0.93 0.94 43
1 0.96 0.97 0.97 71
accuracy 0.96 114
macro avg 0.96 0.95 0.95 114
weighted avg 0.96 0.96 0.96 114
Python · 7
Community reviews
Непогана учбова реалізація. Approve.
