TelegramOpen channel
Sofiia N
Merged · Machine Learning

Алгоритм Градієнтного бустингу

На цьому уроці розглянемо градієнтний бустінг: для чого він використовується та як його можна застосувати для побудови високоточних моделей прогнозування. Детально розберемо принцип послідовного навчання слабких моделей і побудуємо алгоритм градієнтного бустінгу з нуля, щоб зрозуміти його внутрішню логіку та роботу.

Code and results

Published version after Merge. Results were saved by the author; code does not run on this page.

Python · 1
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import train_test_split
from sklearn.datasets import fetch_california_housing, load_breast_cancer
from sklearn.ensemble import GradientBoostingRegressor, GradientBoostingClassifier
from sklearn.metrics import r2_score, accuracy_score, classification_report
import numpy as np
Python · 2
X, y = fetch_california_housing(return_X_y=True, as_frame=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

gbm = GradientBoostingRegressor(
    n_estimators=200,
    learning_rate=0.1,
    max_depth=3,
    random_state=42,
)
gbm.fit(X_train, y_train)

pred = gbm.predict(X_test)
print(r2_score(y_test, pred))
Results saved by the author in Colab
0.8004451261281281
Python · 3
class GBRegression:
  def __init__(self, n=100, max_depth=3, lr=0.1, random_state=42):
      self.n = n
      self.max_depth = max_depth
      self.lr = lr
      self.models = []
      self.random_state = random_state

  def fit(self, X, y):
    self.models = []
    self.init_pred = np.mean(y)
    total_pred = np.full(len(y), self.init_pred)
    residual = y - total_pred

    for i in range(self.n):
      model = DecisionTreeRegressor(max_depth=self.max_depth, random_state=self.random_state)
      model.fit(X, residual)

      self.models.append(model)
      total_pred += self.lr * model.predict(X)
      residual = y - total_pred

    return self

  def predict(self, X):
    pred = np.full(len(X), self.init_pred)
    for model in self.models:
      pred += self.lr * model.predict(X)
    return pred

gbm = GBRegression(n=200)
gbm.fit(X_train, y_train)

pred = gbm.predict(X_test)
print(r2_score(y_test, pred))
Results saved by the author in Colab
0.800599655795802
Python · 4
X, y = load_breast_cancer(return_X_y=True, as_frame=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

gbm_clf = GradientBoostingClassifier(
    n_estimators=200,
    learning_rate=0.1,
    max_depth=3,
    random_state=42,
)

gbm_clf.fit(X_train, y_train)
pred = gbm_clf.predict(X_test)
print(accuracy_score(y_test, pred))
print(classification_report(y_test, pred))
Results saved by the author in Colab
0.956140350877193
              precision    recall  f1-score   support

           0       0.95      0.93      0.94        43
           1       0.96      0.97      0.97        71

    accuracy                           0.96       114
   macro avg       0.96      0.95      0.95       114
weighted avg       0.96      0.96      0.96       114

Python · 5
class GBClassification:
  def __init__(self, n=100, max_depth=3, lr=0.1, random_state=42):
    self.n = n
    self.max_depth = max_depth
    self.lr = lr
    self.random_state = random_state
    self.models = []

  @staticmethod
  def sigmoid(z):
    return 1 / (1 + np.exp(-z))

  def fit(self, X, y):
    self.models = []

    p0 = np.mean(y)
    self.init_pred = np.log(p0 / (1 - p0))
    total_pred = np.full(len(y), self.init_pred)
    residual = y - self.sigmoid(total_pred)

    for i in range(self.n):
      model = DecisionTreeRegressor(max_depth=self.max_depth, random_state=self.random_state)
      model.fit(X, residual)
      self.models.append(model)

      total_pred += self.lr * model.predict(X)
      residual = y - self.sigmoid(total_pred)

    return self

  def predict(self, X, threshold=0.5):
    pred = np.full(len(X), self.init_pred)
    for model in self.models:
      pred += self.lr * model.predict(X)
    proba = self.sigmoid(pred)
    return (proba >= threshold).astype(int)
Python · 6
gbm_clf = GBClassification()
gbm_clf.fit(X_train, y_train)

pred = gbm_clf.predict(X_test)
print(accuracy_score(y_test, pred))
print(classification_report(y_test, pred))
Results saved by the author in Colab
0.956140350877193
              precision    recall  f1-score   support

           0       0.95      0.93      0.94        43
           1       0.96      0.97      0.97        71

    accuracy                           0.96       114
   macro avg       0.96      0.95      0.95       114
weighted avg       0.96      0.96      0.96       114

Python · 7

Community reviews

Anar LavrenovApprove

Непогана учбова реалізація. Approve.