Scikit-Learn Classification

Real-World Scenario

A bank’s fraud detection team needs to classify 2 million daily transactions as legitimate or fraudulent. They need to compare multiple algorithms, understand the precision-recall tradeoff, and deploy the model that maximizes fraud detection while keeping the false-positive rate below 1% (to avoid blocking legitimate customers). This is the full classification workflow.

Logistic Regression — The Baseline

from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import classification_report, roc_auc_score
import numpy as np

X, y = load_breast_cancer(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)

scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s  = scaler.transform(X_test)

lr = LogisticRegression(C=1.0, max_iter=1000, random_state=42)
lr.fit(X_train_s, y_train)

print("Logistic Regression:")
print(classification_report(y_test, lr.predict(X_test_s)))
print(f"AUC-ROC: {roc_auc_score(y_test, lr.predict_proba(X_test_s)[:, 1]):.4f}")

Random Forest Classifier

from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score, classification_report
import numpy as np

X, y = load_breast_cancer(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)

rf = RandomForestClassifier(
    n_estimators=200,      # number of trees
    max_depth=None,        # grow until leaves are pure
    min_samples_split=5,   # min samples to split a node
    min_samples_leaf=2,    # min samples in a leaf
    class_weight="balanced",  # handle imbalance
    n_jobs=-1,             # use all CPU cores
    random_state=42,
)
rf.fit(X_train, y_train)

print("Random Forest:")
print(classification_report(y_test, rf.predict(X_test)))
print(f"AUC-ROC: {roc_auc_score(y_test, rf.predict_proba(X_test)[:, 1]):.4f}")

# Feature importances — how much each feature reduces impurity
feature_names = load_breast_cancer().feature_names
importances = rf.feature_importances_
top_idx = np.argsort(importances)[::-1][:10]
print("\nTop 10 features:")
for i in top_idx:
    print(f"  {feature_names[i]:40s}: {importances[i]:.4f}")

Gradient Boosting Classifier

from sklearn.ensemble import GradientBoostingClassifier, HistGradientBoostingClassifier
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score

X, y = load_breast_cancer(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)

# HistGradientBoostingClassifier — faster, supports missing values natively
model = HistGradientBoostingClassifier(
    max_iter=300,
    learning_rate=0.05,
    max_depth=4,
    min_samples_leaf=20,
    l2_regularization=0.1,
    random_state=42,
)
model.fit(X_train, y_train)

print(f"GBM AUC-ROC: {roc_auc_score(y_test, model.predict_proba(X_test)[:, 1]):.4f}")

Support Vector Machine

from sklearn.svm import SVC
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import roc_auc_score

X, y = load_breast_cancer(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)

scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s  = scaler.transform(X_test)

# probability=True needed for predict_proba — slightly slower
svm = SVC(C=10, kernel="rbf", gamma="scale", probability=True, random_state=42)
svm.fit(X_train_s, y_train)

print(f"SVM AUC-ROC: {roc_auc_score(y_test, svm.predict_proba(X_test_s)[:, 1]):.4f}")

Comparing Multiple Classifiers

from sklearn.datasets import make_classification
from sklearn.model_selection import cross_validate, StratifiedKFold
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.svm import SVC
import pandas as pd
import numpy as np

X, y = make_classification(n_samples=2000, n_features=20, n_informative=10,
                            weights=[0.7, 0.3], random_state=42)

classifiers = {
    "Logistic Regression": Pipeline([
        ("scaler", StandardScaler()),
        ("clf",    LogisticRegression(class_weight="balanced", max_iter=1000)),
    ]),
    "Random Forest": RandomForestClassifier(
        n_estimators=200, class_weight="balanced", n_jobs=-1, random_state=42
    ),
    "Gradient Boosting": GradientBoostingClassifier(n_estimators=200, random_state=42),
    "SVM": Pipeline([
        ("scaler", StandardScaler()),
        ("clf",    SVC(class_weight="balanced", probability=True)),
    ]),
}

cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
results = []

for name, clf in classifiers.items():
    scores = cross_validate(clf, X, y, cv=cv, scoring=["roc_auc", "f1", "accuracy"])
    results.append({
        "Model":    name,
        "AUC-ROC":  f"{scores['test_roc_auc'].mean():.4f} ± {scores['test_roc_auc'].std():.4f}",
        "F1":       f"{scores['test_f1'].mean():.4f} ± {scores['test_f1'].std():.4f}",
        "Accuracy": f"{scores['test_accuracy'].mean():.4f} ± {scores['test_accuracy'].std():.4f}",
    })

print(pd.DataFrame(results).to_string(index=False))

Adjusting the Decision Threshold

from sklearn.datasets import make_classification
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import precision_recall_curve, f1_score
import numpy as np

X, y = make_classification(n_samples=5000, weights=[0.95, 0.05], random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)

model = GradientBoostingClassifier(random_state=42)
model.fit(X_train, y_train)
y_prob = model.predict_proba(X_test)[:, 1]

# Find threshold that maximizes F1 score
precision, recall, thresholds = precision_recall_curve(y_test, y_prob)
f1_scores = 2 * precision * recall / (precision + recall + 1e-8)
best_threshold = thresholds[np.argmax(f1_scores[:-1])]
print(f"Default threshold (0.5):    F1 = {f1_score(y_test, y_prob >= 0.5):.4f}")
print(f"Optimal threshold ({best_threshold:.2f}): F1 = {f1_score(y_test, y_prob >= best_threshold):.4f}")