Scikit-Learn Feature Engineering

Real-World Scenario

A machine learning engineer joins a company and receives a raw dataset: ages stored as strings, salary in different currencies, job titles with 200 unique values, dates, geographic coordinates, and a dozen numeric features on wildly different scales. No model trains well on this raw data. Feature engineering transforms it into something a model can learn from — it’s where most of the accuracy improvements come from in practice.

Scaling Numeric Features

import numpy as np
from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler

rng = np.random.default_rng(42)
X = rng.standard_normal((100, 3)) * np.array([1, 100, 10000])  # different scales

# StandardScaler: (x - mean) / std → zero mean, unit variance
ss = StandardScaler()
X_standard = ss.fit_transform(X)
print(f"StandardScaler - Mean: {X_standard.mean(axis=0).round(6)}, Std: {X_standard.std(axis=0).round(4)}")

# MinMaxScaler: (x - min) / (max - min) → [0, 1]
mms = MinMaxScaler()
X_minmax = mms.fit_transform(X)
print(f"MinMaxScaler   - Min: {X_minmax.min(axis=0).round(4)}, Max: {X_minmax.max(axis=0).round(4)}")

# RobustScaler: uses median and IQR — resistant to outliers
# Best when your data has significant outliers
rs = RobustScaler()
X_robust = rs.fit_transform(X)

# Always fit on training data only — apply to test with transform()
from sklearn.model_selection import train_test_split
X_train, X_test = train_test_split(X, test_size=0.2, random_state=42)

scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)   # fit + transform on train
X_test_scaled  = scaler.transform(X_test)        # transform only on test (no refit)

Encoding Categorical Features

import pandas as pd
import numpy as np
from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, LabelEncoder

df = pd.DataFrame({
    "color":   ["red", "blue", "green", "red", "blue"],
    "size":    ["S", "M", "L", "XL", "M"],
    "grade":   ["A", "B", "C", "A", "B"],
})

# One-Hot Encoding — for nominal categories (no inherent order)
ohe = OneHotEncoder(sparse_output=False, handle_unknown="ignore")
color_encoded = ohe.fit_transform(df[["color"]])
print("One-hot columns:", ohe.get_feature_names_out())
print(color_encoded)
# [[0. 0. 1.]  — red
#  [1. 0. 0.]  — blue
#  [0. 1. 0.]  — green

# Ordinal Encoding — for ordered categories
size_order = [["S", "M", "L", "XL"]]
oe = OrdinalEncoder(categories=size_order)
size_encoded = oe.fit_transform(df[["size"]])
print("Size ordinal:", size_encoded.flatten())   # [0. 1. 2. 3. 1.]

# LabelEncoder — for the target variable y only (not features)
le = LabelEncoder()
grade_encoded = le.fit_transform(df["grade"])
print("Grade labels:", grade_encoded)   # [0 1 2 0 1]

Handling Missing Values

import numpy as np
import pandas as pd
from sklearn.impute import SimpleImputer, KNNImputer

rng = np.random.default_rng(42)
X = rng.standard_normal((200, 4))

# Inject 10% missing values
mask = rng.random((200, 4)) < 0.1
X[mask] = np.nan

# SimpleImputer
imputer_mean   = SimpleImputer(strategy="mean")
imputer_median = SimpleImputer(strategy="median")
imputer_mode   = SimpleImputer(strategy="most_frequent")
imputer_const  = SimpleImputer(strategy="constant", fill_value=0)

X_mean_filled = imputer_mean.fit_transform(X)
print("Missing after imputation:", np.isnan(X_mean_filled).sum())   # 0

# KNN Imputer — fills with weighted mean of k nearest neighbors
# Better for structured data where feature correlation matters
knn_imputer = KNNImputer(n_neighbors=5)
X_knn_filled = knn_imputer.fit_transform(X)

# Add indicator columns to preserve the "was missing" signal
from sklearn.impute import MissingIndicator
indicator = MissingIndicator(features="missing-only")
X_indicators = indicator.fit_transform(X)  # binary columns showing which were missing
print("Indicator shape:", X_indicators.shape)

Feature Creation

import numpy as np
import pandas as pd
from sklearn.preprocessing import PolynomialFeatures

rng = np.random.default_rng(42)
X = rng.standard_normal((100, 2))  # two features: x1, x2

# Polynomial features — add x1², x2², x1*x2 interaction terms
poly = PolynomialFeatures(degree=2, include_bias=False)
X_poly = poly.fit_transform(X)

print("Original features:", X.shape[1])          # 2
print("After degree-2 poly:", X_poly.shape[1])   # 5: x1, x2, x1², x1*x2, x2²
print("Feature names:", poly.get_feature_names_out(["x1", "x2"]))
# ['x1', 'x2', 'x1^2', 'x1 x2', 'x2^2']

# Real-world feature engineering with Pandas + Scikit-Learn
df = pd.DataFrame({
    "order_date": pd.date_range("2024-01-01", periods=100, freq="D"),
    "age":        rng.integers(18, 70, 100),
    "salary":     rng.uniform(30000, 200000, 100),
    "num_orders": rng.integers(0, 50, 100),
    "last_order_days": rng.integers(0, 365, 100),
})

# Temporal features
df["day_of_week"]  = df["order_date"].dt.dayofweek
df["month"]        = df["order_date"].dt.month
df["is_weekend"]   = (df["day_of_week"] >= 5).astype(int)
df["quarter"]      = df["order_date"].dt.quarter

# Ratio features
df["orders_per_year"] = df["num_orders"] / ((df["age"] - 18).clip(lower=1))

# Bin continuous variables
df["age_group"] = pd.cut(df["age"], bins=[18, 30, 45, 60, 100],
                           labels=["18-30", "30-45", "45-60", "60+"])
df["salary_tier"] = pd.qcut(df["salary"], q=4, labels=["Q1", "Q2", "Q3", "Q4"])

print(df.head(5))

Feature Selection

from sklearn.datasets import load_diabetes
from sklearn.feature_selection import (
    SelectKBest, f_regression,
    SelectFromModel, RFE
)
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.linear_model import Lasso
import numpy as np

X, y = load_diabetes(return_X_y=True)
print(f"Original features: {X.shape[1]}")   # 10

# Filter method: select k best features based on F-statistic
selector_kbest = SelectKBest(score_func=f_regression, k=5)
X_kbest = selector_kbest.fit_transform(X, y)
print(f"After SelectKBest: {X_kbest.shape[1]}")

# Embedded method: Lasso drives unimportant feature coefficients to 0
lasso = Lasso(alpha=0.01)
selector_lasso = SelectFromModel(lasso)
X_lasso = selector_lasso.fit_transform(X, y)
print(f"After Lasso selection: {X_lasso.shape[1]}")

# Wrapper method: Recursive Feature Elimination
gbm = GradientBoostingRegressor(n_estimators=100, random_state=42)
rfe = RFE(estimator=gbm, n_features_to_select=5, step=1)
X_rfe = rfe.fit_transform(X, y)
print(f"After RFE: {X_rfe.shape[1]}")
print(f"Selected features: {np.where(rfe.support_)[0]}")