特徴量スケーリングの理論 — 標準化・正規化・なぜ必要か

import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler

np.random.seed(42)

# 外れ値を含むデータの生成
n = 200
x_normal = np.random.normal(50, 10, n)
# 外れ値を追加
x_with_outliers = np.append(x_normal, [120, 130, 150, -20, -30])

fig, axes = plt.subplots(2, 2, figsize=(14, 10))

# (a) 元のデータ
ax = axes[0, 0]
ax.hist(x_with_outliers, bins=30, color="steelblue", alpha=0.8, edgecolor="gray")
ax.set_title(f"Original Data (mean={np.mean(x_with_outliers):.1f}, std={np.std(x_with_outliers):.1f})", fontsize=12)
ax.set_xlabel("Value", fontsize=11)
ax.grid(True, alpha=0.3)

# (b) 標準化
ax = axes[0, 1]
x_std = (x_with_outliers - np.mean(x_with_outliers)) / np.std(x_with_outliers)
ax.hist(x_std, bins=30, color="coral", alpha=0.8, edgecolor="gray")
ax.set_title(f"Standardized (mean={np.mean(x_std):.2f}, std={np.std(x_std):.2f})", fontsize=12)
ax.set_xlabel("Value", fontsize=11)
ax.grid(True, alpha=0.3)

# (c) Min-Max
ax = axes[1, 0]
x_mm = (x_with_outliers - np.min(x_with_outliers)) / (np.max(x_with_outliers) - np.min(x_with_outliers))
ax.hist(x_mm, bins=30, color="lightgreen", alpha=0.8, edgecolor="gray")
ax.set_title(f"Min-Max Normalized (range=[{np.min(x_mm):.2f}, {np.max(x_mm):.2f}])", fontsize=12)
ax.set_xlabel("Value", fontsize=11)
ax.grid(True, alpha=0.3)

# (d) ロバストスケーリング
ax = axes[1, 1]
median = np.median(x_with_outliers)
q25, q75 = np.percentile(x_with_outliers, [25, 75])
x_robust = (x_with_outliers - median) / (q75 - q25)
ax.hist(x_robust, bins=30, color="plum", alpha=0.8, edgecolor="gray")
ax.set_title(f"Robust Scaled (median={np.median(x_robust):.2f}, IQR={np.percentile(x_robust, 75)-np.percentile(x_robust, 25):.2f})", fontsize=12)
ax.set_xlabel("Value", fontsize=11)
ax.grid(True, alpha=0.3)

plt.suptitle("Feature Scaling Methods with Outliers", fontsize=14, y=1.02)
plt.tight_layout()
plt.savefig("scaling_methods.png", dpi=150, bbox_inches="tight")
plt.show()

import numpy as np
import matplotlib.pyplot as plt

np.random.seed(42)

# スケールの異なる2次元データ
n = 100
x1 = np.random.randn(n) * 100  # スケール大
x2 = np.random.randn(n) * 0.1  # スケール小
X_raw = np.column_stack([x1, x2])
w_true = np.array([0.5, 30.0])
y = X_raw @ w_true + np.random.randn(n) * 5

# 標準化
X_scaled = (X_raw - X_raw.mean(axis=0)) / X_raw.std(axis=0)
y_centered = y - y.mean()

def gd_linear(X, y, lr, n_steps):
    """勾配降下法による線形回帰"""
    w = np.zeros(X.shape[1])
    losses = []
    trajectory = [w.copy()]
    for _ in range(n_steps):
        pred = X @ w
        loss = 0.5 * np.mean((pred - y)**2)
        losses.append(loss)
        grad = X.T @ (pred - y) / len(y)
        w = w - lr * grad
        trajectory.append(w.copy())
    return np.array(trajectory), np.array(losses)

fig, axes = plt.subplots(2, 2, figsize=(14, 10))

# (a) スケーリングなしの等高線と軌跡
ax = axes[0, 0]
w1_range = np.linspace(-0.5, 1.5, 100)
w2_range = np.linspace(-10, 70, 100)
W1, W2 = np.meshgrid(w1_range, w2_range)
Z = np.zeros_like(W1)
for i in range(W1.shape[0]):
    for j in range(W1.shape[1]):
        w = np.array([W1[i,j], W2[i,j]])
        Z[i,j] = 0.5 * np.mean((X_raw @ w - y)**2)

ax.contour(W1, W2, np.log10(Z+1), levels=20, cmap="Blues", alpha=0.7)
traj_raw, losses_raw = gd_linear(X_raw, y, 1e-6, 200)
ax.plot(traj_raw[:, 0], traj_raw[:, 1], "r-o", markersize=2, linewidth=1)
ax.plot(*w_true, "g*", markersize=15, label="True $w$")
ax.set_xlabel("$w_1$", fontsize=12)
ax.set_ylabel("$w_2$", fontsize=12)
ax.set_title("Without Scaling (GD trajectory)", fontsize=13)
ax.legend(fontsize=10)
ax.grid(True, alpha=0.3)

# (b) スケーリング後の等高線と軌跡
ax = axes[0, 1]
w1_range_s = np.linspace(-80, 80, 100)
w2_range_s = np.linspace(-20, 20, 100)
W1s, W2s = np.meshgrid(w1_range_s, w2_range_s)
Zs = np.zeros_like(W1s)
for i in range(W1s.shape[0]):
    for j in range(W1s.shape[1]):
        w = np.array([W1s[i,j], W2s[i,j]])
        Zs[i,j] = 0.5 * np.mean((X_scaled @ w - y_centered)**2)

ax.contour(W1s, W2s, np.log10(Zs+1), levels=20, cmap="Blues", alpha=0.7)
traj_scaled, losses_scaled = gd_linear(X_scaled, y_centered, 0.1, 200)
ax.plot(traj_scaled[:, 0], traj_scaled[:, 1], "r-o", markersize=2, linewidth=1)
ax.set_xlabel("$w_1$ (scaled)", fontsize=12)
ax.set_ylabel("$w_2$ (scaled)", fontsize=12)
ax.set_title("With Scaling (GD trajectory)", fontsize=13)
ax.grid(True, alpha=0.3)

# (c) 損失曲線の比較
ax = axes[1, 0]
ax.plot(losses_raw / losses_raw[0], "r-", linewidth=2, label="Without scaling")
ax.plot(losses_scaled / losses_scaled[0], "b-", linewidth=2, label="With scaling")
ax.set_xlabel("Iteration", fontsize=12)
ax.set_ylabel("Normalized Loss", fontsize=12)
ax.set_title("Convergence Comparison", fontsize=13)
ax.legend(fontsize=11)
ax.grid(True, alpha=0.3)
ax.set_yscale("log")

# (d) 条件数の比較
ax = axes[1, 1]
cond_raw = np.linalg.cond(X_raw.T @ X_raw)
cond_scaled = np.linalg.cond(X_scaled.T @ X_scaled)

bars = ax.bar(["Without Scaling", "With Scaling"],
              [cond_raw, cond_scaled],
              color=["coral", "steelblue"], alpha=0.8, edgecolor="gray")
ax.set_ylabel("Condition Number $\\kappa$", fontsize=12)
ax.set_title("Condition Number of $X^TX$", fontsize=13)
ax.grid(True, alpha=0.3, axis="y")

for bar, val in zip(bars, [cond_raw, cond_scaled]):
    ax.text(bar.get_x() + bar.get_width()/2., bar.get_height() + max(cond_raw, cond_scaled)*0.02,
            f"{val:.0f}", ha="center", fontsize=11, fontweight="bold")

plt.tight_layout()
plt.savefig("scaling_convergence.png", dpi=150, bbox_inches="tight")
plt.show()

import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.linear_model import Ridge

np.random.seed(42)

# データ生成
n = 200
d = 10
X = np.random.randn(n, d) * np.random.uniform(1, 100, d)
y = X @ np.random.randn(d) + np.random.randn(n) * 5

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)

# 間違った方法: 全データでスケーリング → 分割
scaler_wrong = StandardScaler()
X_all_scaled = scaler_wrong.fit_transform(X)
X_train_wrong = X_all_scaled[:len(X_train)]
X_test_wrong = X_all_scaled[len(X_train):]

# 正しい方法: 訓練データでfit → テストデータにtransform
scaler_correct = StandardScaler()
X_train_correct = scaler_correct.fit_transform(X_train)
X_test_correct = scaler_correct.transform(X_test)

fig, axes = plt.subplots(1, 2, figsize=(14, 5.5))

# (a) 手順の図示
ax = axes[0]
steps_correct = [
    "1. Train/Test Split",
    "2. Fit scaler on Train",
    "3. Transform Train",
    "4. Transform Test\n   (using Train stats)",
    "5. Train Model",
    "6. Evaluate on Test"
]

steps_wrong = [
    "1. Fit scaler on ALL data",
    "2. Transform ALL data",
    "3. Train/Test Split",
    "4. Train Model",
    "5. Evaluate on Test",
    ""
]

y_pos = np.arange(len(steps_correct))[::-1]
ax.barh(y_pos + 0.2, [1]*6, height=0.35, color="green", alpha=0.3, label="Correct")
ax.barh(y_pos - 0.2, [1]*6, height=0.35, color="red", alpha=0.3, label="Wrong (data leak)")

for i, (sc, sw) in enumerate(zip(steps_correct, steps_wrong)):
    ax.text(0.05, y_pos[i] + 0.2, sc, va="center", fontsize=9, color="darkgreen")
    ax.text(0.05, y_pos[i] - 0.2, sw, va="center", fontsize=9, color="darkred")

ax.set_xlim(0, 1)
ax.set_yticks([])
ax.set_xticks([])
ax.set_title("Correct vs Wrong Scaling Procedure", fontsize=13)
ax.legend(fontsize=10, loc="lower right")

# (b) テストデータでの特徴量の平均の比較
ax = axes[1]

# 正しい方法: テストデータの平均は0からずれる
test_means_correct = X_test_correct.mean(axis=0)
test_means_wrong = X_test_wrong.mean(axis=0)

x_pos = np.arange(d)
width = 0.35

ax.bar(x_pos - width/2, test_means_correct, width, color="green", alpha=0.7,
       label="Correct (train-fit)")
ax.bar(x_pos + width/2, test_means_wrong, width, color="red", alpha=0.7,
       label="Wrong (all-fit)")
ax.axhline(0, color="gray", linewidth=1)

ax.set_xlabel("Feature index", fontsize=12)
ax.set_ylabel("Test data mean (after scaling)", fontsize=12)
ax.set_title("Test Data Mean: Correct vs Wrong Scaling", fontsize=13)
ax.legend(fontsize=10)
ax.grid(True, alpha=0.3, axis="y")
ax.set_xticks(x_pos)

plt.tight_layout()
plt.savefig("scaling_data_leak.png", dpi=150, bbox_inches="tight")
plt.show()