過学習の対策まとめ — Dropout・正則化・Early Stopping・Data Augmentation

import numpy as np

# Early Stopping の概念的な実装
class EarlyStopping:
    """バリデーション損失が改善しなくなったら学習を停止"""

    def __init__(self, patience=10, min_delta=1e-4):
        self.patience = patience
        self.min_delta = min_delta
        self.best_loss = np.inf
        self.counter = 0
        self.best_weights = None

    def check(self, val_loss, model_weights):
        if val_loss < self.best_loss - self.min_delta:
            self.best_loss = val_loss
            self.counter = 0
            self.best_weights = [w.copy() for w in model_weights]
            return False  # 継続
        else:
            self.counter += 1
            if self.counter >= self.patience:
                return True  # 停止
            return False  # 継続

# 使用例
early_stopper = EarlyStopping(patience=10)
print(f"patience={early_stopper.patience}: "
      f"バリデーション損失が{early_stopper.patience}エポック改善しなければ停止")
print(f"min_delta={early_stopper.min_delta}: "
      f"改善と見なす最小の損失減少量")

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_moons
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

np.random.seed(42)

def sigmoid(z):
    return 1 / (1 + np.exp(-np.clip(z, -500, 500)))

class RegularizedMLP:
    """正則化付きMLP"""

    def __init__(self, dims, lr=0.1, l2_lambda=0, dropout_rate=0):
        self.Ws = []
        self.bs = []
        self.lr = lr
        self.l2_lambda = l2_lambda
        self.dropout_rate = dropout_rate
        self.training = True

        for i in range(len(dims) - 1):
            self.Ws.append(np.random.randn(dims[i+1], dims[i])
                           * np.sqrt(2.0 / dims[i]))
            self.bs.append(np.zeros((dims[i+1], 1)))

    def forward(self, X):
        self.hs = [X]
        self.zs = []
        self.masks = []
        h = X
        for i in range(len(self.Ws)):
            z = self.Ws[i] @ h + self.bs[i]
            self.zs.append(z)
            if i < len(self.Ws) - 1:
                h = np.maximum(0, z)
                # Dropout（学習時のみ）
                if self.training and self.dropout_rate > 0:
                    mask = (np.random.rand(*h.shape) > self.dropout_rate).astype(float)
                    h = h * mask / (1 - self.dropout_rate)
                    self.masks.append(mask)
                else:
                    self.masks.append(None)
            else:
                h = sigmoid(z)
            self.hs.append(h)
        return h

    def backward(self, t):
        m = t.shape[1]
        delta = self.hs[-1] - t
        for i in range(len(self.Ws) - 1, -1, -1):
            # L2正則化の勾配を追加
            dW = (1/m) * delta @ self.hs[i].T + self.l2_lambda * self.Ws[i]
            db = (1/m) * np.sum(delta, axis=1, keepdims=True)
            if i > 0:
                delta = (self.Ws[i].T @ delta) * (self.zs[i-1] > 0).astype(float)
                # Dropoutマスクの適用
                if self.masks[i-1] is not None:
                    delta = delta * self.masks[i-1] / (1 - self.dropout_rate)
            self.Ws[i] -= self.lr * dW
            self.bs[i] -= self.lr * db

    def compute_loss(self, y, t):
        eps = 1e-8
        loss = -np.mean(t * np.log(y + eps) + (1-t) * np.log(1-y + eps))
        # L2正則化項
        if self.l2_lambda > 0:
            l2_term = sum(np.sum(W**2) for W in self.Ws)
            loss += 0.5 * self.l2_lambda * l2_term
        return loss

# --- データ準備（わざと少量にして過学習を起こしやすくする） ---
X_data, y_data = make_moons(n_samples=200, noise=0.2, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(
    X_data, y_data, test_size=0.3, random_state=42)
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

X_tr = X_train.T
t_tr = y_train.reshape(1, -1)
X_te = X_test.T
t_te = y_test.reshape(1, -1)

# --- 各正則化手法の比較 ---
configs = [
    ("No Regularization", {"l2_lambda": 0, "dropout_rate": 0}),
    ("L2 Regularization\n($\\lambda=0.01$)", {"l2_lambda": 0.01, "dropout_rate": 0}),
    ("Dropout (p=0.3)", {"l2_lambda": 0, "dropout_rate": 0.3}),
    ("L2 + Dropout", {"l2_lambda": 0.005, "dropout_rate": 0.2}),
]

dims = [2, 128, 128, 128, 1]
epochs = 2000
colors = ["tab:red", "tab:blue", "tab:green", "tab:orange"]

fig, axes = plt.subplots(2, 2, figsize=(14, 10))

for ax_idx, (name, config) in enumerate(configs):
    ax = axes[ax_idx // 2, ax_idx % 2]
    np.random.seed(42)
    model = RegularizedMLP(dims, lr=0.3, **config)

    train_losses = []
    test_losses = []

    for epoch in range(epochs):
        model.training = True
        y_pred = model.forward(X_tr)
        train_loss = model.compute_loss(y_pred, t_tr)
        model.backward(t_tr)
        train_losses.append(train_loss)

        model.training = False
        y_test_pred = model.forward(X_te)
        test_loss_val = -np.mean(
            t_te * np.log(y_test_pred + 1e-8)
            + (1 - t_te) * np.log(1 - y_test_pred + 1e-8))
        test_losses.append(test_loss_val)

    ax.plot(train_losses, linewidth=2, color="tab:blue",
            alpha=0.8, label="Train")
    ax.plot(test_losses, linewidth=2, color="tab:orange",
            alpha=0.8, label="Test")
    ax.set_xlabel("Epoch", fontsize=11)
    ax.set_ylabel("Loss", fontsize=11)
    ax.set_title(name, fontsize=12)
    ax.legend(fontsize=10)
    ax.grid(True, alpha=0.3)
    ax.set_ylim(0, 1.5)

    # 最終精度を表示
    model.training = False
    y_final = (model.forward(X_te) > 0.5).astype(int)
    acc = np.mean(y_final == t_te)
    gap = train_losses[-1] - test_losses[-1]
    ax.text(0.98, 0.95, f"Test acc: {acc:.1%}\nGap: {abs(gap):.3f}",
            transform=ax.transAxes, fontsize=10, ha="right", va="top",
            bbox=dict(boxstyle="round,pad=0.3", facecolor="lightyellow"))

plt.tight_layout()
plt.savefig("regularization_comparison.png", dpi=150, bbox_inches="tight")
plt.show()