重み初期化の理論 — Xavier・He初期化の導出

import numpy as np
import matplotlib.pyplot as plt

np.random.seed(42)

# --- 異なる初期化での分散の伝播 ---
n_layers = 20
n_units = 256
n_samples = 1000

def forward_pass(init_scale, activation, n_layers, n_units, n_samples):
    """順伝播で各層の出力分散を記録"""
    x = np.random.randn(n_units, n_samples)
    variances = [np.var(x)]

    for l in range(n_layers):
        W = np.random.randn(n_units, n_units) * init_scale
        z = W @ x

        if activation == "linear":
            x = z
        elif activation == "sigmoid":
            x = 1 / (1 + np.exp(-np.clip(z, -500, 500)))
        elif activation == "tanh":
            x = np.tanh(z)
        elif activation == "relu":
            x = np.maximum(0, z)

        variances.append(np.var(x))

    return variances

fig, axes = plt.subplots(2, 2, figsize=(14, 10))

# (a) 線形ネットワーク: 異なるスケール
ax = axes[0, 0]
scales = [0.5/np.sqrt(n_units), 1.0/np.sqrt(n_units), 2.0/np.sqrt(n_units)]
labels = [r"$0.5/\sqrt{n}$", r"$1/\sqrt{n}$ (Xavier)", r"$2/\sqrt{n}$"]
colors = ["tab:blue", "tab:green", "tab:red"]

for scale, label, color in zip(scales, labels, colors):
    variances = forward_pass(scale, "linear", n_layers, n_units, n_samples)
    ax.semilogy(variances, "o-", linewidth=2, markersize=3,
                color=color, label=label)

ax.set_xlabel("Layer", fontsize=12)
ax.set_ylabel("Variance (log scale)", fontsize=12)
ax.set_title("Linear Network: Effect of Init Scale", fontsize=13)
ax.legend(fontsize=10)
ax.grid(True, alpha=0.3)

# (b) tanh + Xavier vs 不適切な初期化
ax = axes[0, 1]
configs = [
    (1.0/np.sqrt(n_units), "tanh", r"Xavier ($1/\sqrt{n}$)"),
    (0.01, "tanh", "Too small (0.01)"),
    (1.0, "tanh", "Too large (1.0)"),
]
colors = ["tab:green", "tab:blue", "tab:red"]

for (scale, act, label), color in zip(configs, colors):
    variances = forward_pass(scale, act, n_layers, n_units, n_samples)
    ax.semilogy(variances, "o-", linewidth=2, markersize=3,
                color=color, label=label)

ax.set_xlabel("Layer", fontsize=12)
ax.set_ylabel("Variance (log scale)", fontsize=12)
ax.set_title("tanh Network: Xavier vs Others", fontsize=13)
ax.legend(fontsize=10)
ax.grid(True, alpha=0.3)

# (c) ReLU + He vs Xavier
ax = axes[1, 0]
configs = [
    (np.sqrt(2.0/n_units), "relu", r"He ($\sqrt{2/n}$)"),
    (np.sqrt(1.0/n_units), "relu", r"Xavier ($\sqrt{1/n}$)"),
    (0.01, "relu", "Too small (0.01)"),
]
colors = ["tab:green", "tab:orange", "tab:blue"]

for (scale, act, label), color in zip(configs, colors):
    variances = forward_pass(scale, act, n_layers, n_units, n_samples)
    ax.semilogy(variances, "o-", linewidth=2, markersize=3,
                color=color, label=label)

ax.set_xlabel("Layer", fontsize=12)
ax.set_ylabel("Variance (log scale)", fontsize=12)
ax.set_title("ReLU Network: He vs Xavier", fontsize=13)
ax.legend(fontsize=10)
ax.grid(True, alpha=0.3)

# (d) 各層の出力のヒストグラム（ReLU + He初期化）
ax = axes[1, 1]
x = np.random.randn(n_units, n_samples)
layer_indices = [0, 5, 10, 19]
colors_hist = ["tab:blue", "tab:orange", "tab:green", "tab:red"]

for l in range(n_layers):
    W = np.random.randn(n_units, n_units) * np.sqrt(2.0 / n_units)
    z = W @ x
    x = np.maximum(0, z)

    if l in layer_indices:
        # 正の値のみの分布を表示
        positive_vals = x[x > 0].flatten()
        ax.hist(positive_vals, bins=50, density=True, alpha=0.5,
                color=colors_hist[layer_indices.index(l)],
                label=f"Layer {l+1}")

ax.set_xlabel("Activation value", fontsize=12)
ax.set_ylabel("Density", fontsize=12)
ax.set_title("ReLU Activation Distribution (He init)", fontsize=13)
ax.legend(fontsize=10)
ax.grid(True, alpha=0.3)
ax.set_xlim(0, 5)

plt.tight_layout()
plt.savefig("weight_init_variance.png", dpi=150, bbox_inches="tight")
plt.show()

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_moons
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

np.random.seed(42)

def sigmoid(z):
    return 1 / (1 + np.exp(-np.clip(z, -500, 500)))

class DeepMLP:
    """可変深さのMLP"""

    def __init__(self, dims, init_method="he", lr=0.01):
        self.Ws = []
        self.bs = []
        self.lr = lr

        for i in range(len(dims) - 1):
            n_in, n_out = dims[i], dims[i+1]
            if init_method == "he":
                scale = np.sqrt(2.0 / n_in)
            elif init_method == "xavier":
                scale = np.sqrt(2.0 / (n_in + n_out))
            elif init_method == "small":
                scale = 0.01
            elif init_method == "large":
                scale = 1.0
            else:
                scale = np.sqrt(1.0 / n_in)

            self.Ws.append(np.random.randn(n_out, n_in) * scale)
            self.bs.append(np.zeros((n_out, 1)))

    def forward(self, X):
        self.hs = [X]
        self.zs = []
        h = X
        for i in range(len(self.Ws)):
            z = self.Ws[i] @ h + self.bs[i]
            self.zs.append(z)
            if i < len(self.Ws) - 1:
                h = np.maximum(0, z)  # ReLU
            else:
                h = sigmoid(z)  # 出力層
            self.hs.append(h)
        return h

    def backward(self, t):
        m = t.shape[1]
        delta = self.hs[-1] - t

        for i in range(len(self.Ws) - 1, -1, -1):
            dW = (1/m) * delta @ self.hs[i].T
            db = (1/m) * np.sum(delta, axis=1, keepdims=True)

            if i > 0:
                delta = (self.Ws[i].T @ delta) * (self.zs[i-1] > 0).astype(float)

            self.Ws[i] -= self.lr * dW
            self.bs[i] -= self.lr * db

    def compute_loss(self, y, t):
        eps = 1e-8
        return -np.mean(t * np.log(y + eps) + (1-t) * np.log(1-y + eps))

# データ準備
X_data, y_data = make_moons(n_samples=1000, noise=0.2, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(
    X_data, y_data, test_size=0.2, random_state=42)
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

X_tr = X_train.T
t_tr = y_train.reshape(1, -1)
X_te = X_test.T
t_te = y_test.reshape(1, -1)

# --- 深いネットワーク（8隠れ層）での比較 ---
dims = [2, 64, 64, 64, 64, 64, 64, 64, 64, 1]
init_methods = ["he", "xavier", "small", "large"]
init_labels = ["He init", "Xavier init", "Small (0.01)", "Large (1.0)"]
colors = ["tab:green", "tab:orange", "tab:blue", "tab:red"]
epochs = 3000

fig, axes = plt.subplots(1, 2, figsize=(14, 5.5))

# (a) 損失曲線
ax = axes[0]
results = {}

for method, label, color in zip(init_methods, init_labels, colors):
    np.random.seed(42)
    model = DeepMLP(dims, init_method=method, lr=0.1)
    losses = []
    for epoch in range(epochs):
        y_pred = model.forward(X_tr)
        loss = model.compute_loss(y_pred, t_tr)
        model.backward(t_tr)
        losses.append(loss)

    ax.plot(losses, linewidth=2, color=color, label=label, alpha=0.8)

    y_test_pred = (model.forward(X_te) > 0.5).astype(int)
    acc = np.mean(y_test_pred == t_te)
    results[label] = acc

ax.set_xlabel("Epoch", fontsize=12)
ax.set_ylabel("Loss", fontsize=12)
ax.set_title("Training Loss (8 hidden layers, ReLU)", fontsize=13)
ax.set_yscale("log")
ax.legend(fontsize=10)
ax.grid(True, alpha=0.3)
ax.set_ylim(1e-3, 2)

# (b) テスト精度
ax = axes[1]
bars = ax.bar(results.keys(), results.values(), color=colors, alpha=0.8,
              edgecolor="gray", linewidth=1)
for bar, acc in zip(bars, results.values()):
    ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.01,
            f"{acc:.1%}", ha="center", fontsize=11, fontweight="bold")
ax.set_ylabel("Test Accuracy", fontsize=12)
ax.set_title("Test Accuracy by Init Method", fontsize=13)
ax.set_ylim(0, 1.1)
ax.grid(True, alpha=0.3, axis="y")
plt.xticks(rotation=15)

plt.tight_layout()
plt.savefig("weight_init_learning.png", dpi=150, bbox_inches="tight")
plt.show()

import numpy as np

# PyTorchでの初期化例（概念的なコード）
# import torch
# import torch.nn as nn
#
# layer = nn.Linear(256, 128)
#
# # He初期化（ReLU用）
# nn.init.kaiming_normal_(layer.weight, mode='fan_in', nonlinearity='relu')
#
# # Xavier初期化（tanh用）
# nn.init.xavier_normal_(layer.weight)
#
# # バイアスはゼロ初期化
# nn.init.zeros_(layer.bias)

# NumPyでの同等実装
n_in, n_out = 256, 128

# He初期化（fan_in モード）
W_he = np.random.randn(n_out, n_in) * np.sqrt(2.0 / n_in)

# Xavier初期化
W_xavier = np.random.randn(n_out, n_in) * np.sqrt(2.0 / (n_in + n_out))

print(f"He init:     Var(W) = {np.var(W_he):.6f}, "
      f"理論値 = {2.0/n_in:.6f}")
print(f"Xavier init: Var(W) = {np.var(W_xavier):.6f}, "
      f"理論値 = {2.0/(n_in+n_out):.6f}")