誤差逆伝播法の理論と実装 — 計算グラフで理解する

import numpy as np

np.random.seed(42)

def sigmoid(z):
    return 1 / (1 + np.exp(-np.clip(z, -500, 500)))

def sigmoid_derivative(a):
    return a * (1 - a)

def relu(z):
    return np.maximum(0, z)

def relu_derivative(z):
    return (z > 0).astype(float)

def compute_loss(y_pred, t):
    """二値交差エントロピー損失"""
    eps = 1e-8
    return -np.mean(t * np.log(y_pred + eps)
                    + (1 - t) * np.log(1 - y_pred + eps))

# ネットワークのパラメータ
input_dim, hidden_dim, output_dim = 3, 4, 1
W1 = np.random.randn(hidden_dim, input_dim) * 0.5
b1 = np.zeros((hidden_dim, 1))
W2 = np.random.randn(output_dim, hidden_dim) * 0.5
b2 = np.zeros((output_dim, 1))

# テストデータ
X = np.random.randn(input_dim, 5)
t = np.array([[1, 0, 1, 0, 1]])

# --- 解析勾配（逆伝播） ---
# 順伝播
z1 = W1 @ X + b1
h1 = sigmoid(z1)
z2 = W2 @ h1 + b2
h2 = sigmoid(z2)

# 逆伝播
m = X.shape[1]
delta2 = h2 - t
dW2_analytical = (1/m) * delta2 @ h1.T
db2_analytical = (1/m) * np.sum(delta2, axis=1, keepdims=True)
delta1 = (W2.T @ delta2) * sigmoid_derivative(h1)
dW1_analytical = (1/m) * delta1 @ X.T
db1_analytical = (1/m) * np.sum(delta1, axis=1, keepdims=True)

# --- 数値勾配 ---
epsilon = 1e-5

def forward_loss(W1, b1, W2, b2, X, t):
    z1 = W1 @ X + b1
    h1 = sigmoid(z1)
    z2 = W2 @ h1 + b2
    h2 = sigmoid(z2)
    return compute_loss(h2, t)

def numerical_gradient(param, param_name, W1, b1, W2, b2, X, t):
    """中心差分による数値勾配"""
    grad = np.zeros_like(param)
    it = np.nditer(param, flags=['multi_index'])
    while not it.finished:
        idx = it.multi_index
        old_val = param[idx]

        param[idx] = old_val + epsilon
        loss_plus = forward_loss(W1, b1, W2, b2, X, t)

        param[idx] = old_val - epsilon
        loss_minus = forward_loss(W1, b1, W2, b2, X, t)

        grad[idx] = (loss_plus - loss_minus) / (2 * epsilon)
        param[idx] = old_val
        it.iternext()
    return grad

# 各パラメータの数値勾配を計算
dW1_numerical = numerical_gradient(W1, "W1", W1, b1, W2, b2, X, t)
db1_numerical = numerical_gradient(b1, "b1", W1, b1, W2, b2, X, t)
dW2_numerical = numerical_gradient(W2, "W2", W1, b1, W2, b2, X, t)
db2_numerical = numerical_gradient(b2, "b2", W1, b1, W2, b2, X, t)

# 相対誤差の計算
def relative_error(analytical, numerical):
    eps = 1e-8
    return np.max(np.abs(analytical - numerical)
                  / (np.maximum(np.abs(analytical),
                                np.abs(numerical)) + eps))

print("グラディエントチェック結果:")
print(f"  dW1: 相対誤差 = {relative_error(dW1_analytical, dW1_numerical):.2e}")
print(f"  db1: 相対誤差 = {relative_error(db1_analytical, db1_numerical):.2e}")
print(f"  dW2: 相対誤差 = {relative_error(dW2_analytical, dW2_numerical):.2e}")
print(f"  db2: 相対誤差 = {relative_error(db2_analytical, db2_numerical):.2e}")

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_circles
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

np.random.seed(42)

class Linear:
    """全結合層（アフィン変換）"""

    def __init__(self, in_features, out_features):
        # He初期化
        self.W = np.random.randn(out_features, in_features) * np.sqrt(2.0 / in_features)
        self.b = np.zeros((out_features, 1))
        self.dW = None
        self.db = None

    def forward(self, X):
        """順伝播: z = Wx + b"""
        self.X = X  # 逆伝播で使用
        return self.W @ X + self.b

    def backward(self, dout):
        """逆伝播: 上流の勾配doutから各勾配を計算"""
        m = self.X.shape[1]
        self.dW = (1/m) * dout @ self.X.T
        self.db = (1/m) * np.sum(dout, axis=1, keepdims=True)
        return self.W.T @ dout  # 前の層への勾配


class ReLU:
    """ReLU活性化関数"""

    def forward(self, Z):
        self.Z = Z
        return np.maximum(0, Z)

    def backward(self, dout):
        return dout * (self.Z > 0).astype(float)


class Sigmoid:
    """シグモイド活性化関数"""

    def forward(self, Z):
        self.out = 1 / (1 + np.exp(-np.clip(Z, -500, 500)))
        return self.out

    def backward(self, dout):
        return dout * self.out * (1 - self.out)


class BCELoss:
    """二値交差エントロピー損失"""

    def forward(self, y_pred, t):
        self.y_pred = y_pred
        self.t = t
        eps = 1e-8
        return -np.mean(t * np.log(y_pred + eps)
                        + (1 - t) * np.log(1 - y_pred + eps))

    def backward(self):
        # シグモイド + BCEの場合の簡約形
        return self.y_pred - self.t

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_circles
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

np.random.seed(42)

# --- 層クラスの定義（前のコードブロックと同じ） ---
class Linear:
    def __init__(self, in_features, out_features):
        self.W = np.random.randn(out_features, in_features) * np.sqrt(2.0 / in_features)
        self.b = np.zeros((out_features, 1))
        self.dW = None
        self.db = None

    def forward(self, X):
        self.X = X
        return self.W @ X + self.b

    def backward(self, dout):
        m = self.X.shape[1]
        self.dW = (1/m) * dout @ self.X.T
        self.db = (1/m) * np.sum(dout, axis=1, keepdims=True)
        return self.W.T @ dout

class ReLU:
    def forward(self, Z):
        self.Z = Z
        return np.maximum(0, Z)

    def backward(self, dout):
        return dout * (self.Z > 0).astype(float)

class Sigmoid:
    def forward(self, Z):
        self.out = 1 / (1 + np.exp(-np.clip(Z, -500, 500)))
        return self.out

    def backward(self, dout):
        return dout * self.out * (1 - self.out)

class BCELoss:
    def forward(self, y_pred, t):
        self.y_pred = y_pred
        self.t = t
        eps = 1e-8
        return -np.mean(t * np.log(y_pred + eps)
                        + (1 - t) * np.log(1 - y_pred + eps))

    def backward(self):
        return self.y_pred - self.t

class MLP:
    """多層パーセプトロン"""

    def __init__(self, layer_dims, lr=0.1):
        self.layers = []
        self.lr = lr
        for i in range(len(layer_dims) - 1):
            self.layers.append(Linear(layer_dims[i], layer_dims[i+1]))
            if i < len(layer_dims) - 2:
                self.layers.append(ReLU())
            else:
                self.layers.append(Sigmoid())
        self.loss_fn = BCELoss()

    def forward(self, X):
        out = X
        for layer in self.layers:
            out = layer.forward(out)
        return out

    def backward(self):
        dout = self.loss_fn.backward()
        for layer in reversed(self.layers):
            dout = layer.backward(dout)

    def update(self):
        for layer in self.layers:
            if isinstance(layer, Linear):
                layer.W -= self.lr * layer.dW
                layer.b -= self.lr * layer.db

    def train_step(self, X, t):
        y_pred = self.forward(X)
        loss = self.loss_fn.forward(y_pred, t)
        self.backward()
        self.update()
        return loss

# --- 同心円データセット ---
X_data, y_data = make_circles(n_samples=600, noise=0.1,
                               factor=0.3, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(
    X_data, y_data, test_size=0.2, random_state=42)

scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

X_tr = X_train.T
t_tr = y_train.reshape(1, -1)
X_te = X_test.T
t_te = y_test.reshape(1, -1)

# --- 異なる深さのネットワークで学習 ---
configs = [
    ([2, 16, 1], "1 hidden layer (16 units)"),
    ([2, 16, 16, 1], "2 hidden layers (16-16)"),
    ([2, 16, 16, 16, 1], "3 hidden layers (16-16-16)"),
]

fig, axes = plt.subplots(2, 3, figsize=(16, 10))

for col, (dims, title) in enumerate(configs):
    model = MLP(dims, lr=0.5)
    losses = []

    for epoch in range(3000):
        loss = model.train_step(X_tr, t_tr)
        losses.append(loss)

    # 損失の推移
    ax = axes[0, col]
    ax.plot(losses, linewidth=1.5, color="steelblue")
    ax.set_xlabel("Epoch", fontsize=11)
    ax.set_ylabel("Loss", fontsize=11)
    ax.set_title(title, fontsize=12)
    ax.set_yscale("log")
    ax.grid(True, alpha=0.3)

    # 判定境界
    ax = axes[1, col]
    xx, yy = np.meshgrid(np.linspace(-3, 3, 200),
                          np.linspace(-3, 3, 200))
    grid = np.c_[xx.ravel(), yy.ravel()].T
    zz = model.forward(grid).reshape(xx.shape)

    ax.contourf(xx, yy, zz, levels=50, cmap="RdBu_r", alpha=0.7)
    ax.contour(xx, yy, zz, levels=[0.5], colors="black", linewidths=2)

    for label, marker, color in [(0, "o", "red"), (1, "^", "blue")]:
        mask = y_train == label
        ax.scatter(X_train[mask, 0], X_train[mask, 1], s=15, c=color,
                   marker=marker, alpha=0.5, edgecolors="none")

    y_pred_test = (model.forward(X_te) > 0.5).astype(int)
    acc = np.mean(y_pred_test == t_te)
    ax.set_title(f"Test accuracy = {acc:.1%}", fontsize=12)
    ax.set_xlim(-3, 3)
    ax.set_ylim(-3, 3)
    ax.set_aspect("equal")

plt.tight_layout()
plt.savefig("backprop_depth_comparison.png", dpi=150, bbox_inches="tight")
plt.show()

import numpy as np
import matplotlib.pyplot as plt

np.random.seed(42)

# --- 勾配消失の可視化: シグモイド vs ReLU ---
depth = 20
hidden_dim = 50
input_dim = 50

fig, axes = plt.subplots(1, 2, figsize=(14, 5.5))

for ax, (act_name, act_fn, act_deriv, init_scale) in zip(
    axes,
    [("Sigmoid", lambda z: 1/(1+np.exp(-np.clip(z, -500, 500))),
      lambda a: a * (1 - a), 1.0),
     ("ReLU", lambda z: np.maximum(0, z),
      lambda z: (z > 0).astype(float), np.sqrt(2.0 / hidden_dim))]
):
    grad_norms = []

    for trial in range(50):
        # ネットワークの構築
        Ws = []
        for l in range(depth):
            d_in = input_dim if l == 0 else hidden_dim
            Ws.append(np.random.randn(hidden_dim, d_in) * init_scale)

        # 順伝播
        x = np.random.randn(input_dim, 1)
        hs = [x]
        zs = []
        for l in range(depth):
            z = Ws[l] @ hs[-1]
            zs.append(z)
            h = act_fn(z)
            hs.append(h)

        # 逆伝播（出力を仮にhsの最終層として、delta = 1）
        delta = np.ones_like(hs[-1])
        norms = []
        for l in range(depth - 1, -1, -1):
            if act_name == "Sigmoid":
                delta = delta * act_deriv(hs[l+1])
            else:
                delta = delta * act_deriv(zs[l])
            delta = Ws[l].T @ delta
            norms.append(np.linalg.norm(delta))

        grad_norms.append(norms[::-1])

    grad_norms = np.array(grad_norms)
    mean_norms = np.mean(grad_norms, axis=0)
    std_norms = np.std(grad_norms, axis=0)

    layers = np.arange(1, depth + 1)
    ax.semilogy(layers, mean_norms, "o-", linewidth=2, markersize=4,
                color="steelblue")
    ax.fill_between(layers,
                    np.maximum(mean_norms - std_norms, 1e-20),
                    mean_norms + std_norms,
                    alpha=0.2, color="steelblue")
    ax.set_xlabel("Layer (from input)", fontsize=12)
    ax.set_ylabel("Gradient norm (log scale)", fontsize=12)
    ax.set_title(f"Gradient Flow with {act_name}", fontsize=13)
    ax.grid(True, alpha=0.3)

plt.tight_layout()
plt.savefig("gradient_flow.png", dpi=150, bbox_inches="tight")
plt.show()

import numpy as np

# PyTorchでの自動微分の例（概念的なコード）
# import torch
# import torch.nn as nn
#
# x = torch.randn(2, requires_grad=True)
# y = x[0]**2 + 3*x[1]
# y.backward()    # 自動微分で勾配を計算
# print(x.grad)   # [2*x[0], 3]

# NumPyで同等の手動計算
x = np.array([2.0, 1.0])
y = x[0]**2 + 3*x[1]

# 解析的な勾配
dy_dx0 = 2 * x[0]  # = 4.0
dy_dx1 = 3.0

print("手動計算による勾配:")
print(f"  dy/dx0 = {dy_dx0}")
print(f"  dy/dx1 = {dy_dx1}")
print(f"  y = {y}")