交差エントロピーを完全理解 — 情報理論からの導出と実装

import numpy as np
import matplotlib.pyplot as plt

# 二値分布のエントロピー
p = np.linspace(0.001, 0.999, 500)
entropy = -p * np.log(p) - (1 - p) * np.log(1 - p)

fig, axes = plt.subplots(1, 2, figsize=(14, 5.5))

# (a) 二値エントロピー関数
ax = axes[0]
ax.plot(p, entropy, "b-", linewidth=2.5)
ax.axvline(0.5, color="red", linestyle="--", linewidth=1.5, alpha=0.7)
ax.set_xlabel("$p$ (probability of class 1)", fontsize=13)
ax.set_ylabel("$H(p)$", fontsize=13)
ax.set_title("Binary Entropy Function", fontsize=14)
ax.grid(True, alpha=0.3)
ax.annotate("Maximum at p = 0.5\n(most uncertain)", xy=(0.5, np.log(2)),
            xytext=(0.7, 0.5), fontsize=10,
            arrowprops=dict(arrowstyle="->", color="red"), color="red")

# (b) K値分布の最大エントロピー
ax = axes[1]
K_values = np.arange(2, 21)
max_entropy = np.log(K_values)

ax.bar(K_values, max_entropy, color="steelblue", alpha=0.8, edgecolor="gray")
ax.set_xlabel("Number of classes $K$", fontsize=13)
ax.set_ylabel("Maximum entropy $\\ln K$", fontsize=13)
ax.set_title("Maximum Entropy for K Classes", fontsize=14)
ax.grid(True, alpha=0.3, axis="y")

plt.tight_layout()
plt.savefig("entropy_basics.png", dpi=150, bbox_inches="tight")
plt.show()

import numpy as np
import matplotlib.pyplot as plt

# ソフトマックス + 交差エントロピー vs ソフトマックス + MSE の勾配比較
z = np.linspace(-6, 6, 500)

# 正解ラベル y=1 の場合
# ソフトマックスの出力（二値の場合はシグモイド）
p = 1.0 / (1.0 + np.exp(-z))

# 交差エントロピーの勾配: dp/dz * dL/dp = p(1-p) * (-1/p) = -(1-p) → dL/dz = p - 1
grad_ce = p - 1  # y=1の場合

# MSEの勾配: dL/dz = -2(1 - p) * p * (1 - p) = -2p(1-p)^2
grad_mse = -2 * p * (1 - p) ** 2

fig, axes = plt.subplots(1, 2, figsize=(14, 5.5))

# (a) 損失関数
ax = axes[0]
loss_ce = -np.log(p + 1e-15)
loss_mse = (1 - p) ** 2

ax.plot(z, loss_ce, "b-", linewidth=2.5, label="Cross-Entropy: $-\\ln p$")
ax.plot(z, loss_mse, "r-", linewidth=2.5, label="MSE: $(1-p)^2$")
ax.set_xlabel("Logit $z$", fontsize=13)
ax.set_ylabel("Loss (for $y=1$)", fontsize=13)
ax.set_title("Loss Functions vs Logit", fontsize=14)
ax.legend(fontsize=11)
ax.grid(True, alpha=0.3)
ax.set_ylim(-0.5, 8)

# (b) 勾配
ax = axes[1]
ax.plot(z, np.abs(grad_ce), "b-", linewidth=2.5, label="|CE gradient| = $1-p$")
ax.plot(z, np.abs(grad_mse), "r-", linewidth=2.5, label="|MSE gradient| = $2p(1-p)^2$")
ax.set_xlabel("Logit $z$", fontsize=13)
ax.set_ylabel("|Gradient| (for $y=1$)", fontsize=13)
ax.set_title("Gradient Magnitude vs Logit", fontsize=14)
ax.legend(fontsize=11)
ax.grid(True, alpha=0.3)

ax.annotate("MSE gradient vanishes\nwhen $z \\ll 0$ (wrong prediction)",
            xy=(-5, 0.01), xytext=(-3, 0.5), fontsize=10,
            arrowprops=dict(arrowstyle="->", color="red"), color="red")

plt.tight_layout()
plt.savefig("ce_vs_mse_gradient.png", dpi=150, bbox_inches="tight")
plt.show()

import numpy as np

def softmax_stable(z):
    """数値安定なソフトマックス"""
    c = np.max(z, axis=-1, keepdims=True)
    exp_z = np.exp(z - c)
    return exp_z / np.sum(exp_z, axis=-1, keepdims=True)

def cross_entropy_from_probs(y_onehot, probs):
    """確率からの交差エントロピー（素朴な実装）"""
    # log(0) を防ぐためにクリップ
    probs_clipped = np.clip(probs, 1e-15, 1 - 1e-15)
    return -np.sum(y_onehot * np.log(probs_clipped), axis=-1)

def cross_entropy_from_logits(y_onehot, logits):
    """ロジットからの交差エントロピー（数値安定版）"""
    # log-sum-exp トリック
    c = np.max(logits, axis=-1, keepdims=True)
    log_sum_exp = c.squeeze() + np.log(np.sum(np.exp(logits - c), axis=-1))
    correct_logit = np.sum(y_onehot * logits, axis=-1)
    return -correct_logit + log_sum_exp

# テスト
np.random.seed(42)
K = 5
n = 10
logits = np.random.randn(n, K) * 10  # 大きなロジット

# one-hotラベルの生成
y_true = np.random.randint(0, K, n)
y_onehot = np.eye(K)[y_true]

# 2つの方法で計算
probs = softmax_stable(logits)
loss_from_probs = cross_entropy_from_probs(y_onehot, probs)
loss_from_logits = cross_entropy_from_logits(y_onehot, logits)

print("Cross-entropy from probabilities:", loss_from_probs[:5])
print("Cross-entropy from logits:       ", loss_from_logits[:5])
print("Max difference:", np.max(np.abs(loss_from_probs - loss_from_logits)))

# 極端なロジットでの比較
extreme_logits = np.array([[1000, 0, 0, 0, 0]])
y_extreme = np.array([[1, 0, 0, 0, 0]])

print("\n--- Extreme logits [1000, 0, 0, 0, 0] ---")
probs_extreme = softmax_stable(extreme_logits)
print("Softmax output:", probs_extreme)
print("CE from logits:", cross_entropy_from_logits(y_extreme, extreme_logits))
print("CE from probs: ", cross_entropy_from_probs(y_extreme, probs_extreme))

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Polygon

def ternary_to_cartesian(p):
    """3クラスの確率を2次元座標に変換"""
    x = p[1] + 0.5 * p[2]
    y = np.sqrt(3) / 2 * p[2]
    return x, y

# 確率単体上の交差エントロピー
fig, axes = plt.subplots(1, 3, figsize=(16, 5.5))

# 真の分布を3パターン
true_dists = [
    np.array([1.0, 0.0, 0.0]),
    np.array([0.5, 0.3, 0.2]),
    np.array([1/3, 1/3, 1/3]),
]
titles = [
    "True dist: (1, 0, 0)\nOne-hot label",
    "True dist: (0.5, 0.3, 0.2)\nSoft label",
    "True dist: (1/3, 1/3, 1/3)\nUniform"
]

for ax, p_true, title in zip(axes, true_dists, titles):
    # 確率単体の三角形を描画
    triangle = Polygon(
        [ternary_to_cartesian([1,0,0]),
         ternary_to_cartesian([0,1,0]),
         ternary_to_cartesian([0,0,1])],
        fill=False, edgecolor="black", linewidth=2
    )
    ax.add_patch(triangle)

    # グリッド上で交差エントロピーを計算
    resolution = 200
    ce_values = np.full((resolution, resolution), np.nan)

    for i in range(resolution):
        for j in range(resolution):
            p1 = i / (resolution - 1)
            p2 = j / (resolution - 1)
            p3 = 1 - p1 - p2

            if p3 >= 0.001 and p1 >= 0.001 and p2 >= 0.001:
                q = np.array([p1, p2, p3])
                ce = -np.sum(p_true * np.log(q + 1e-15))
                x, y = ternary_to_cartesian(q)
                # プロット座標に変換
                xi = int(x * (resolution - 1))
                yi = int(y * (resolution - 1))
                if 0 <= xi < resolution and 0 <= yi < resolution:
                    ce_values[yi, xi] = ce

    # 等高線プロット
    x_range = np.linspace(0, 1, resolution)
    y_range = np.linspace(0, np.sqrt(3)/2, resolution)
    X, Y = np.meshgrid(x_range, y_range)

    levels = np.linspace(0, 4, 20)
    cs = ax.contourf(X, Y, ce_values, levels=levels, cmap="RdYlBu_r", alpha=0.8)
    ax.contour(X, Y, ce_values, levels=levels, colors="gray", linewidths=0.5, alpha=0.5)

    # 真の分布の位置をマーク
    tx, ty = ternary_to_cartesian(p_true)
    ax.scatter(tx, ty, color="red", s=100, zorder=5, edgecolors="black", linewidth=1.5)

    # 頂点ラベル
    corners = [[1,0,0], [0,1,0], [0,0,1]]
    labels = ["(1,0,0)", "(0,1,0)", "(0,0,1)"]
    offsets = [(-0.08, -0.05), (0.02, -0.05), (0.02, 0.03)]
    for corner, label, offset in zip(corners, labels, offsets):
        cx, cy = ternary_to_cartesian(corner)
        ax.annotate(label, xy=(cx, cy), xytext=(cx+offset[0], cy+offset[1]),
                   fontsize=8, ha="center")

    ax.set_title(title, fontsize=12)
    ax.set_xlim(-0.1, 1.1)
    ax.set_ylim(-0.1, 1.0)
    ax.set_aspect("equal")
    ax.axis("off")

plt.tight_layout()
plt.savefig("cross_entropy_simplex.png", dpi=150, bbox_inches="tight")
plt.show()

import numpy as np
import matplotlib.pyplot as plt

# ラベルスムージングの効果
K = 5
n_epochs = 200
lr = 0.5

def softmax(z):
    c = np.max(z)
    exp_z = np.exp(z - c)
    return exp_z / np.sum(exp_z)

def train_with_smoothing(epsilon, n_epochs, lr):
    """ラベルスムージング付きで1サンプルの学習をシミュレート"""
    z = np.zeros(K)  # ロジットの初期値
    true_class = 0

    # スムージングされたラベル
    y_smooth = np.full(K, epsilon / K)
    y_smooth[true_class] = 1 - epsilon + epsilon / K

    logit_history = []
    prob_history = []

    for epoch in range(n_epochs):
        p = softmax(z)
        logit_history.append(z.copy())
        prob_history.append(p.copy())

        # 勾配: p - y
        grad = p - y_smooth
        z -= lr * grad

    return np.array(logit_history), np.array(prob_history)

fig, axes = plt.subplots(1, 2, figsize=(14, 5.5))

# (a) 正解クラスの確率の推移
ax = axes[0]
for eps, color, label in [(0.0, "blue", "$\\varepsilon=0$ (no smoothing)"),
                           (0.1, "green", "$\\varepsilon=0.1$"),
                           (0.3, "red", "$\\varepsilon=0.3$")]:
    _, probs = train_with_smoothing(eps, n_epochs, lr)
    ax.plot(probs[:, 0], color=color, linewidth=2, label=label)

ax.set_xlabel("Epoch", fontsize=12)
ax.set_ylabel("$\\hat{p}_{\\text{correct}}$", fontsize=12)
ax.set_title("Predicted Probability of Correct Class", fontsize=13)
ax.legend(fontsize=10)
ax.grid(True, alpha=0.3)

# (b) ロジットの推移
ax = axes[1]
for eps, color, label in [(0.0, "blue", "$\\varepsilon=0$"),
                           (0.1, "green", "$\\varepsilon=0.1$"),
                           (0.3, "red", "$\\varepsilon=0.3$")]:
    logits, _ = train_with_smoothing(eps, n_epochs, lr)
    # 正解と不正解クラスのロジット差
    ax.plot(logits[:, 0] - np.mean(logits[:, 1:], axis=1),
            color=color, linewidth=2, label=label)

ax.set_xlabel("Epoch", fontsize=12)
ax.set_ylabel("Logit gap (correct - avg others)", fontsize=12)
ax.set_title("Logit Difference: Correct vs Others", fontsize=13)
ax.legend(fontsize=10)
ax.grid(True, alpha=0.3)

plt.tight_layout()
plt.savefig("label_smoothing.png", dpi=150, bbox_inches="tight")
plt.show()

import numpy as np
import matplotlib.pyplot as plt

p_t = np.linspace(0.001, 0.999, 500)

fig, ax = plt.subplots(figsize=(10, 6))

gammas = [0, 0.5, 1, 2, 5]
colors = plt.cm.viridis(np.linspace(0, 0.9, len(gammas)))

for gamma, color in zip(gammas, colors):
    focal = -(1 - p_t) ** gamma * np.log(p_t)
    label = f"$\\gamma={gamma}$" + (" (CE)" if gamma == 0 else "")
    ax.plot(p_t, focal, color=color, linewidth=2.5, label=label)

ax.set_xlabel("Predicted probability of correct class $\\hat{p}_t$", fontsize=13)
ax.set_ylabel("Focal Loss", fontsize=13)
ax.set_title("Focal Loss for Different $\\gamma$ Values", fontsize=14)
ax.legend(fontsize=11)
ax.grid(True, alpha=0.3)
ax.set_ylim(-0.5, 5)

ax.annotate("Easy examples\n(loss suppressed)", xy=(0.85, 0.1), fontsize=10,
            color="darkgreen")
ax.annotate("Hard examples\n(loss preserved)", xy=(0.1, 3.5), fontsize=10,
            color="darkred")

plt.tight_layout()
plt.savefig("focal_loss.png", dpi=150, bbox_inches="tight")
plt.show()