DeBERTaのDisentangled Attention — コンテンツと位置を分離した注意機構

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
import matplotlib.pyplot as plt


def build_relative_position(seq_len, max_relative_distance):
    """相対位置インデックス行列を生成する

    Args:
        seq_len: 系列長
        max_relative_distance: 最大相対距離 k

    Returns:
        relative_pos: (seq_len, seq_len) の相対位置インデックス行列
                      値は [0, 2*k] の範囲（-k を 0 にシフト済み）
    """
    # 位置インデックス [0, 1, ..., seq_len-1]
    positions = torch.arange(seq_len, dtype=torch.long)

    # 相対位置行列: positions[i] - positions[j]
    # (seq_len, 1) - (1, seq_len) -> (seq_len, seq_len)
    relative_pos = positions.unsqueeze(1) - positions.unsqueeze(0)

    # クリッピング: [-k, k] に制限
    relative_pos = torch.clamp(relative_pos, -max_relative_distance, max_relative_distance)

    # 非負にシフト: [-k, k] -> [0, 2k]
    relative_pos = relative_pos + max_relative_distance

    return relative_pos

# 相対位置行列の可視化
seq_len = 8
k = 4  # 小さい k で構造を見やすくする
rel_pos = build_relative_position(seq_len, k)

fig, axes = plt.subplots(1, 2, figsize=(14, 5))

# 生のインデックス（シフト前）
raw_rel = rel_pos - k  # シフトを戻す
im1 = axes[0].imshow(raw_rel.numpy(), cmap='RdBu_r', vmin=-k, vmax=k)
axes[0].set_title('Relative Position (raw)', fontsize=13, fontweight='bold')
axes[0].set_xlabel('Key position j')
axes[0].set_ylabel('Query position i')
for i in range(seq_len):
    for j in range(seq_len):
        axes[0].text(j, i, f'{raw_rel[i, j].item():+d}', ha='center', va='center', fontsize=9)
plt.colorbar(im1, ax=axes[0])

# クリッピング後のインデックス
im2 = axes[1].imshow(rel_pos.numpy(), cmap='viridis', vmin=0, vmax=2*k)
axes[1].set_title(f'Relative Position Index (clipped, k={k})', fontsize=13, fontweight='bold')
axes[1].set_xlabel('Key position j')
axes[1].set_ylabel('Query position i')
for i in range(seq_len):
    for j in range(seq_len):
        axes[1].text(j, i, f'{rel_pos[i, j].item()}', ha='center', va='center', fontsize=9,
                     color='white' if rel_pos[i, j].item() < k else 'black')
plt.colorbar(im2, ax=axes[1])

plt.tight_layout()
plt.savefig('relative_position_matrix.png', dpi=150, bbox_inches='tight')
plt.show()

class DisentangledAttention(nn.Module):
    """DeBERTaのDisentangled Attention

    コンテンツと位置を分離した3つの注意行列（C2C, C2P, P2C）を計算する。
    """

    def __init__(self, d_model, n_heads, max_relative_distance=512, dropout=0.1):
        super().__init__()
        assert d_model % n_heads == 0

        self.d_model = d_model
        self.n_heads = n_heads
        self.d_head = d_model // n_heads
        self.max_relative_distance = max_relative_distance

        # コンテンツ用の射影行列
        self.W_q_c = nn.Linear(d_model, d_model)  # Content Query
        self.W_k_c = nn.Linear(d_model, d_model)  # Content Key
        self.W_v_c = nn.Linear(d_model, d_model)  # Content Value

        # 位置用の射影行列（Valueは不要）
        self.W_q_p = nn.Linear(d_model, d_model)  # Position Query
        self.W_k_p = nn.Linear(d_model, d_model)  # Position Key

        # 相対位置埋め込みテーブル: 2k+1 個のベクトル
        self.rel_embeddings = nn.Embedding(
            2 * max_relative_distance + 1, d_model
        )

        # 出力射影
        self.W_o = nn.Linear(d_model, d_model)
        self.dropout = nn.Dropout(dropout)

        # スケーリングファクタ: 3つの項を足すので sqrt(3 * d_head)
        self.scale = (3 * self.d_head) ** 0.5

    def _split_heads(self, x):
        """(batch, seq_len, d_model) -> (batch, n_heads, seq_len, d_head)"""
        batch, seq_len, _ = x.shape
        x = x.view(batch, seq_len, self.n_heads, self.d_head)
        return x.transpose(1, 2)

    def _merge_heads(self, x):
        """(batch, n_heads, seq_len, d_head) -> (batch, seq_len, d_model)"""
        batch, _, seq_len, _ = x.shape
        x = x.transpose(1, 2).contiguous()
        return x.view(batch, seq_len, self.d_model)

    def forward(self, H, attention_mask=None):
        """
        Args:
            H: コンテンツ表現 (batch, seq_len, d_model)
            attention_mask: (batch, 1, 1, seq_len) or None

        Returns:
            output: (batch, seq_len, d_model)
            attention_weights: (batch, n_heads, seq_len, seq_len)
        """
        batch, seq_len, _ = H.shape

        # --- コンテンツの Query, Key, Value を計算 ---
        q_c = self._split_heads(self.W_q_c(H))  # (B, H, S, D)
        k_c = self._split_heads(self.W_k_c(H))  # (B, H, S, D)
        v_c = self._split_heads(self.W_v_c(H))  # (B, H, S, D)

        # --- 相対位置埋め込みの取得 ---
        rel_pos_idx = build_relative_position(seq_len, self.max_relative_distance)
        rel_pos_idx = rel_pos_idx.to(H.device)  # (S, S)
        rel_emb = self.rel_embeddings(rel_pos_idx)  # (S, S, d_model)

        # --- 1. Content-to-Content (C2C) ---
        # 標準的な Attention: q_c @ k_c^T
        A_c2c = torch.matmul(q_c, k_c.transpose(-1, -2))  # (B, H, S, S)

        # --- 2. Content-to-Position (C2P) ---
        # q_c[i] @ (rel_emb[i,j] @ W_k_p)^T
        k_p = self.W_k_p(rel_emb)  # (S, S, d_model)
        # ヘッド分割: (S, S, n_heads, d_head) -> (n_heads, S, S, d_head)
        k_p = k_p.view(seq_len, seq_len, self.n_heads, self.d_head).permute(2, 0, 1, 3)
        # q_c: (B, H, S, D), k_p: (H, S, S, D)
        # A_c2p[b, h, i, j] = q_c[b, h, i, :] @ k_p[h, i, j, :]
        A_c2p = torch.einsum('bhid,hijd->bhij', q_c, k_p)  # (B, H, S, S)

        # --- 3. Position-to-Content (P2C) ---
        # (rel_emb[j,i] @ W_q_p) @ k_c[j]^T
        q_p = self.W_q_p(rel_emb)  # (S, S, d_model)
        q_p = q_p.view(seq_len, seq_len, self.n_heads, self.d_head).permute(2, 0, 1, 3)
        # q_p[h, j, i, :] は位置jからiへの相対位置Query
        # P2Cでは、位置 j|i の相対位置Queryと k_c[j] の内積を取る
        # A_p2c[b, h, i, j] = q_p[h, j, i, :] @ k_c[b, h, j, :]
        A_p2c = torch.einsum('hjid,bhjd->bhij', q_p, k_c)  # (B, H, S, S)

        # --- 3つの注意スコアを統合 ---
        A = (A_c2c + A_c2p + A_p2c) / self.scale

        # マスクの適用
        if attention_mask is not None:
            A = A + attention_mask  # mask は 0 or -inf

        # softmax で正規化
        attn_weights = F.softmax(A, dim=-1)
        attn_weights = self.dropout(attn_weights)

        # --- Attention 出力（Value はコンテンツのみ）---
        output = torch.matmul(attn_weights, v_c)  # (B, H, S, D)
        output = self._merge_heads(output)          # (B, S, d_model)
        output = self.W_o(output)                   # (B, S, d_model)

        return output, attn_weights

# パラメータ設定
d_model = 64
n_heads = 4
seq_len = 10
batch_size = 1
max_k = 8

# モデルの初期化
torch.manual_seed(42)
attn = DisentangledAttention(d_model, n_heads, max_relative_distance=max_k)

# ダミー入力
H = torch.randn(batch_size, seq_len, d_model)

# フォワードパス
output, weights = attn(H)

print(f"入力形状:   {H.shape}")
print(f"出力形状:   {output.shape}")
print(f"重み形状:   {weights.shape}")
print(f"重みの和:   {weights[0, 0].sum(dim=-1)}")  # 各行の和が1になることを確認

# 注意重みの可視化（ヘッドごと）
fig, axes = plt.subplots(1, 4, figsize=(18, 4))

for h in range(n_heads):
    w = weights[0, h].detach().numpy()
    im = axes[h].imshow(w, cmap='Blues', vmin=0)
    axes[h].set_title(f'Head {h}', fontsize=12, fontweight='bold')
    axes[h].set_xlabel('Key position')
    axes[h].set_ylabel('Query position')
    plt.colorbar(im, ax=axes[h], fraction=0.046)

plt.suptitle('Disentangled Attention Weights by Head', fontsize=14, fontweight='bold', y=1.02)
plt.tight_layout()
plt.savefig('disentangled_attention_heads.png', dpi=150, bbox_inches='tight')
plt.show()

# 3つの注意行列の個別計算と可視化
torch.manual_seed(42)
attn2 = DisentangledAttention(d_model, n_heads, max_relative_distance=max_k)

# 内部で各項を個別に取得するための簡易計算
with torch.no_grad():
    q_c = attn2._split_heads(attn2.W_q_c(H))
    k_c = attn2._split_heads(attn2.W_k_c(H))

    rel_pos_idx = build_relative_position(seq_len, max_k).to(H.device)
    rel_emb = attn2.rel_embeddings(rel_pos_idx)

    # C2C
    A_c2c = torch.matmul(q_c, k_c.transpose(-1, -2))

    # C2P
    k_p = attn2.W_k_p(rel_emb).view(seq_len, seq_len, n_heads, d_model // n_heads).permute(2, 0, 1, 3)
    A_c2p = torch.einsum('bhid,hijd->bhij', q_c, k_p)

    # P2C
    q_p = attn2.W_q_p(rel_emb).view(seq_len, seq_len, n_heads, d_model // n_heads).permute(2, 0, 1, 3)
    A_p2c = torch.einsum('hjid,bhjd->bhij', q_p, k_c)

# Head 0 の各項を可視化
fig, axes = plt.subplots(1, 3, figsize=(16, 4.5))
titles = ['Content-to-Content (C2C)', 'Content-to-Position (C2P)', 'Position-to-Content (P2C)']
matrices = [A_c2c, A_c2p, A_p2c]

for idx, (title, mat) in enumerate(zip(titles, matrices)):
    data = mat[0, 0].detach().numpy()
    im = axes[idx].imshow(data, cmap='RdBu_r', vmin=-np.abs(data).max(), vmax=np.abs(data).max())
    axes[idx].set_title(title, fontsize=12, fontweight='bold')
    axes[idx].set_xlabel('Key position j')
    axes[idx].set_ylabel('Query position i')
    plt.colorbar(im, ax=axes[idx], fraction=0.046)

plt.suptitle('Three Attention Components (Head 0, before softmax)', fontsize=14, fontweight='bold', y=1.02)
plt.tight_layout()
plt.savefig('three_attention_components.png', dpi=150, bbox_inches='tight')
plt.show()

class EnhancedMaskDecoder(nn.Module):
    """DeBERTaのEnhanced Mask Decoder

    Encoder出力に絶対位置を注入し、追加のTransformer層で
    MLM予測を生成する。
    """

    def __init__(self, d_model, n_heads, max_seq_len=512,
                 max_relative_distance=512, vocab_size=30522, dropout=0.1):
        super().__init__()

        # 絶対位置埋め込み
        self.absolute_pos_embedding = nn.Embedding(max_seq_len, d_model)

        # EMD用のTransformer層（Disentangled Attention + FFN）
        self.emd_attention = DisentangledAttention(
            d_model, n_heads, max_relative_distance, dropout
        )
        self.emd_norm1 = nn.LayerNorm(d_model)
        self.emd_ffn = nn.Sequential(
            nn.Linear(d_model, d_model * 4),
            nn.GELU(),
            nn.Linear(d_model * 4, d_model),
            nn.Dropout(dropout)
        )
        self.emd_norm2 = nn.LayerNorm(d_model)
        self.dropout = nn.Dropout(dropout)

        # MLM予測ヘッド
        self.mlm_head = nn.Sequential(
            nn.Linear(d_model, d_model),
            nn.GELU(),
            nn.LayerNorm(d_model),
            nn.Linear(d_model, vocab_size)
        )

    def forward(self, encoder_output, masked_positions=None):
        """
        Args:
            encoder_output: Encoder の最終出力 (batch, seq_len, d_model)
            masked_positions: マスクされた位置 (batch, n_masked) or None

        Returns:
            logits: MLM予測のlogits
        """
        batch, seq_len, d_model = encoder_output.shape

        # --- 絶対位置の注入 ---
        positions = torch.arange(seq_len, device=encoder_output.device)
        abs_pos_emb = self.absolute_pos_embedding(positions)  # (S, D)

        # Encoder出力 + 絶対位置埋め込み
        H_emd = encoder_output + abs_pos_emb.unsqueeze(0)  # (B, S, D)

        # --- EMD Transformer Layer ---
        # Self-Attention + Residual + LayerNorm
        attn_out, _ = self.emd_attention(H_emd)
        H_emd = self.emd_norm1(H_emd + self.dropout(attn_out))

        # FFN + Residual + LayerNorm
        ffn_out = self.emd_ffn(H_emd)
        H_emd = self.emd_norm2(H_emd + ffn_out)

        # --- MLM予測 ---
        if masked_positions is not None:
            # マスク位置のみの表現を取得
            batch_idx = torch.arange(batch, device=H_emd.device).unsqueeze(1)
            masked_repr = H_emd[batch_idx, masked_positions]  # (B, n_masked, D)
            logits = self.mlm_head(masked_repr)
        else:
            # 全位置のMLM予測
            logits = self.mlm_head(H_emd)

        return logits

# EMDの動作確認
torch.manual_seed(42)

d_model = 64
n_heads = 4
max_seq_len = 32
vocab_size = 1000
batch_size = 2
seq_len = 10

emd = EnhancedMaskDecoder(
    d_model=d_model,
    n_heads=n_heads,
    max_seq_len=max_seq_len,
    max_relative_distance=8,
    vocab_size=vocab_size
)

# ダミーのEncoder出力
encoder_output = torch.randn(batch_size, seq_len, d_model)

# マスク位置（各サンプルで2箇所をマスク）
masked_positions = torch.tensor([[2, 5], [1, 7]])

# フォワードパス
logits = emd(encoder_output, masked_positions)

print(f"Encoder出力形状:  {encoder_output.shape}")
print(f"マスク位置:       {masked_positions}")
print(f"MLM logits形状:   {logits.shape}")
print(f"予測結果（Top-3）:")
top3 = torch.topk(logits[0, 0], k=3)
for rank, (val, idx) in enumerate(zip(top3.values, top3.indices), 1):
    print(f"  第{rank}位: token_id={idx.item()}, logit={val.item():.3f}")

class DeBERTaEncoder(nn.Module):
    """DeBERTaのEncoder（Disentangled Attention を使う Transformer Encoder）"""

    def __init__(self, d_model, n_heads, n_layers, max_relative_distance=512, dropout=0.1):
        super().__init__()
        self.layers = nn.ModuleList()
        for _ in range(n_layers):
            layer = nn.ModuleDict({
                'attention': DisentangledAttention(d_model, n_heads, max_relative_distance, dropout),
                'norm1': nn.LayerNorm(d_model),
                'ffn': nn.Sequential(
                    nn.Linear(d_model, d_model * 4),
                    nn.GELU(),
                    nn.Linear(d_model * 4, d_model),
                    nn.Dropout(dropout)
                ),
                'norm2': nn.LayerNorm(d_model),
            })
            self.layers.append(layer)
        self.dropout = nn.Dropout(dropout)

    def forward(self, H, attention_mask=None):
        for layer in self.layers:
            # Self-Attention + Residual + LayerNorm
            attn_out, _ = layer['attention'](H, attention_mask)
            H = layer['norm1'](H + self.dropout(attn_out))
            # FFN + Residual + LayerNorm
            ffn_out = layer['ffn'](H)
            H = layer['norm2'](H + ffn_out)
        return H


class DeBERTaModel(nn.Module):
    """DeBERTaの全体モデル（事前学習用）"""

    def __init__(self, vocab_size=30522, d_model=768, n_heads=12,
                 n_layers=12, max_seq_len=512, max_relative_distance=512, dropout=0.1):
        super().__init__()

        # トークン埋め込み（位置埋め込みは加算しない）
        self.token_embedding = nn.Embedding(vocab_size, d_model)
        self.embedding_norm = nn.LayerNorm(d_model)
        self.embedding_dropout = nn.Dropout(dropout)

        # Encoder（Disentangled Attention ベース）
        self.encoder = DeBERTaEncoder(
            d_model, n_heads, n_layers, max_relative_distance, dropout
        )

        # Enhanced Mask Decoder
        self.emd = EnhancedMaskDecoder(
            d_model, n_heads, max_seq_len, max_relative_distance, vocab_size, dropout
        )

    def forward(self, input_ids, attention_mask=None, masked_positions=None):
        # トークン埋め込み（絶対位置は加算しない）
        H = self.token_embedding(input_ids)
        H = self.embedding_norm(H)
        H = self.embedding_dropout(H)

        # Encoder: Disentangled Attention で文脈を構築
        encoder_output = self.encoder(H, attention_mask)

        # EMD: 絶対位置を注入してMLM予測
        logits = self.emd(encoder_output, masked_positions)

        return logits

    def count_parameters(self):
        return sum(p.numel() for p in self.parameters() if p.requires_grad)


# モデル全体の確認
model = DeBERTaModel(
    vocab_size=1000,
    d_model=64,
    n_heads=4,
    n_layers=2,
    max_seq_len=32,
    max_relative_distance=8,
    dropout=0.1
)

input_ids = torch.randint(0, 1000, (2, 10))
masked_positions = torch.tensor([[2, 5], [1, 7]])

logits = model(input_ids, masked_positions=masked_positions)

print(f"入力形状:       {input_ids.shape}")
print(f"出力形状:       {logits.shape}")
print(f"パラメータ数:   {model.count_parameters():,}")
print(f"\nモデル構造:")
print(f"  トークン埋め込み: vocab_size=1000, d_model=64")
print(f"  Encoder: 2層 x DisentangledAttention(4 heads)")
print(f"  EMD: 1層 + MLM head")

import matplotlib.pyplot as plt
import numpy as np

# SuperGLUEベンチマークのスコア（論文・リーダーボードから）
models = ['BERT-Large', 'RoBERTa-Large', 'DeBERTa-Large', 'DeBERTa V3-Large', 'Human']
superglue_avg = [69.0, 84.6, 88.3, 91.4, 89.8]

colors = ['#42A5F5', '#66BB6A', '#FF7043', '#AB47BC', '#BDBDBD']

fig, ax = plt.subplots(figsize=(10, 6))
bars = ax.barh(models, superglue_avg, color=colors, edgecolor='white', linewidth=1.5, height=0.6)

# 人間のベースラインに赤い破線を追加
ax.axvline(x=89.8, color='red', linestyle='--', alpha=0.7, label='Human Baseline (89.8)')

# スコアをバーの横に表示
for bar, score in zip(bars, superglue_avg):
    ax.text(bar.get_width() + 0.5, bar.get_y() + bar.get_height()/2,
            f'{score:.1f}', ha='left', va='center', fontsize=12, fontweight='bold')

ax.set_xlabel('SuperGLUE Score', fontsize=13)
ax.set_title('SuperGLUE Benchmark Comparison', fontsize=15, fontweight='bold')
ax.set_xlim(60, 97)
ax.legend(fontsize=11)
ax.grid(axis='x', alpha=0.3)

plt.tight_layout()
plt.savefig('superglue_comparison.png', dpi=150, bbox_inches='tight')
plt.show()

import matplotlib.pyplot as plt
import numpy as np

# GLUEベンチマークの個別タスクスコア
tasks = ['MNLI', 'QQP', 'QNLI', 'SST-2', 'CoLA', 'STS-B', 'MRPC', 'RTE']

bert_scores =     [86.6, 91.3, 92.3, 93.2, 60.6, 90.0, 88.0, 70.4]
roberta_scores =  [90.2, 92.2, 94.7, 96.4, 68.0, 92.4, 90.9, 86.6]
deberta_scores =  [91.1, 92.3, 95.3, 96.8, 70.5, 92.8, 92.0, 88.3]
deberta_v3 =      [91.8, 92.7, 96.0, 97.1, 72.4, 93.0, 92.5, 92.4]

x = np.arange(len(tasks))
width = 0.2

fig, ax = plt.subplots(figsize=(16, 7))

b1 = ax.bar(x - 1.5*width, bert_scores, width, label='BERT-Large', color='#42A5F5', alpha=0.85)
b2 = ax.bar(x - 0.5*width, roberta_scores, width, label='RoBERTa-Large', color='#66BB6A', alpha=0.85)
b3 = ax.bar(x + 0.5*width, deberta_scores, width, label='DeBERTa-Large', color='#FF7043', alpha=0.85)
b4 = ax.bar(x + 1.5*width, deberta_v3, width, label='DeBERTa V3-Large', color='#AB47BC', alpha=0.85)

ax.set_xlabel('GLUE Tasks', fontsize=13)
ax.set_ylabel('Score', fontsize=13)
ax.set_title('GLUE Benchmark: Detailed Task Comparison', fontsize=15, fontweight='bold')
ax.set_xticks(x)
ax.set_xticklabels(tasks, fontsize=11)
ax.legend(fontsize=11, loc='lower right')
ax.set_ylim(55, 100)
ax.grid(axis='y', alpha=0.3)

# 各バーの上にスコアを表示
for bars in [b1, b2, b3, b4]:
    for bar in bars:
        height = bar.get_height()
        ax.annotate(f'{height:.1f}',
                    xy=(bar.get_x() + bar.get_width() / 2, height),
                    xytext=(0, 3), textcoords="offset points",
                    ha='center', va='bottom', fontsize=7)

plt.tight_layout()
plt.savefig('glue_detailed_comparison.png', dpi=150, bbox_inches='tight')
plt.show()

# 改善幅の表示
print("=== DeBERTa V3 vs BERT-Large の改善幅 ===")
for task, bert, dv3 in zip(tasks, bert_scores, deberta_v3):
    diff = dv3 - bert
    print(f"  {task:8s}: {bert:.1f} -> {dv3:.1f} (+{diff:.1f})")
avg_improvement = np.mean(np.array(deberta_v3) - np.array(bert_scores))
print(f"  平均改善: +{avg_improvement:.1f}")

import matplotlib.pyplot as plt
import numpy as np

# He et al. (2021) の消去実験データ（MNLIの精度）
configs = [
    'BERT-Large\n(Baseline)',
    '+ Relative\nPosition',
    '+ Disentangled\nAttention',
    '+ EMD',
    'DeBERTa\n(Full)'
]
mnli_scores = [86.6, 88.1, 89.5, 90.1, 91.1]

fig, ax = plt.subplots(figsize=(11, 6))

colors = ['#90CAF9', '#42A5F5', '#1E88E5', '#1565C0', '#0D47A1']
bars = ax.bar(range(len(configs)), mnli_scores, color=colors, edgecolor='white', linewidth=1.5, width=0.65)

# 改善幅の矢印
for i in range(1, len(configs)):
    diff = mnli_scores[i] - mnli_scores[i-1]
    ax.annotate(f'+{diff:.1f}',
                xy=(i, mnli_scores[i]),
                xytext=(i, mnli_scores[i] + 0.4),
                ha='center', fontsize=11, color='red', fontweight='bold')

# スコア表示
for i, (bar, score) in enumerate(zip(bars, mnli_scores)):
    ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() - 0.5,
            f'{score:.1f}', ha='center', va='top', fontsize=12, fontweight='bold', color='white')

ax.set_ylabel('MNLI-m Accuracy (%)', fontsize=13)
ax.set_title('DeBERTa Ablation Study (MNLI-m)', fontsize=15, fontweight='bold')
ax.set_xticks(range(len(configs)))
ax.set_xticklabels(configs, fontsize=10)
ax.set_ylim(85, 92.5)
ax.grid(axis='y', alpha=0.3)

# 累積改善の表示
total = mnli_scores[-1] - mnli_scores[0]
ax.text(len(configs)-1, 92, f'Total: +{total:.1f}', ha='center', fontsize=12,
        fontweight='bold', color='darkred',
        bbox=dict(boxstyle='round,pad=0.3', facecolor='lightyellow', edgecolor='darkred'))

plt.tight_layout()
plt.savefig('deberta_ablation.png', dpi=150, bbox_inches='tight')
plt.show()

# 各改良の寄与率
print("=== 各改良の寄与率（MNLI-m）===")
total = mnli_scores[-1] - mnli_scores[0]
for i in range(1, len(configs)):
    diff = mnli_scores[i] - mnli_scores[i-1]
    pct = diff / total * 100
    label = configs[i].replace('\n', ' ')
    print(f"  {label}: +{diff:.1f} ({pct:.0f}%)")
print(f"  合計: +{total:.1f}")