GPTのアーキテクチャと自己回帰生成を解説

import numpy as np
import matplotlib.pyplot as plt

class CausalSelfAttention:
    """因果マスク付き Self-Attention"""
    def __init__(self, d_model, n_heads):
        self.d_model = d_model
        self.n_heads = n_heads
        self.d_k = d_model // n_heads

        # Q, K, V, O の重み行列
        scale = np.sqrt(2.0 / d_model)
        self.W_q = np.random.randn(d_model, d_model) * scale
        self.W_k = np.random.randn(d_model, d_model) * scale
        self.W_v = np.random.randn(d_model, d_model) * scale
        self.W_o = np.random.randn(d_model, d_model) * scale

    def forward(self, x):
        """
        x: (seq_len, d_model)
        returns: (seq_len, d_model)
        """
        T, D = x.shape

        # Q, K, V の計算
        Q = x @ self.W_q  # (T, d_model)
        K = x @ self.W_k
        V = x @ self.W_v

        # Multi-Head に分割
        Q = Q.reshape(T, self.n_heads, self.d_k).transpose(1, 0, 2)  # (n_heads, T, d_k)
        K = K.reshape(T, self.n_heads, self.d_k).transpose(1, 0, 2)
        V = V.reshape(T, self.n_heads, self.d_k).transpose(1, 0, 2)

        # スケーリング付き内積アテンション
        scores = Q @ K.transpose(0, 2, 1) / np.sqrt(self.d_k)  # (n_heads, T, T)

        # 因果マスクの適用
        causal_mask = np.triu(np.full((T, T), -1e9), k=1)  # 上三角を -inf に
        scores = scores + causal_mask[np.newaxis, :, :]

        # ソフトマックス
        scores_max = np.max(scores, axis=-1, keepdims=True)
        exp_scores = np.exp(scores - scores_max)
        attn_weights = exp_scores / np.sum(exp_scores, axis=-1, keepdims=True)

        # 重み付き和
        out = attn_weights @ V  # (n_heads, T, d_k)

        # ヘッドの連結
        out = out.transpose(1, 0, 2).reshape(T, D)  # (T, d_model)

        # 出力射影
        out = out @ self.W_o

        return out, attn_weights

# 因果マスクの可視化
T = 8
causal_mask = np.triu(np.full((T, T), -np.inf), k=1)
exp_mask = np.where(causal_mask == 0, 1, 0)

plt.figure(figsize=(6, 5))
plt.imshow(exp_mask, cmap='Blues', interpolation='nearest')
plt.xlabel("Key Position (j)")
plt.ylabel("Query Position (i)")
plt.title("Causal Mask (1 = attend, 0 = masked)")
for i in range(T):
    for j in range(T):
        plt.text(j, i, str(exp_mask[i, j]), ha='center', va='center', fontsize=10)
plt.colorbar()
plt.tight_layout()
plt.show()

import numpy as np
import matplotlib.pyplot as plt

def gelu(x):
    """GELU 活性化関数"""
    return 0.5 * x * (1 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * x**3)))

def softmax(x, axis=-1):
    """数値安定なソフトマックス"""
    x_max = np.max(x, axis=axis, keepdims=True)
    exp_x = np.exp(x - x_max)
    return exp_x / np.sum(exp_x, axis=axis, keepdims=True)

def layer_norm(x, gamma, beta, eps=1e-5):
    """レイヤー正規化"""
    mean = np.mean(x, axis=-1, keepdims=True)
    var = np.var(x, axis=-1, keepdims=True)
    return gamma * (x - mean) / np.sqrt(var + eps) + beta

class MiniGPT:
    """小規模 GPT の実装"""
    def __init__(self, vocab_size, d_model, n_heads, n_layers, max_seq_len):
        self.vocab_size = vocab_size
        self.d_model = d_model
        self.n_heads = n_heads
        self.n_layers = n_layers
        self.d_k = d_model // n_heads
        self.d_ff = 4 * d_model

        scale = 0.02

        # トークン埋め込み
        self.token_emb = np.random.randn(vocab_size, d_model) * scale
        # 位置埋め込み
        self.pos_emb = np.random.randn(max_seq_len, d_model) * scale

        # Transformer ブロックのパラメータ
        self.layers = []
        for _ in range(n_layers):
            layer = {
                # LayerNorm 1
                'ln1_gamma': np.ones(d_model),
                'ln1_beta': np.zeros(d_model),
                # Multi-Head Attention
                'W_q': np.random.randn(d_model, d_model) * scale,
                'W_k': np.random.randn(d_model, d_model) * scale,
                'W_v': np.random.randn(d_model, d_model) * scale,
                'W_o': np.random.randn(d_model, d_model) * scale,
                # LayerNorm 2
                'ln2_gamma': np.ones(d_model),
                'ln2_beta': np.zeros(d_model),
                # FFN
                'W1': np.random.randn(d_model, self.d_ff) * scale,
                'b1': np.zeros(self.d_ff),
                'W2': np.random.randn(self.d_ff, d_model) * scale,
                'b2': np.zeros(d_model),
            }
            self.layers.append(layer)

        # 最終レイヤー正規化
        self.ln_f_gamma = np.ones(d_model)
        self.ln_f_beta = np.zeros(d_model)

        # 出力射影（トークン埋め込みと重み共有）
        self.W_out = self.token_emb  # weight tying

    def causal_attention(self, x, layer_params):
        """因果マスク付き Multi-Head Self-Attention"""
        T, D = x.shape
        Q = x @ layer_params['W_q']
        K = x @ layer_params['W_k']
        V = x @ layer_params['W_v']

        # Multi-Head 分割
        Q = Q.reshape(T, self.n_heads, self.d_k).transpose(1, 0, 2)
        K = K.reshape(T, self.n_heads, self.d_k).transpose(1, 0, 2)
        V = V.reshape(T, self.n_heads, self.d_k).transpose(1, 0, 2)

        # アテンションスコア + 因果マスク
        scores = Q @ K.transpose(0, 2, 1) / np.sqrt(self.d_k)
        mask = np.triu(np.full((T, T), -1e9), k=1)
        scores = scores + mask[np.newaxis, :, :]

        attn = softmax(scores, axis=-1)
        out = attn @ V
        out = out.transpose(1, 0, 2).reshape(T, D)
        out = out @ layer_params['W_o']
        return out

    def ffn(self, x, layer_params):
        """Position-wise Feed-Forward Network"""
        h = gelu(x @ layer_params['W1'] + layer_params['b1'])
        return h @ layer_params['W2'] + layer_params['b2']

    def forward(self, token_ids):
        """
        token_ids: (seq_len,) の整数配列
        returns: (seq_len, vocab_size) のロジット
        """
        T = len(token_ids)

        # 埋め込み
        x = self.token_emb[token_ids] + self.pos_emb[:T]

        # Transformer ブロック（Pre-Norm）
        for layer_params in self.layers:
            # Self-Attention
            h = layer_norm(x, layer_params['ln1_gamma'], layer_params['ln1_beta'])
            x = x + self.causal_attention(h, layer_params)

            # FFN
            h = layer_norm(x, layer_params['ln2_gamma'], layer_params['ln2_beta'])
            x = x + self.ffn(h, layer_params)

        # 最終 LayerNorm
        x = layer_norm(x, self.ln_f_gamma, self.ln_f_beta)

        # ロジット
        logits = x @ self.W_out.T  # (T, vocab_size)
        return logits

    def generate(self, start_tokens, max_new_tokens, temperature=1.0,
                 top_k=None, top_p=None):
        """テキスト生成"""
        tokens = list(start_tokens)

        for _ in range(max_new_tokens):
            # 順伝播
            logits = self.forward(np.array(tokens))
            next_logits = logits[-1]  # 最後の位置のロジット

            # Temperature の適用
            next_logits = next_logits / temperature

            # 確率分布の計算
            probs = softmax(next_logits)

            # Top-k フィルタリング
            if top_k is not None:
                top_k_indices = np.argsort(probs)[-top_k:]
                mask = np.zeros_like(probs)
                mask[top_k_indices] = 1
                probs = probs * mask
                probs = probs / (probs.sum() + 1e-10)

            # Top-p (Nucleus) フィルタリング
            if top_p is not None:
                sorted_indices = np.argsort(probs)[::-1]
                sorted_probs = probs[sorted_indices]
                cumsum = np.cumsum(sorted_probs)
                cutoff_idx = np.searchsorted(cumsum, top_p) + 1
                allowed = sorted_indices[:cutoff_idx]
                mask = np.zeros_like(probs)
                mask[allowed] = 1
                probs = probs * mask
                probs = probs / (probs.sum() + 1e-10)

            # サンプリング
            next_token = np.random.choice(len(probs), p=probs)
            tokens.append(next_token)

        return tokens

# モデルの生成とテスト
np.random.seed(42)
vocab_size = 50
d_model = 32
n_heads = 4
n_layers = 2
max_seq_len = 64

model = MiniGPT(vocab_size, d_model, n_heads, n_layers, max_seq_len)

# 簡易トークナイザ
idx2token = {i: f"tok_{i}" for i in range(vocab_size)}
idx2token[0] = "<BOS>"
idx2token[1] = "<EOS>"

# テキスト生成のデモ
start = [0]  # <BOS> から開始
print("=== Greedy Decoding (temperature=0.1) ===")
generated = model.generate(start, max_new_tokens=10, temperature=0.1)
print("生成トークン:", [idx2token[t] for t in generated])

print("\n=== Temperature Sampling (temperature=1.0) ===")
generated = model.generate(start, max_new_tokens=10, temperature=1.0)
print("生成トークン:", [idx2token[t] for t in generated])

print("\n=== Top-k Sampling (k=5) ===")
generated = model.generate(start, max_new_tokens=10, temperature=1.0, top_k=5)
print("生成トークン:", [idx2token[t] for t in generated])

print("\n=== Top-p Sampling (p=0.9) ===")
generated = model.generate(start, max_new_tokens=10, temperature=1.0, top_p=0.9)
print("生成トークン:", [idx2token[t] for t in generated])

import numpy as np
import matplotlib.pyplot as plt

# ロジットの例
np.random.seed(0)
logits = np.array([3.0, 2.5, 1.0, 0.5, 0.3, -0.5, -1.0, -2.0, -3.0, -4.0])
vocab = [f"w{i}" for i in range(len(logits))]

fig, axes = plt.subplots(2, 2, figsize=(12, 10))

# 1. Temperature の効果
for i, tau in enumerate([0.5, 1.0, 2.0]):
    probs = np.exp(logits / tau) / np.sum(np.exp(logits / tau))
    axes[0, 0].bar(np.arange(len(logits)) + i*0.25 - 0.25, probs, width=0.25,
                   label=f"T={tau}", alpha=0.8)
axes[0, 0].set_xticks(range(len(logits)))
axes[0, 0].set_xticklabels(vocab)
axes[0, 0].set_title("Temperature Scaling")
axes[0, 0].set_ylabel("Probability")
axes[0, 0].legend()

# 2. Top-k Sampling
probs_orig = np.exp(logits) / np.sum(np.exp(logits))
for i, k in enumerate([3, 5]):
    probs = probs_orig.copy()
    top_k_idx = np.argsort(probs)[-k:]
    mask = np.zeros_like(probs)
    mask[top_k_idx] = 1
    probs = probs * mask
    probs = probs / probs.sum()
    axes[0, 1].bar(np.arange(len(logits)) + i*0.3 - 0.15, probs, width=0.3,
                   label=f"k={k}", alpha=0.8)
axes[0, 1].set_xticks(range(len(logits)))
axes[0, 1].set_xticklabels(vocab)
axes[0, 1].set_title("Top-k Sampling")
axes[0, 1].set_ylabel("Probability")
axes[0, 1].legend()

# 3. Top-p Sampling
for i, p in enumerate([0.7, 0.9]):
    probs = probs_orig.copy()
    sorted_idx = np.argsort(probs)[::-1]
    sorted_probs = probs[sorted_idx]
    cumsum = np.cumsum(sorted_probs)
    cutoff = np.searchsorted(cumsum, p) + 1
    allowed = sorted_idx[:cutoff]
    mask = np.zeros_like(probs)
    mask[allowed] = 1
    probs = probs * mask
    probs = probs / probs.sum()
    axes[1, 0].bar(np.arange(len(logits)) + i*0.3 - 0.15, probs, width=0.3,
                   label=f"p={p}", alpha=0.8)
axes[1, 0].set_xticks(range(len(logits)))
axes[1, 0].set_xticklabels(vocab)
axes[1, 0].set_title("Top-p (Nucleus) Sampling")
axes[1, 0].set_ylabel("Probability")
axes[1, 0].legend()

# 4. 全戦略の比較
strategies = {
    "Greedy": np.eye(len(logits))[np.argmax(logits)],
    "T=1.0": probs_orig,
    "Top-k=3": None,
    "Top-p=0.9": None,
}
# Top-k=3
probs_topk = probs_orig.copy()
top3 = np.argsort(probs_topk)[-3:]
mask = np.zeros_like(probs_topk)
mask[top3] = 1
probs_topk = probs_topk * mask
strategies["Top-k=3"] = probs_topk / probs_topk.sum()

# Top-p=0.9
probs_topp = probs_orig.copy()
sorted_idx = np.argsort(probs_topp)[::-1]
cumsum = np.cumsum(probs_topp[sorted_idx])
cutoff = np.searchsorted(cumsum, 0.9) + 1
allowed = sorted_idx[:cutoff]
mask = np.zeros_like(probs_topp)
mask[allowed] = 1
probs_topp = probs_topp * mask
strategies["Top-p=0.9"] = probs_topp / probs_topp.sum()

x_pos = np.arange(len(logits))
width = 0.2
for i, (name, p) in enumerate(strategies.items()):
    axes[1, 1].bar(x_pos + i*width - 0.3, p, width=width, label=name, alpha=0.8)
axes[1, 1].set_xticks(range(len(logits)))
axes[1, 1].set_xticklabels(vocab)
axes[1, 1].set_title("Comparison of Strategies")
axes[1, 1].set_ylabel("Probability")
axes[1, 1].legend(fontsize=8)

plt.suptitle("Text Generation Strategies", fontsize=14, y=1.01)
plt.tight_layout()
plt.show()