評価メトリクス（BLEU・ROUGE・BERTScore）— テキスト生成の自動評価を理解する

import math
from collections import Counter


def get_ngrams(tokens, n):
    """トークン列からn-gramのリストを生成する"""
    return [tuple(tokens[i:i+n]) for i in range(len(tokens) - n + 1)]


def modified_precision(candidate, references, n):
    """Modified Precisionを計算する"""
    # 生成文のn-gramカウント
    cand_ngrams = get_ngrams(candidate, n)
    cand_counts = Counter(cand_ngrams)

    if len(cand_ngrams) == 0:
        return 0.0

    # 各参照文のn-gramカウントの最大値でクリッピング
    max_ref_counts = Counter()
    for ref in references:
        ref_counts = Counter(get_ngrams(ref, n))
        for ngram, count in ref_counts.items():
            max_ref_counts[ngram] = max(max_ref_counts[ngram], count)

    # クリッピング: 生成文のカウントを参照文の最大カウントで上限
    clipped_total = 0
    for ngram, count in cand_counts.items():
        clipped_total += min(count, max_ref_counts.get(ngram, 0))

    total = sum(cand_counts.values())
    return clipped_total / total


def brevity_penalty(candidate, references):
    """Brevity Penaltyを計算する"""
    c = len(candidate)
    # 参照文の中で生成文に最も近い長さを選択
    ref_lens = [len(ref) for ref in references]
    r = min(ref_lens, key=lambda ref_len: (abs(ref_len - c), ref_len))

    if c > r:
        return 1.0
    elif c == 0:
        return 0.0
    else:
        return math.exp(1 - r / c)


def compute_bleu(candidate, references, max_n=4, weights=None):
    """BLEUスコアを計算する"""
    if weights is None:
        weights = [1.0 / max_n] * max_n

    # Modified Precisionを各nについて計算
    precisions = []
    for n in range(1, max_n + 1):
        p = modified_precision(candidate, references, n)
        precisions.append(p)

    # いずれかのprecisionが0なら、BLEUは0
    if any(p == 0 for p in precisions):
        return 0.0

    # 対数の重み付き平均 → 幾何平均
    log_avg = sum(w * math.log(p) for w, p in zip(weights, precisions))
    bp = brevity_penalty(candidate, references)

    return bp * math.exp(log_avg)


# --- 計算例 ---
reference1 = "the cat is on the mat".split()
reference2 = "there is a cat on the mat".split()
candidate = "the cat sat on the mat".split()

score = compute_bleu(candidate, [reference1, reference2], max_n=4)
print(f"候補文: {' '.join(candidate)}")
print(f"参照文1: {' '.join(reference1)}")
print(f"参照文2: {' '.join(reference2)}")
print(f"BLEU-4: {score:.4f}")
print()

# 各n-gramのModified Precisionも表示
for n in range(1, 5):
    p = modified_precision(candidate, [reference1, reference2], n)
    print(f"  p_{n} (Modified {n}-gram Precision): {p:.4f}")

bp = brevity_penalty(candidate, [reference1, reference2])
print(f"  Brevity Penalty: {bp:.4f}")

# BLEUの限界を示す例
print("=== BLEUの限界: 同義語 ===")
ref = "the cat quickly ran across the field".split()
cand_synonym = "the cat rapidly sprinted over the meadow".split()
cand_exact = "the cat quickly ran across the field".split()

score_synonym = compute_bleu(cand_synonym, [ref], max_n=4)
score_exact = compute_bleu(cand_exact, [ref], max_n=4)

print(f"参照文:     {' '.join(ref)}")
print(f"同義語文:   {' '.join(cand_synonym)} → BLEU: {score_synonym:.4f}")
print(f"完全一致文: {' '.join(cand_exact)} → BLEU: {score_exact:.4f}")
print()

print("=== BLEUの限界: 繰り返し ===")
ref2 = "the cat is on the mat".split()
cand_repeat = "the the the the the the".split()
cand_good = "the cat is on the mat".split()

score_repeat = compute_bleu(cand_repeat, [ref2], max_n=1, weights=[1.0])
score_good = compute_bleu(cand_good, [ref2], max_n=1, weights=[1.0])

print(f"参照文:   {' '.join(ref2)}")
print(f"繰り返し: {' '.join(cand_repeat)} → BLEU-1: {score_repeat:.4f}")
print(f"正解文:   {' '.join(cand_good)} → BLEU-1: {score_good:.4f}")

from collections import Counter


def rouge_n(candidate, reference, n):
    """ROUGE-N のRecall, Precision, F1を計算する"""
    cand_ngrams = get_ngrams(candidate, n)
    ref_ngrams = get_ngrams(reference, n)

    cand_counts = Counter(cand_ngrams)
    ref_counts = Counter(ref_ngrams)

    # 共通n-gramのカウント（各n-gramの最小出現数を合計）
    overlap = 0
    for ngram in ref_counts:
        overlap += min(ref_counts[ngram], cand_counts.get(ngram, 0))

    # 分母が0の場合の処理
    total_ref = sum(ref_counts.values())
    total_cand = sum(cand_counts.values())

    recall = overlap / total_ref if total_ref > 0 else 0.0
    precision = overlap / total_cand if total_cand > 0 else 0.0

    if precision + recall > 0:
        f1 = 2 * precision * recall / (precision + recall)
    else:
        f1 = 0.0

    return {"recall": recall, "precision": precision, "f1": f1}


def lcs_length(x, y):
    """最長共通部分列の長さを動的計画法で計算する"""
    m, n = len(x), len(y)
    # DPテーブル: L[i][j]はx[:i]とy[:j]のLCS長
    L = [[0] * (n + 1) for _ in range(m + 1)]

    for i in range(1, m + 1):
        for j in range(1, n + 1):
            if x[i - 1] == y[j - 1]:
                L[i][j] = L[i - 1][j - 1] + 1
            else:
                L[i][j] = max(L[i - 1][j], L[i][j - 1])

    return L[m][n]


def rouge_l(candidate, reference, beta=1.2):
    """ROUGE-L のRecall, Precision, F1を計算する"""
    lcs = lcs_length(reference, candidate)
    m = len(reference)
    n = len(candidate)

    recall = lcs / m if m > 0 else 0.0
    precision = lcs / n if n > 0 else 0.0

    if precision + recall > 0:
        f1 = ((1 + beta**2) * precision * recall) / \
             (recall + beta**2 * precision)
    else:
        f1 = 0.0

    return {"recall": recall, "precision": precision, "f1": f1}


# --- 計算例 ---
reference = "police killed the gunman".split()
candidate1 = "police kill the gunman".split()
candidate2 = "the gunman police killed".split()

print("=== ROUGE-1 ===")
for cand, label in [(candidate1, "候補1"), (candidate2, "候補2")]:
    result = rouge_n(cand, reference, n=1)
    print(f"{label}: {' '.join(cand)}")
    print(f"  Recall={result['recall']:.3f}, "
          f"Precision={result['precision']:.3f}, "
          f"F1={result['f1']:.3f}")

print("\n=== ROUGE-2 ===")
for cand, label in [(candidate1, "候補1"), (candidate2, "候補2")]:
    result = rouge_n(cand, reference, n=2)
    print(f"{label}: {' '.join(cand)}")
    print(f"  Recall={result['recall']:.3f}, "
          f"Precision={result['precision']:.3f}, "
          f"F1={result['f1']:.3f}")

print("\n=== ROUGE-L ===")
for cand, label in [(candidate1, "候補1"), (candidate2, "候補2")]:
    result = rouge_l(cand, reference)
    print(f"{label}: {' '.join(cand)}")
    print(f"  Recall={result['recall']:.3f}, "
          f"Precision={result['precision']:.3f}, "
          f"F1={result['f1']:.3f}")
    lcs = lcs_length(reference, cand)
    print(f"  LCS長={lcs}")

import numpy as np


def cosine_similarity_matrix(embeddings1, embeddings2):
    """2つの埋め込み行列間のコサイン類似度行列を計算する"""
    # 正規化
    norms1 = np.linalg.norm(embeddings1, axis=1, keepdims=True)
    norms2 = np.linalg.norm(embeddings2, axis=1, keepdims=True)
    normed1 = embeddings1 / (norms1 + 1e-8)
    normed2 = embeddings2 / (norms2 + 1e-8)

    # コサイン類似度行列: (m, d) @ (d, n) = (m, n)
    return normed1 @ normed2.T


def bert_score(ref_embeddings, cand_embeddings, idf_weights=None):
    """BERTScoreのPrecision, Recall, F1を計算する"""
    # コサイン類似度行列
    sim_matrix = cosine_similarity_matrix(ref_embeddings, cand_embeddings)
    m, n = sim_matrix.shape

    if idf_weights is not None:
        # IDF重み付きRecall
        ref_max = np.max(sim_matrix, axis=1)  # 各参照トークンの最大類似度
        recall = np.sum(idf_weights * ref_max) / np.sum(idf_weights)

        # IDF重み付きPrecision（生成文側のIDFは省略、均等重み）
        cand_max = np.max(sim_matrix, axis=0)
        precision = np.mean(cand_max)
    else:
        # Recall: 参照文の各トークンに対して最大類似度を取り、平均
        recall = np.mean(np.max(sim_matrix, axis=1))

        # Precision: 生成文の各トークンに対して最大類似度を取り、平均
        precision = np.mean(np.max(sim_matrix, axis=0))

    if precision + recall > 0:
        f1 = 2 * precision * recall / (precision + recall)
    else:
        f1 = 0.0

    return {
        "precision": float(precision),
        "recall": float(recall),
        "f1": float(f1),
        "similarity_matrix": sim_matrix
    }


# --- 概念的デモ: 手動の埋め込みで仕組みを確認 ---
# 意味の近い単語は類似したベクトルになるよう手動で設定
np.random.seed(42)
d = 64  # 埋め込み次元

# 基底ベクトルを作成（意味カテゴリごと）
animal_base = np.random.randn(d)
action_base = np.random.randn(d)
location_base = np.random.randn(d)
article_base = np.random.randn(d)

# 意味の近い単語は基底にノイズを加えて生成
word_embeddings = {
    "the": article_base + 0.1 * np.random.randn(d),
    "a": article_base + 0.1 * np.random.randn(d),
    "cat": animal_base + 0.1 * np.random.randn(d),
    "dog": animal_base + 0.15 * np.random.randn(d),
    "feline": animal_base + 0.12 * np.random.randn(d),
    "sat": action_base + 0.1 * np.random.randn(d),
    "sits": action_base + 0.12 * np.random.randn(d),
    "rested": action_base + 0.15 * np.random.randn(d),
    "on": location_base + 0.1 * np.random.randn(d),
    "upon": location_base + 0.12 * np.random.randn(d),
    "mat": location_base + 0.3 * np.random.randn(d),
    "rug": location_base + 0.32 * np.random.randn(d),
}

ref_tokens = ["the", "cat", "sat", "on", "the", "mat"]
cand1_tokens = ["the", "cat", "sat", "on", "the", "mat"]  # 完全一致
cand2_tokens = ["a", "feline", "rested", "upon", "a", "rug"]  # 同義語

ref_emb = np.array([word_embeddings[t] for t in ref_tokens])
cand1_emb = np.array([word_embeddings[t] for t in cand1_tokens])
cand2_emb = np.array([word_embeddings[t] for t in cand2_tokens])

result1 = bert_score(ref_emb, cand1_emb)
result2 = bert_score(ref_emb, cand2_emb)

print("=== BERTScore概念デモ ===")
print(f"参照文:   {' '.join(ref_tokens)}")
print(f"候補1（完全一致）: {' '.join(cand1_tokens)}")
print(f"  P={result1['precision']:.4f}, "
      f"R={result1['recall']:.4f}, "
      f"F1={result1['f1']:.4f}")
print(f"候補2（同義語）:   {' '.join(cand2_tokens)}")
print(f"  P={result2['precision']:.4f}, "
      f"R={result2['recall']:.4f}, "
      f"F1={result2['f1']:.4f}")

import matplotlib.pyplot as plt


def plot_similarity_matrix(sim_matrix, ref_tokens, cand_tokens, title):
    """類似度行列をヒートマップで可視化する"""
    fig, ax = plt.subplots(figsize=(8, 6))
    im = ax.imshow(sim_matrix, cmap="YlOrRd", vmin=0, vmax=1)

    ax.set_xticks(range(len(cand_tokens)))
    ax.set_yticks(range(len(ref_tokens)))
    ax.set_xticklabels(cand_tokens, fontsize=11)
    ax.set_yticklabels(ref_tokens, fontsize=11)
    ax.set_xlabel("Candidate tokens", fontsize=12)
    ax.set_ylabel("Reference tokens", fontsize=12)
    ax.set_title(title, fontsize=13)

    # 各セルに数値を表示
    for i in range(len(ref_tokens)):
        for j in range(len(cand_tokens)):
            color = "white" if sim_matrix[i, j] > 0.7 else "black"
            ax.text(j, i, f"{sim_matrix[i, j]:.2f}",
                    ha="center", va="center", color=color, fontsize=9)

    fig.colorbar(im, ax=ax, label="Cosine Similarity")
    plt.tight_layout()
    plt.show()


# 完全一致の場合
plot_similarity_matrix(
    result1["similarity_matrix"],
    ref_tokens, cand1_tokens,
    "Similarity Matrix: Exact Match"
)

# 同義語の場合
plot_similarity_matrix(
    result2["similarity_matrix"],
    ref_tokens, cand2_tokens,
    "Similarity Matrix: Synonyms"
)

import numpy as np
from collections import Counter
import matplotlib.pyplot as plt


def compare_metrics(reference, candidate, word_embeddings, label=""):
    """3メトリクスを計算して結果を返す"""
    ref_tokens = reference.split()
    cand_tokens = candidate.split()

    # BLEU（max_n=2に制限: 短い文ではn=4だと0になりやすい）
    bleu = compute_bleu(cand_tokens, [ref_tokens], max_n=2,
                        weights=[0.5, 0.5])

    # ROUGE-1, ROUGE-2, ROUGE-L
    r1 = rouge_n(cand_tokens, ref_tokens, n=1)
    r2 = rouge_n(cand_tokens, ref_tokens, n=2)
    rl = rouge_l(cand_tokens, ref_tokens)

    # BERTScore（手動埋め込み）
    ref_emb = np.array([word_embeddings.get(
        t, np.random.randn(64)) for t in ref_tokens])
    cand_emb = np.array([word_embeddings.get(
        t, np.random.randn(64)) for t in cand_tokens])
    bs = bert_score(ref_emb, cand_emb)

    return {
        "label": label,
        "BLEU-2": bleu,
        "ROUGE-1 F1": r1["f1"],
        "ROUGE-2 F1": r2["f1"],
        "ROUGE-L F1": rl["f1"],
        "BERTScore F1": bs["f1"]
    }


# 比較用の単語埋め込み辞書（前のセクションで定義済み）
np.random.seed(42)
d = 64

# 意味カテゴリの基底ベクトル
bases = {k: np.random.randn(d) for k in [
    "article", "animal", "action_sit", "action_run",
    "prep", "surface", "speed", "field"
]}

emb = {
    "the": bases["article"] + 0.1 * np.random.randn(d),
    "a": bases["article"] + 0.1 * np.random.randn(d),
    "cat": bases["animal"] + 0.1 * np.random.randn(d),
    "feline": bases["animal"] + 0.12 * np.random.randn(d),
    "dog": bases["animal"] + 0.2 * np.random.randn(d),
    "sat": bases["action_sit"] + 0.1 * np.random.randn(d),
    "sits": bases["action_sit"] + 0.12 * np.random.randn(d),
    "rested": bases["action_sit"] + 0.15 * np.random.randn(d),
    "ran": bases["action_run"] + 0.1 * np.random.randn(d),
    "sprinted": bases["action_run"] + 0.12 * np.random.randn(d),
    "on": bases["prep"] + 0.1 * np.random.randn(d),
    "upon": bases["prep"] + 0.12 * np.random.randn(d),
    "across": bases["prep"] + 0.2 * np.random.randn(d),
    "over": bases["prep"] + 0.18 * np.random.randn(d),
    "mat": bases["surface"] + 0.1 * np.random.randn(d),
    "rug": bases["surface"] + 0.15 * np.random.randn(d),
    "quickly": bases["speed"] + 0.1 * np.random.randn(d),
    "rapidly": bases["speed"] + 0.12 * np.random.randn(d),
    "field": bases["field"] + 0.1 * np.random.randn(d),
    "meadow": bases["field"] + 0.15 * np.random.randn(d),
}

# テストケースの定義
reference = "the cat sat on the mat"
test_cases = [
    ("the cat sat on the mat", "完全一致"),
    ("a feline rested upon a rug", "同義語置換"),
    ("the mat on sat cat the", "語順反転"),
    ("the cat", "情報欠落（短文）"),
    ("the cat sat on the mat and then the dog also sat", "冗長"),
    ("the dog ran across the field", "意味が異なる"),
]

# 全ケースを計算
results = []
for cand, label in test_cases:
    r = compare_metrics(reference, cand, emb, label)
    results.append(r)

# 結果テーブルの表示
header = f"{'ケース':<18} {'BLEU-2':>8} {'R1-F1':>8} {'R2-F1':>8} {'RL-F1':>8} {'BS-F1':>8}"
print(f"参照文: {reference}\n")
print(header)
print("-" * len(header))
for r in results:
    print(f"{r['label']:<18} "
          f"{r['BLEU-2']:>8.4f} "
          f"{r['ROUGE-1 F1']:>8.4f} "
          f"{r['ROUGE-2 F1']:>8.4f} "
          f"{r['ROUGE-L F1']:>8.4f} "
          f"{r['BERTScore F1']:>8.4f}")

import numpy as np
import matplotlib.pyplot as plt


def plot_radar_comparison(results, metrics_keys, title):
    """複数ケースのメトリクスをレーダーチャートで比較する"""
    n_metrics = len(metrics_keys)
    angles = np.linspace(0, 2 * np.pi, n_metrics, endpoint=False).tolist()
    angles += angles[:1]  # 閉じるために先頭を末尾に追加

    fig, axes = plt.subplots(2, 3, figsize=(15, 10),
                              subplot_kw=dict(polar=True))
    axes = axes.flatten()
    colors = ["#2ecc71", "#3498db", "#e74c3c",
              "#f39c12", "#9b59b6", "#1abc9c"]

    for idx, (r, ax) in enumerate(zip(results, axes)):
        values = [r[k] for k in metrics_keys]
        values += values[:1]

        ax.fill(angles, values, alpha=0.25, color=colors[idx])
        ax.plot(angles, values, "o-", linewidth=2, color=colors[idx])
        ax.set_xticks(angles[:-1])
        ax.set_xticklabels(
            ["BLEU-2", "R1-F1", "R2-F1", "RL-F1", "BS-F1"],
            fontsize=9
        )
        ax.set_ylim(0, 1.05)
        ax.set_title(r["label"], fontsize=11, pad=15)

    plt.suptitle(title, fontsize=14, y=1.02)
    plt.tight_layout()
    plt.show()


metrics_keys = [
    "BLEU-2", "ROUGE-1 F1", "ROUGE-2 F1",
    "ROUGE-L F1", "BERTScore F1"
]
plot_radar_comparison(
    results, metrics_keys,
    "Comparison of Evaluation Metrics Across Different Scenarios"
)