Hugging Face Tokenizers実践ガイド — 高速トークナイザーの使い方とカスタム訓練

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.normalizers import NFD, Lowercase, StripAccents, Sequence as NormalizerSequence

# パイプラインの各ステージを個別に確認
# 1. Normalizer
normalizer = NormalizerSequence([NFD(), Lowercase(), StripAccents()])
text = "Héllo Wörld! Café"
normalized = normalizer.normalize_str(text)
print(f"入力:     {text}")
print(f"正規化後: {normalized}")

# 2. Pre-tokenizer
pre_tokenizer = Whitespace()
pre_tokenized = pre_tokenizer.pre_tokenize_str(normalized)
print(f"事前トークン化: {pre_tokenized}")

# 3. Model以降は訓練済みトークナイザーで確認
from tokenizers import pre_tokenizers
from tokenizers.pre_tokenizers import ByteLevel

# ByteLevel Pre-tokenizerの動作確認
byte_level_pre = ByteLevel(add_prefix_space=False)
sample = "Hello world!"
byte_result = byte_level_pre.pre_tokenize_str(sample)
print(f"\nByteLevel事前トークン化:")
print(f"入力: {sample}")
print(f"結果: {byte_result}")

pip install tokenizers datasets matplotlib

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import ByteLevel
from tokenizers.normalizers import NFC, Lowercase, Sequence as NormalizerSequence
from tokenizers.processors import ByteLevel as ByteLevelProcessor
from tokenizers.decoders import ByteLevel as ByteLevelDecoder

# 1. BPEモデルで空のトークナイザーを作成
tokenizer = Tokenizer(BPE(unk_token="<unk>"))

# 2. Normalizerの設定: Unicode NFC正規化 + 小文字化
tokenizer.normalizer = NormalizerSequence([NFC(), Lowercase()])

# 3. Pre-tokenizerの設定: ByteLevel（GPT-2方式）
tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=False)

# 4. Post-processorの設定
tokenizer.post_processor = ByteLevelProcessor(trim_offsets=False)

# 5. Decoderの設定
tokenizer.decoder = ByteLevelDecoder()

# トレーナーの設定
trainer = BpeTrainer(
    vocab_size=8000,           # 語彙サイズ
    min_frequency=2,           # 最低出現頻度
    special_tokens=["<unk>", "<s>", "</s>", "<pad>", "<mask>"],
    show_progress=True
)

# 訓練データの準備（サンプルテキスト）
training_texts = [
    "自然言語処理は人工知能の重要な分野です。",
    "トークナイゼーションはテキストをトークンに分割する処理です。",
    "深層学習モデルはトークンIDの列を入力として受け取ります。",
    "BPEは最も広く使われているサブワードトークナイゼーション手法です。",
    "Transformerモデルの性能はトークナイザーの品質に大きく依存します。",
    "機械学習では大量のテキストデータを効率的に前処理する必要があります。",
    "サブワード分割は未知語の問題を効果的に解決します。",
    "ニューラルネットワークは言語の構造を自動的に学習できます。",
    "事前学習済みモデルをファインチューニングすることで高精度を達成できます。",
    "Hugging Faceのライブラリは自然言語処理の民主化に貢献しています。",
] * 100  # 訓練データを増やす

# イテレータから訓練
tokenizer.train_from_iterator(training_texts, trainer=trainer)

# 結果の確認
print(f"語彙サイズ: {tokenizer.get_vocab_size()}")

# テスト
test_text = "自然言語処理のトークナイゼーション"
output = tokenizer.encode(test_text)
print(f"\n入力テキスト: {test_text}")
print(f"トークン:     {output.tokens}")
print(f"トークンID:   {output.ids}")

import matplotlib.pyplot as plt
import numpy as np
from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import Whitespace

# 異なる語彙サイズでの分割結果を比較
vocab_sizes = [300, 500, 1000, 2000, 4000, 8000]
test_sentence = "The tokenization process efficiently handles unknown words through subword splitting"

# 英語のサンプルコーパス
english_corpus = [
    "The transformer model uses self-attention mechanisms for sequence processing.",
    "Natural language processing has been revolutionized by deep learning approaches.",
    "Tokenization is a fundamental preprocessing step in modern NLP pipelines.",
    "Subword tokenization effectively handles out-of-vocabulary words.",
    "The attention mechanism allows the model to focus on relevant parts of the input.",
    "Pre-trained language models can be fine-tuned for various downstream tasks.",
    "Byte pair encoding is one of the most popular subword tokenization algorithms.",
    "The vocabulary size affects both model performance and computational efficiency.",
    "Machine learning models require numerical inputs rather than raw text strings.",
    "Transfer learning enables knowledge sharing across different NLP tasks.",
] * 200

avg_tokens_list = []
vocab_actual = []

for vs in vocab_sizes:
    tok = Tokenizer(BPE(unk_token="<unk>"))
    tok.pre_tokenizer = Whitespace()
    trainer = BpeTrainer(
        vocab_size=vs,
        min_frequency=2,
        special_tokens=["<unk>", "<pad>"]
    )
    tok.train_from_iterator(english_corpus, trainer=trainer)

    encoded = tok.encode(test_sentence)
    avg_tokens_list.append(len(encoded.tokens))
    vocab_actual.append(tok.get_vocab_size())

# 可視化
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))

# 左: 語彙サイズ vs トークン数
ax1.plot(vocab_actual, avg_tokens_list, 'o-', color='#00d4ff', linewidth=2, markersize=8)
ax1.set_xlabel('Vocabulary Size', fontsize=12)
ax1.set_ylabel('Number of Tokens', fontsize=12)
ax1.set_title('Effect of Vocabulary Size on Token Count', fontsize=13)
ax1.set_xscale('log')
ax1.grid(True, alpha=0.3)
ax1.set_facecolor('#1a1a2e')
fig.patch.set_facecolor('#0d0d1a')
ax1.tick_params(colors='white')
ax1.xaxis.label.set_color('white')
ax1.yaxis.label.set_color('white')
ax1.title.set_color('white')
for spine in ax1.spines.values():
    spine.set_color('#333')

# 右: 各語彙サイズでのトークン分割結果
token_texts = []
for vs in [300, 1000, 8000]:
    tok = Tokenizer(BPE(unk_token="<unk>"))
    tok.pre_tokenizer = Whitespace()
    trainer = BpeTrainer(vocab_size=vs, min_frequency=2, special_tokens=["<unk>", "<pad>"])
    tok.train_from_iterator(english_corpus, trainer=trainer)

    short_text = "tokenization"
    encoded = tok.encode(short_text)
    token_texts.append((vs, encoded.tokens))

y_positions = [0.7, 0.5, 0.3]
ax2.set_xlim(0, 1)
ax2.set_ylim(0, 1)
ax2.set_facecolor('#1a1a2e')

for (vs, tokens), y_pos in zip(token_texts, y_positions):
    label = f"vocab={vs}: "
    token_str = " | ".join(tokens)
    ax2.text(0.05, y_pos, label, fontsize=11, color='#ffaa00',
             verticalalignment='center', fontfamily='monospace')
    ax2.text(0.35, y_pos, token_str, fontsize=11, color='#00d4ff',
             verticalalignment='center', fontfamily='monospace')

ax2.set_title('Subword Splits of "tokenization"', fontsize=13, color='white')
ax2.set_xticks([])
ax2.set_yticks([])
for spine in ax2.spines.values():
    spine.set_color('#333')

plt.tight_layout()
plt.savefig('bpe_vocab_analysis.png', dpi=150, bbox_inches='tight',
            facecolor='#0d0d1a')
plt.show()

from tokenizers import Tokenizer
from tokenizers.models import WordPiece
from tokenizers.trainers import WordPieceTrainer
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.normalizers import BertNormalizer

# WordPieceモデルで空のトークナイザーを作成
wp_tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))

# BERTと同じ正規化を使用
wp_tokenizer.normalizer = BertNormalizer(
    clean_text=True,
    handle_chinese_chars=True,
    strip_accents=True,
    lowercase=True
)

# 空白で事前トークン化
wp_tokenizer.pre_tokenizer = Whitespace()

# トレーナーの設定
wp_trainer = WordPieceTrainer(
    vocab_size=8000,
    min_frequency=2,
    special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],
    continuing_subword_prefix="##"
)

# 訓練（BPEと同じコーパスを使用）
english_corpus = [
    "The transformer model uses self-attention mechanisms for sequence processing.",
    "Natural language processing has been revolutionized by deep learning approaches.",
    "Tokenization is a fundamental preprocessing step in modern NLP pipelines.",
    "Subword tokenization effectively handles out-of-vocabulary words.",
    "The attention mechanism allows the model to focus on relevant parts of the input.",
    "Pre-trained language models can be fine-tuned for various downstream tasks.",
    "Byte pair encoding is one of the most popular subword tokenization algorithms.",
    "The vocabulary size affects both model performance and computational efficiency.",
    "Machine learning models require numerical inputs rather than raw text strings.",
    "Transfer learning enables knowledge sharing across different NLP tasks.",
] * 200

wp_tokenizer.train_from_iterator(english_corpus, wp_trainer)

# テスト
test_text = "Tokenization preprocessing is fundamentally important"
wp_output = wp_tokenizer.encode(test_text)

print(f"語彙サイズ: {wp_tokenizer.get_vocab_size()}")
print(f"\n入力:     {test_text}")
print(f"トークン: {wp_output.tokens}")
print(f"ID:       {wp_output.ids}")

# ##プレフィックスの確認
print("\n--- WordPieceの##プレフィックス ---")
for token in wp_output.tokens:
    if token.startswith("##"):
        print(f"  {token} ← 単語の途中のサブワード")
    else:
        print(f"  {token} ← 単語の先頭")

from tokenizers import Tokenizer
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer
from tokenizers.pre_tokenizers import Metaspace
from tokenizers.normalizers import NFC

# Unigramモデルで空のトークナイザーを作成
uni_tokenizer = Tokenizer(Unigram())

# 正規化
uni_tokenizer.normalizer = NFC()

# SentencePiece風のPre-tokenizer（▁で単語境界を表現）
uni_tokenizer.pre_tokenizer = Metaspace()

# トレーナーの設定
uni_trainer = UnigramTrainer(
    vocab_size=8000,
    special_tokens=["<unk>", "<s>", "</s>", "<pad>"],
    unk_token="<unk>"
)

# 訓練
english_corpus_extended = [
    "The transformer model uses self-attention mechanisms for sequence processing.",
    "Natural language processing has been revolutionized by deep learning approaches.",
    "Tokenization is a fundamental preprocessing step in modern NLP pipelines.",
    "Subword tokenization effectively handles out-of-vocabulary words.",
    "The attention mechanism allows the model to focus on relevant parts of the input.",
    "Pre-trained language models can be fine-tuned for various downstream tasks.",
    "Byte pair encoding is one of the most popular subword tokenization algorithms.",
    "The vocabulary size affects both model performance and computational efficiency.",
    "Machine learning models require numerical inputs rather than raw text strings.",
    "Transfer learning enables knowledge sharing across different NLP tasks.",
    "Recurrent neural networks process sequential data one step at a time.",
    "Convolutional layers extract local features from input representations.",
    "Gradient descent optimizes model parameters by minimizing the loss function.",
    "Regularization techniques prevent overfitting by constraining model complexity.",
    "Data augmentation increases training set diversity without collecting new data.",
] * 200

uni_tokenizer.train_from_iterator(english_corpus_extended, uni_trainer)

# テスト
test_text = "Tokenization preprocessing is fundamentally important"
uni_output = uni_tokenizer.encode(test_text)

print(f"語彙サイズ: {uni_tokenizer.get_vocab_size()}")
print(f"\n入力:     {test_text}")
print(f"トークン: {uni_output.tokens}")
print(f"ID:       {uni_output.ids}")

import matplotlib.pyplot as plt
import matplotlib
matplotlib.rcParams['font.family'] = ['Arial', 'Helvetica', 'DejaVu Sans']

# 3つのトークナイザーで同じテキストをエンコード
test_sentences = [
    "tokenization",
    "preprocessing",
    "fundamentally",
    "computational",
    "revolutionized"
]

fig, ax = plt.subplots(figsize=(16, 8))
ax.set_facecolor('#1a1a2e')
fig.patch.set_facecolor('#0d0d1a')

models = {
    "BPE": None,     # 前のセクションで構築済みと仮定
    "WordPiece": wp_tokenizer,
    "Unigram": uni_tokenizer
}

# BPEトークナイザーを再構築
from tokenizers import Tokenizer as Tok
from tokenizers.models import BPE as BPEModel
from tokenizers.trainers import BpeTrainer as BPETr
from tokenizers.pre_tokenizers import Whitespace as WS

bpe_tok = Tok(BPEModel(unk_token="<unk>"))
bpe_tok.pre_tokenizer = WS()
bpe_tr = BPETr(vocab_size=8000, min_frequency=2, special_tokens=["<unk>", "<pad>"])
bpe_tok.train_from_iterator(english_corpus, bpe_tr)

tokenizers_dict = {
    "BPE": bpe_tok,
    "WordPiece": wp_tokenizer,
    "Unigram": uni_tokenizer
}

colors = {"BPE": "#00d4ff", "WordPiece": "#ff6b6b", "Unigram": "#51cf66"}

y_base = len(test_sentences) - 1
row_height = 1.0

for i, word in enumerate(test_sentences):
    y = (len(test_sentences) - 1 - i) * (row_height * 3.5) + 1.5

    # 単語ラベル
    ax.text(0.5, y + row_height * 2, f'"{word}"', fontsize=13,
            color='white', fontweight='bold', fontfamily='monospace')

    for j, (name, tok) in enumerate(tokenizers_dict.items()):
        y_row = y + (2 - j) * row_height * 0.7
        encoded = tok.encode(word)
        tokens = encoded.tokens

        # モデル名
        ax.text(0.5, y_row, f"{name}:", fontsize=10, color=colors[name],
                fontweight='bold')

        # トークン表示
        token_str = " | ".join(tokens)
        ax.text(4.0, y_row, token_str, fontsize=10, color=colors[name],
                fontfamily='monospace')

        # トークン数
        ax.text(15.0, y_row, f"({len(tokens)} tokens)", fontsize=9,
                color='#888888')

ax.set_xlim(0, 17)
ax.set_ylim(0, len(test_sentences) * 3.5 + 1)
ax.set_xticks([])
ax.set_yticks([])
ax.set_title('Comparison of BPE, WordPiece, and Unigram Tokenization',
             fontsize=14, color='white', pad=20)
for spine in ax.spines.values():
    spine.set_color('#333')

plt.tight_layout()
plt.savefig('tokenizer_comparison.png', dpi=150, bbox_inches='tight',
            facecolor='#0d0d1a')
plt.show()

from tokenizers import Tokenizer
from tokenizers.models import WordPiece
from tokenizers.trainers import WordPieceTrainer
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.processors import TemplateProcessing

# WordPieceトークナイザーを構築
tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
tokenizer.pre_tokenizer = Whitespace()

trainer = WordPieceTrainer(
    vocab_size=8000,
    special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],
    continuing_subword_prefix="##"
)

corpus = [
    "The quick brown fox jumps over the lazy dog.",
    "Machine learning models process text as numerical tokens.",
    "Attention mechanisms enable transformers to capture long-range dependencies.",
    "Fine-tuning pre-trained models yields state-of-the-art results.",
    "Natural language understanding requires both syntax and semantics.",
] * 500

tokenizer.train_from_iterator(corpus, trainer)

# BERT風テンプレートの設定
# 単一文: [CLS] text [SEP]
# 文ペア: [CLS] text_A [SEP] text_B [SEP]
tokenizer.post_processor = TemplateProcessing(
    single="[CLS] $A [SEP]",
    pair="[CLS] $A [SEP] $B:1 [SEP]:1",
    special_tokens=[
        ("[CLS]", tokenizer.token_to_id("[CLS]")),
        ("[SEP]", tokenizer.token_to_id("[SEP]")),
    ],
)

# 単一文のエンコード
text = "Attention is all you need"
output = tokenizer.encode(text)
print("--- 単一文エンコード ---")
print(f"入力:       {text}")
print(f"トークン:   {output.tokens}")
print(f"ID:         {output.ids}")
print(f"Type IDs:   {output.type_ids}")

# 文ペアのエンコード
text_a = "What is attention"
text_b = "Attention is a mechanism"
output_pair = tokenizer.encode(text_a, text_b)
print("\n--- 文ペアエンコード ---")
print(f"テキストA:  {text_a}")
print(f"テキストB:  {text_b}")
print(f"トークン:   {output_pair.tokens}")
print(f"ID:         {output_pair.ids}")
print(f"Type IDs:   {output_pair.type_ids}")
print(f"Attn Mask:  {output_pair.attention_mask}")

from tokenizers import Tokenizer
from tokenizers.models import WordPiece
from tokenizers.trainers import WordPieceTrainer
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.processors import TemplateProcessing

# 前のセクションで構築したトークナイザーを想定
# パディングの有効化
tokenizer.enable_padding(
    pad_id=tokenizer.token_to_id("[PAD]"),
    pad_token="[PAD]",
    length=15  # 固定長にする場合（Noneで最大長に合わせる）
)

# トランケーションの有効化
tokenizer.enable_truncation(max_length=15)

# バッチエンコード
texts = [
    "Short text",
    "A somewhat longer piece of text for testing",
    "Medium length text"
]

outputs = tokenizer.encode_batch(texts)

print("--- パディング・トランケーション ---")
for text, output in zip(texts, outputs):
    print(f"\n入力: {text}")
    print(f"トークン:     {output.tokens}")
    print(f"Attn Mask:    {output.attention_mask}")
    # パディング部分を確認
    pad_count = output.attention_mask.count(0)
    print(f"パディング数: {pad_count}")

import time
import numpy as np
import matplotlib.pyplot as plt
from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import ByteLevel
from tokenizers.processors import ByteLevel as ByteLevelProcessor
from tokenizers.decoders import ByteLevel as ByteLevelDecoder

# BPEトークナイザーを訓練
tokenizer = Tokenizer(BPE(unk_token="<unk>"))
tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=False)
tokenizer.post_processor = ByteLevelProcessor(trim_offsets=False)
tokenizer.decoder = ByteLevelDecoder()

trainer = BpeTrainer(
    vocab_size=5000,
    min_frequency=2,
    special_tokens=["<unk>", "<s>", "</s>", "<pad>"]
)

# 訓練コーパス
corpus = [
    "Natural language processing enables computers to understand human language.",
    "Deep learning models have achieved remarkable results across various tasks.",
    "Transfer learning reduces the need for large task-specific datasets.",
    "The attention mechanism is the core component of the transformer architecture.",
    "Tokenization converts raw text into numerical representations for neural networks.",
] * 500

tokenizer.train_from_iterator(corpus, trainer)

# ベンチマーク用テキスト生成
np.random.seed(42)
sample_texts = [
    "This is a sample sentence for benchmarking tokenization speed and throughput. " * np.random.randint(1, 5)
    for _ in range(10000)
]

# 1. 逐次処理（1つずつエンコード）
start = time.perf_counter()
for text in sample_texts:
    tokenizer.encode(text)
sequential_time = time.perf_counter() - start

# 2. バッチ処理（全てまとめてエンコード）
start = time.perf_counter()
tokenizer.encode_batch(sample_texts)
batch_time = time.perf_counter() - start

print(f"テキスト数:   {len(sample_texts)}")
print(f"逐次処理:     {sequential_time:.3f}秒")
print(f"バッチ処理:   {batch_time:.3f}秒")
print(f"速度向上比:   {sequential_time / batch_time:.1f}x")

# バッチサイズごとの処理時間を計測
batch_sizes = [100, 500, 1000, 2000, 5000, 10000]
batch_times = []
throughputs = []

for bs in batch_sizes:
    texts = sample_texts[:bs]

    # 3回計測して中央値を取る
    times = []
    for _ in range(3):
        start = time.perf_counter()
        tokenizer.encode_batch(texts)
        elapsed = time.perf_counter() - start
        times.append(elapsed)

    median_time = np.median(times)
    batch_times.append(median_time)
    throughputs.append(bs / median_time)

# 可視化
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
fig.patch.set_facecolor('#0d0d1a')

# 左: バッチサイズ vs 処理時間
ax1.plot(batch_sizes, batch_times, 'o-', color='#00d4ff', linewidth=2, markersize=8)
ax1.set_xlabel('Batch Size', fontsize=12, color='white')
ax1.set_ylabel('Processing Time (sec)', fontsize=12, color='white')
ax1.set_title('Batch Size vs Processing Time', fontsize=13, color='white')
ax1.grid(True, alpha=0.3)
ax1.set_facecolor('#1a1a2e')
ax1.tick_params(colors='white')
for spine in ax1.spines.values():
    spine.set_color('#333')

# 右: バッチサイズ vs スループット
ax2.plot(batch_sizes, throughputs, 's-', color='#51cf66', linewidth=2, markersize=8)
ax2.set_xlabel('Batch Size', fontsize=12, color='white')
ax2.set_ylabel('Throughput (texts/sec)', fontsize=12, color='white')
ax2.set_title('Batch Size vs Throughput', fontsize=13, color='white')
ax2.grid(True, alpha=0.3)
ax2.set_facecolor('#1a1a2e')
ax2.tick_params(colors='white')
for spine in ax2.spines.values():
    spine.set_color('#333')

plt.tight_layout()
plt.savefig('batch_performance.png', dpi=150, bbox_inches='tight',
            facecolor='#0d0d1a')
plt.show()

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import ByteLevel
from tokenizers.processors import ByteLevel as ByteLevelProcessor
from tokenizers.decoders import ByteLevel as ByteLevelDecoder
import numpy as np

# トークナイザー構築（省略形）
tokenizer = Tokenizer(BPE(unk_token="<unk>"))
tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=False)
tokenizer.post_processor = ByteLevelProcessor(trim_offsets=False)
tokenizer.decoder = ByteLevelDecoder()

trainer = BpeTrainer(
    vocab_size=5000, min_frequency=2,
    special_tokens=["<unk>", "<s>", "</s>", "<pad>"]
)
corpus = ["This is sample text for training the tokenizer. " * 3] * 1000
tokenizer.train_from_iterator(corpus, trainer)

# 動的パディング: バッチ内の最大長に合わせる
tokenizer.enable_padding(
    pad_id=tokenizer.token_to_id("<pad>"),
    pad_token="<pad>",
    direction="right",
    length=None  # Noneでバッチ内最大長に自動調整
)

# テスト用バッチ
texts = [
    "Short.",
    "A medium length sentence for testing.",
    "This is the longest sentence in this batch and determines the padding length for all others."
]

outputs = tokenizer.encode_batch(texts)

print("--- 動的パディングの結果 ---")
for text, output in zip(texts, outputs):
    n_tokens = sum(output.attention_mask)
    n_pad = len(output.attention_mask) - n_tokens
    total = len(output.tokens)
    print(f"入力: {text[:40]:40s} | 有効トークン: {n_tokens:2d} | PAD: {n_pad:2d} | 合計: {total}")

# 固定長パディングとの比較
print("\n--- 固定長パディング（max_length=50）との比較 ---")
tokenizer.enable_padding(
    pad_id=tokenizer.token_to_id("<pad>"),
    pad_token="<pad>",
    length=50
)

outputs_fixed = tokenizer.encode_batch(texts)
total_tokens_fixed = sum(len(o.tokens) for o in outputs_fixed)

# 動的パディングに戻す
tokenizer.enable_padding(
    pad_id=tokenizer.token_to_id("<pad>"),
    pad_token="<pad>",
    length=None
)
outputs_dynamic = tokenizer.encode_batch(texts)
total_tokens_dynamic = sum(len(o.tokens) for o in outputs_dynamic)

print(f"固定長パディング: 合計 {total_tokens_fixed} トークン")
print(f"動的パディング:   合計 {total_tokens_dynamic} トークン")
print(f"削減率:           {(1 - total_tokens_dynamic / total_tokens_fixed) * 100:.1f}%")

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import ByteLevel
from tokenizers.processors import TemplateProcessing
from tokenizers.decoders import ByteLevel as ByteLevelDecoder
import json
import os

# BPEトークナイザーを構築・訓練
tokenizer = Tokenizer(BPE(unk_token="<unk>"))
tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=False)
tokenizer.decoder = ByteLevelDecoder()

trainer = BpeTrainer(
    vocab_size=5000, min_frequency=2,
    special_tokens=["<unk>", "<s>", "</s>", "<pad>", "<mask>"]
)

corpus = [
    "Natural language processing with transformers.",
    "Tokenization is the first step in any NLP pipeline.",
    "Custom tokenizers can be trained on domain-specific data.",
    "The Hugging Face ecosystem provides powerful tools for NLP.",
    "Pre-training and fine-tuning are the two phases of transfer learning.",
] * 500

tokenizer.train_from_iterator(corpus, trainer)

# テンプレートの設定
tokenizer.post_processor = TemplateProcessing(
    single="<s> $A </s>",
    pair="<s> $A  $B:1 </s>:1",
    special_tokens=[
        ("<s>", tokenizer.token_to_id("<s>")),
        ("</s>", tokenizer.token_to_id("</s>")),
    ],
)

# JSON形式で保存
save_dir = "./my_custom_tokenizer"
os.makedirs(save_dir, exist_ok=True)
tokenizer.save(os.path.join(save_dir, "tokenizer.json"))
print(f"トークナイザーを {save_dir}/tokenizer.json に保存しました")

# 再ロード
loaded_tokenizer = Tokenizer.from_file(os.path.join(save_dir, "tokenizer.json"))
test = loaded_tokenizer.encode("Test tokenization")
print(f"ロード後のテスト: {test.tokens}")

from tokenizers import Tokenizer
from transformers import PreTrainedTokenizerFast

# 保存済みのトークナイザーを読み込み
# （前のコードで保存した tokenizer.json を使用）
fast_tokenizer = PreTrainedTokenizerFast(
    tokenizer_file="./my_custom_tokenizer/tokenizer.json",
    bos_token="<s>",
    eos_token="</s>",
    unk_token="<unk>",
    pad_token="<pad>",
    mask_token="<mask>"
)

# Transformers形式で使用
encoded = fast_tokenizer(
    "Custom tokenizer with Transformers integration",
    padding="max_length",
    max_length=20,
    truncation=True,
    return_tensors="pt"  # PyTorchテンソルで返す
)

print("--- PreTrainedTokenizerFast出力 ---")
print(f"input_ids:      {encoded['input_ids']}")
print(f"attention_mask:  {encoded['attention_mask']}")
print(f"shape:           {encoded['input_ids'].shape}")

# デコード
decoded = fast_tokenizer.decode(encoded['input_ids'][0], skip_special_tokens=True)
print(f"\nデコード結果: {decoded}")

# バッチエンコード
texts = [
    "First sentence",
    "Second longer sentence for testing",
    "Third"
]
batch_encoded = fast_tokenizer(
    texts,
    padding=True,
    truncation=True,
    max_length=20,
    return_tensors="pt"
)
print(f"\nバッチ input_ids shape: {batch_encoded['input_ids'].shape}")

# save_pretrainedで保存（Transformers標準形式）
fast_tokenizer.save_pretrained("./my_custom_tokenizer_hf")
print("\nHugging Face形式で保存しました")

from transformers import AutoTokenizer
import matplotlib.pyplot as plt

# 事前学習済みトークナイザーのロード
bert_tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")

# 技術文書の比較テスト
technical_texts = [
    "The self-attention mechanism computes scaled dot-product attention.",
    "Backpropagation through time handles sequential dependencies.",
    "Convolutional neural networks extract hierarchical features.",
]

print("=" * 70)
print("トークナイザー比較: 事前学習済み vs カスタム")
print("=" * 70)

for text in technical_texts:
    print(f"\n入力: {text}")

    # BERT
    bert_tokens = bert_tokenizer.tokenize(text)
    print(f"  BERT ({len(bert_tokens)} tokens): {bert_tokens[:10]}...")

    # GPT-2
    gpt2_tokens = gpt2_tokenizer.tokenize(text)
    print(f"  GPT-2 ({len(gpt2_tokens)} tokens): {gpt2_tokens[:10]}...")

# トークン数の統計を可視化
fig, ax = plt.subplots(figsize=(10, 5))
fig.patch.set_facecolor('#0d0d1a')
ax.set_facecolor('#1a1a2e')

test_texts = [
    "Machine learning model training pipeline",
    "Natural language processing and understanding",
    "Transformer architecture with multi-head attention",
    "Gradient descent optimization algorithm",
    "Recurrent neural network sequence modeling",
]

bert_counts = [len(bert_tokenizer.tokenize(t)) for t in test_texts]
gpt2_counts = [len(gpt2_tokenizer.tokenize(t)) for t in test_texts]

x = range(len(test_texts))
width = 0.35

bars1 = ax.bar([i - width/2 for i in x], bert_counts, width,
               label='BERT (WordPiece)', color='#ff6b6b', alpha=0.8)
bars2 = ax.bar([i + width/2 for i in x], gpt2_counts, width,
               label='GPT-2 (BPE)', color='#00d4ff', alpha=0.8)

ax.set_xlabel('Test Sentences', fontsize=12, color='white')
ax.set_ylabel('Number of Tokens', fontsize=12, color='white')
ax.set_title('Token Count Comparison: BERT vs GPT-2', fontsize=13, color='white')
ax.set_xticks(list(x))
ax.set_xticklabels([f'Text {i+1}' for i in x], color='white')
ax.tick_params(colors='white')
ax.legend(fontsize=10)
for spine in ax.spines.values():
    spine.set_color('#333')

# 各バーの上にトークン数を表示
for bar in bars1:
    height = bar.get_height()
    ax.text(bar.get_x() + bar.get_width()/2., height + 0.1,
            f'{int(height)}', ha='center', va='bottom', color='white', fontsize=9)
for bar in bars2:
    height = bar.get_height()
    ax.text(bar.get_x() + bar.get_width()/2., height + 0.1,
            f'{int(height)}', ha='center', va='bottom', color='white', fontsize=9)

plt.tight_layout()
plt.savefig('tokenizer_count_comparison.png', dpi=150, bbox_inches='tight',
            facecolor='#0d0d1a')
plt.show()

from tokenizers import Tokenizer, pre_tokenizers, normalizers
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import (
    ByteLevel, Digits, Sequence as PreTokenizerSequence
)
from tokenizers.processors import TemplateProcessing
from tokenizers.decoders import ByteLevel as ByteLevelDecoder

# ソースコード特化トークナイザー
code_tokenizer = Tokenizer(BPE(unk_token="<unk>"))

# Pre-tokenizer: 数字を個別に分割 + ByteLevel
code_tokenizer.pre_tokenizer = PreTokenizerSequence([
    Digits(individual_digits=True),  # 数字を1桁ずつ分割
    ByteLevel(add_prefix_space=False)
])

code_tokenizer.decoder = ByteLevelDecoder()

# 特殊トークンにコード関連のものを追加
special_tokens = [
    "<unk>", "<s>", "</s>", "<pad>", "<mask>",
    "<|endoftext|>",  # テキスト終端
    "<|code|>",       # コード開始マーカー
    "<|indent|>",     # インデントマーカー
]

trainer = BpeTrainer(
    vocab_size=10000,
    min_frequency=2,
    special_tokens=special_tokens,
    show_progress=True
)

# Python風のコードコーパス
code_corpus = [
    "def train_model(data, epochs=10):",
    "    model = NeuralNetwork(layers=[64, 32, 16])",
    "    optimizer = Adam(lr=0.001)",
    "    for epoch in range(epochs):",
    "        loss = model.forward(data)",
    "        loss.backward()",
    "        optimizer.step()",
    "    return model",
    "",
    "class TransformerBlock(nn.Module):",
    "    def __init__(self, d_model=512, n_heads=8):",
    "        super().__init__()",
    "        self.attention = MultiHeadAttention(d_model, n_heads)",
    "        self.feed_forward = FeedForward(d_model)",
    "        self.norm1 = LayerNorm(d_model)",
    "        self.norm2 = LayerNorm(d_model)",
    "",
    "    def forward(self, x, mask=None):",
    "        attended = self.attention(x, x, x, mask)",
    "        x = self.norm1(x + attended)",
    "        fed_forward = self.feed_forward(x)",
    "        x = self.norm2(x + fed_forward)",
    "        return x",
    "",
    "import numpy as np",
    "import torch",
    "from torch import nn",
    "from transformers import AutoModel, AutoTokenizer",
    "",
    "def compute_attention(query, key, value, mask=None):",
    "    d_k = query.size(-1)",
    "    scores = torch.matmul(query, key.transpose(-2, -1))",
    "    scores = scores / np.sqrt(d_k)",
    "    if mask is not None:",
    "        scores = scores.masked_fill(mask == 0, -1e9)",
    "    weights = torch.softmax(scores, dim=-1)",
    "    return torch.matmul(weights, value)",
] * 200

code_tokenizer.train_from_iterator(code_corpus, trainer)

# テスト: コードスニペットのトークナイズ
test_code = "def forward(self, x, mask=None):"
output = code_tokenizer.encode(test_code)
print(f"語彙サイズ: {code_tokenizer.get_vocab_size()}")
print(f"\n入力: {test_code}")
print(f"トークン: {output.tokens}")
print(f"トークン数: {len(output.tokens)}")

# デコードの確認
decoded = code_tokenizer.decode(output.ids)
print(f"デコード: {decoded}")
print(f"完全復元: {decoded == test_code}")

# 別のコード例
test_code2 = "scores = torch.matmul(query, key.transpose(-2, -1))"
output2 = code_tokenizer.encode(test_code2)
print(f"\n入力: {test_code2}")
print(f"トークン: {output2.tokens}")
print(f"トークン数: {len(output2.tokens)}")

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import Whitespace

# シンプルなトークナイザーで確認
tokenizer = Tokenizer(BPE(unk_token="<unk>"))
tokenizer.pre_tokenizer = Whitespace()
trainer = BpeTrainer(
    vocab_size=5000, min_frequency=2,
    special_tokens=["<unk>", "<pad>"]
)
corpus = [
    "The transformer architecture was introduced in the paper Attention Is All You Need.",
    "BERT uses bidirectional self-attention for language understanding.",
    "Tokenization maps text to integer sequences for neural network processing.",
] * 500
tokenizer.train_from_iterator(corpus, trainer)

# オフセットの確認
text = "Tokenization is fundamental"
output = tokenizer.encode(text)

print("--- オフセットマッピング ---")
print(f"入力テキスト: {text}")
print(f"{'トークン':15s} {'ID':5s} {'開始':5s} {'終了':5s} {'元テキスト':15s}")
print("-" * 50)

for token, token_id, (start, end) in zip(output.tokens, output.ids, output.offsets):
    original = text[start:end]
    print(f"{token:15s} {token_id:<5d} {start:<5d} {end:<5d} {original:15s}")

from datasets import Dataset
from transformers import PreTrainedTokenizerFast
from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import ByteLevel
from tokenizers.decoders import ByteLevel as ByteLevelDecoder
from tokenizers.processors import TemplateProcessing
import time

# トークナイザーの構築
tokenizer = Tokenizer(BPE(unk_token="<unk>"))
tokenizer.pre_tokenizer = ByteLevel(add_prefix_space=False)
tokenizer.decoder = ByteLevelDecoder()

trainer = BpeTrainer(
    vocab_size=5000, min_frequency=2,
    special_tokens=["<unk>", "<s>", "</s>", "<pad>"]
)

corpus = [
    "The quick brown fox jumps over the lazy dog.",
    "Machine learning is transforming the world of natural language processing.",
    "Deep neural networks can learn complex patterns from data.",
    "Transfer learning enables efficient model adaptation to new tasks.",
    "Attention mechanisms are the foundation of modern language models.",
] * 500

tokenizer.train_from_iterator(corpus, trainer)

tokenizer.post_processor = TemplateProcessing(
    single="<s> $A </s>",
    special_tokens=[
        ("<s>", tokenizer.token_to_id("<s>")),
        ("</s>", tokenizer.token_to_id("</s>")),
    ],
)

# PreTrainedTokenizerFastでラップ
fast_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    bos_token="<s>",
    eos_token="</s>",
    unk_token="<unk>",
    pad_token="<pad>",
)

# サンプルデータセットの作成
texts = [
    f"This is sample document number {i}. It contains text for testing tokenization. "
    f"The transformer model processes this text efficiently."
    for i in range(5000)
]
dataset = Dataset.from_dict({"text": texts})

# トークナイズ関数
def tokenize_function(examples):
    return fast_tokenizer(
        examples["text"],
        padding="max_length",
        truncation=True,
        max_length=64
    )

# バッチ処理でトークナイズ
start = time.perf_counter()
tokenized_dataset = dataset.map(
    tokenize_function,
    batched=True,           # バッチ処理を有効化
    batch_size=1000,        # バッチサイズ
    num_proc=1,             # プロセス数（マルチプロセス可能）
    remove_columns=["text"] # 元のテキスト列を削除
)
elapsed = time.perf_counter() - start

print(f"データセットサイズ: {len(dataset)}")
print(f"トークナイズ時間:   {elapsed:.2f}秒")
print(f"スループット:       {len(dataset)/elapsed:.0f} texts/sec")
print(f"\nカラム: {tokenized_dataset.column_names}")
print(f"サンプル input_ids: {tokenized_dataset[0]['input_ids'][:10]}...")
print(f"サンプル長:         {len(tokenized_dataset[0]['input_ids'])}")