Tokenization Comparison Guide

# How BPE works:
# 1. Start with characters: ['h', 'e', 'l', 'l', 'o']
# 2. Find most frequent pair: 'l', 'l' -> merge to 'll'
# 3. Result: ['h', 'e', 'll', 'o']
# 4. Repeat until vocabulary size reached
 
from tokenizers import Tokenizer, models, trainers
 
# Create BPE tokenizer
tokenizer = Tokenizer(models.BPE())
trainer = trainers.BpeTrainer(vocab_size=5000)
 
# Train
tokenizer.train(["data.txt"], trainer)

# How WordPiece works:
# Similar to BPE but uses likelihood-based merging
# Prefers merges that maximize training data likelihood
 
from tokenizers import Tokenizer, models, trainers
 
tokenizer = Tokenizer(models.WordPiece(unk_token="[UNK]"))
trainer = trainers.WordPieceTrainer(
    vocab_size=30000,
    special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
)
 
tokenizer.train(["data.txt"], trainer)

# How Unigram works:
# Probabilistic approach - each subword has a probability
# Finds best segmentation based on probabilities
 
from tokenizers import Tokenizer, models, trainers
 
tokenizer = Tokenizer(models.Unigram())
trainer = trainers.UnigramTrainer(
    vocab_size=8000,
    special_tokens=["<unk>", "<s>", "</s>"]
)
 
tokenizer.train(["data.txt"], trainer)

# How SentencePiece works:
# Language-agnostic tokenizer that works directly on raw text
# Can use BPE or Unigram algorithm internally
# No need for pre-tokenization or language-specific rules
 
import sentencepiece as spm
 
# Train SentencePiece model (BPE mode)
spm.SentencePieceTrainer.train(
    input='data.txt',
    model_prefix='sentencepiece_bpe',
    vocab_size=8000,
    model_type='bpe',  # or 'unigram'
    character_coverage=0.9995,  # Cover 99.95% of characters
    num_threads=16
)
 
# Load and use
sp = spm.SentencePieceProcessor()
sp.load('sentencepiece_bpe.model')
 
# Encode
pieces = sp.encode_as_pieces('Hello, world!')
ids = sp.encode_as_ids('Hello, world!')
 
# Decode
text = sp.decode_ids([ids])

"""
Compare how different algorithms tokenize the same text
"""
from tokenizers import Tokenizer, models, trainers, pre_tokenizers, normalizers
 
def create_tokenizer(algorithm="bpe"):
    """Create tokenizer with specified algorithm"""
    if algorithm == "bpe":
        tokenizer = Tokenizer(models.BPE())
        trainer = trainers.BpeTrainer(vocab_size=1000)
    elif algorithm == "wordpiece":
        tokenizer = Tokenizer(models.WordPiece(unk_token="[UNK]"))
        trainer = trainers.WordPieceTrainer(vocab_size=1000)
    elif algorithm == "unigram":
        tokenizer = Tokenizer(models.Unigram())
        trainer = trainers.UnigramTrainer(vocab_size=1000)
    # Note: SentencePiece requires separate library (sentencepiece)
    # See SentencePiece section above for usage
    else:
        raise ValueError(f"Unknown algorithm: {algorithm}")
    
    # Add normalizer and pre-tokenizer
    tokenizer.normalizer = normalizers.NFKC()
    tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
    
    return tokenizer, trainer
 
# Train all three
training_data = [
    "The quick brown fox jumps over the lazy dog.",
    "Machine learning is fascinating and powerful.",
    "Tokenization breaks text into smaller pieces.",
] * 100  # Repeat for better training
 
algorithms = ["bpe", "wordpiece", "unigram"]
tokenizers = {}
 
for algo in algorithms:
    tokenizer, trainer = create_tokenizer(algo)
    tokenizer.train_from_iterator(training_data, trainer)
    tokenizers[algo] = tokenizer
 
# Compare on test text
test_text = "Machine learning tokenization example"
 
print("\n" + "="*60)
print("ALGORITHM COMPARISON")
print("="*60)
print(f"\nTest text: '{test_text}'")
print(f"Length: {len(test_text)} characters\n")
 
for algo, tokenizer in tokenizers.items():
    encoding = tokenizer.encode(test_text)
    print(f"\n{algo.upper()}:")
    print(f"  Tokens: {encoding.tokens}")
    print(f"  Count: {len(encoding.ids)} tokens")
    print(f"  IDs: {encoding.ids}")
    
    # Calculate compression
    compression = len(test_text) / len(encoding.ids)
    print(f"  Compression: {compression:.2f} chars/token")

# GPT-2, GPT-3, GPT-4
# Algorithm: BPE with byte-level encoding
 
import tiktoken
 
# GPT-4 / GPT-3.5-turbo
encoding = tiktoken.get_encoding("cl100k_base")
tokens = encoding.encode("Hello, world!")
 
# Vocabulary: ~100,000 tokens
# Trained on: Diverse internet text + code
# Best for: General purpose, code, English

# BERT, DistilBERT, RoBERTa
# Algorithm: WordPiece
 
from transformers import BertTokenizer
 
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
tokens = tokenizer.tokenize("Hello, world!")
 
# Vocabulary: ~30,000 tokens
# Special tokens: [CLS], [SEP], [MASK], [PAD], [UNK]
# Best for: Classification, NER, sentiment analysis

# T5, mT5
# Algorithm: SentencePiece Unigram
 
from transformers import T5Tokenizer
 
tokenizer = T5Tokenizer.from_pretrained('t5-base')
tokens = tokenizer.tokenize("Hello, world!")
 
# Vocabulary: 32,000 tokens
# Multilingual support (mT5)
# Best for: Seq2seq, translation, summarization
 
# Note: Uses SentencePiece library under the hood
# pip install sentencepiece

# LLaMA, LLaMA 2, Code LLaMA
# Algorithm: SentencePiece BPE
 
from transformers import LlamaTokenizer
 
tokenizer = LlamaTokenizer.from_pretrained('meta-llama/Llama-2-7b-hf')
tokens = tokenizer.tokenize("Hello, world!")
 
# Vocabulary: 32,000 tokens
# Trained on: Diverse multilingual data
# Best for: General LLM tasks
 
# Note: Uses SentencePiece library
# pip install sentencepiece protobuf

import time
from tokenizers import Tokenizer
 
# Test data
texts = ["Sample text " + str(i) for i in range(10000)]
 
# Benchmark function
def benchmark_tokenizer(tokenizer, texts, name):
    start = time.time()
    
    # Single encoding
    single_start = time.time()
    for text in texts[:1000]:
        tokenizer.encode(text)
    single_time = time.time() - single_start
    
    # Batch encoding
    batch_start = time.time()
    tokenizer.encode_batch(texts)
    batch_time = time.time() - batch_start
    
    print(f"\n{name}:")
    print(f"  Single: {single_time:.3f}s for 1K texts")
    print(f"  Batch:  {batch_time:.3f}s for 10K texts")
    print(f"  Speedup: {(single_time * 10) / batch_time:.1f}x")
    
    return {
        'single': single_time,
        'batch': batch_time,
        'speedup': (single_time * 10) / batch_time
    }
 
# Compare tokenizers
tokenizers_to_test = {
    'BPE': Tokenizer.from_pretrained('gpt2'),
    'WordPiece': Tokenizer.from_pretrained('bert-base-uncased'),
}
 
results = {}
for name, tokenizer in tokenizers_to_test.items():
    results[name] = benchmark_tokenizer(tokenizer, texts, name)
 
# Print comparison
print("\n" + "="*60)
print("PERFORMANCE COMPARISON")
print("="*60)
for name, result in results.items():
    print(f"{name}: {result['batch']:.3f}s (batch), {result['speedup']:.1f}x speedup")

import tracemalloc
 
def measure_memory(tokenizer, texts):
    """Measure peak memory usage"""
    tracemalloc.start()
    
    # Tokenize
    tokenizer.encode_batch(texts)
    
    current, peak = tracemalloc.get_traced_memory()
    tracemalloc.stop()
    
    return peak / 1024 / 1024  # Convert to MB
 
texts = ["Sample text " + str(i) for i in range(100000)]
 
for name, tokenizer in tokenizers_to_test.items():
    memory_mb = measure_memory(tokenizer, texts[:10000])
    print(f"{name}: {memory_mb:.2f} MB peak memory")

# English text
text = "The quick brown fox jumps over the lazy dog."
 
# GPT tokenizer (optimized for English)
gpt_tokens = tiktoken.get_encoding("cl100k_base").encode(text)
print(f"GPT: {len(gpt_tokens)} tokens")  # ~10 tokens
 
# Result: Very efficient for English

# Multilingual text
texts = {
    'English': "Hello, how are you?",
    'Chinese': "你好吗？",
    'Arabic': "كيف حالك؟",
    'Hindi': "आप कैसे हैं?",
    'Russian': "Как дела?"
}
 
# Compare tokenizers
from transformers import AutoTokenizer
 
# English-optimized
gpt_tokenizer = AutoTokenizer.from_pretrained('gpt2')
 
# Multilingual
xlmr_tokenizer = AutoTokenizer.from_pretrained('xlm-roberta-base')
 
print("\n" + "="*60)
print("LANGUAGE COMPARISON")
print("="*60)
 
for lang, text in texts.items():
    gpt_tokens = gpt_tokenizer.tokenize(text)
    xlmr_tokens = xlmr_tokenizer.tokenize(text)
    
    print(f"\n{lang}: '{text}'")
    print(f"  GPT (English-opt): {len(gpt_tokens)} tokens")
    print(f"  XLM-R (Multilingual): {len(xlmr_tokens)} tokens")
    
    # XLM-R is more efficient for non-English

from transformers import BertTokenizer
 
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
 
# Why BERT?
# - Optimized for classification tasks
# - [CLS] token for sequence representation
# - Fast inference
# - Good English performance

import tiktoken
 
encoding = tiktoken.get_encoding("cl100k_base")
 
# Why GPT?
# - Trained on code
# - Handles syntax well
# - Good at indentation
# - Mixed code + natural language

from transformers import T5Tokenizer
 
tokenizer = T5Tokenizer.from_pretrained('t5-base')
 
# Why T5/mT5?
# - Seq2seq architecture
# - Multilingual support
# - Good for translation tasks

from transformers import RobertaTokenizer
 
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
 
# Why RoBERTa?
# - Optimized for span extraction
# - Character-level alignment
# - Good context understanding

import tiktoken
 
encoding = tiktoken.get_encoding("cl100k_base")
 
# Why GPT?
# - Natural conversation flow
# - Good general knowledge
# - Handles context well

from transformers import XLMRobertaTokenizer
 
tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-base')
 
# Why XLM-R?
# - 100 languages supported
# - Efficient tokenization
# - Cross-lingual transfer

Choose Tokenizer:

Are you using a specific model (GPT-4, BERT, etc.)?
├─ YES → Use that model's tokenizer
└─ NO → Continue...

What's your primary task?
├─ Text Classification/NER → BERT WordPiece
├─ Code Generation → GPT BPE
├─ Translation → mT5 Unigram
├─ General LLM → GPT BPE
└─ Multilingual → XLM-R

What's your language?
├─ English only → GPT/BERT
├─ 2-5 languages → mBERT/XLM-R  
└─ Many languages → XLM-R/mT5

What's your priority?
├─ Speed → HuggingFace Tokenizers (Rust) or SentencePiece (C++)
├─ Accuracy → Use pretrained tokenizers
├─ Language-agnostic → SentencePiece
└─ Custom domain → Train your own

Note: Install SentencePiece for T5, LLaMA, ALBERT models:
  pip install sentencepiece

def evaluate_tokenizer_choice(tokenizer, test_texts, language="English"):
    """
    Evaluate if tokenizer is appropriate for your use case
    """
    total_tokens = 0
    total_chars = 0
    unknown_count = 0
    
    for text in test_texts:
        encoding = tokenizer.encode(text)
        total_tokens += len(encoding.ids)
        total_chars += len(text)
        
        # Check for unknown tokens
        if hasattr(tokenizer, 'unk_token'):
            unknown_count += encoding.tokens.count(tokenizer.unk_token)
    
    # Calculate metrics
    avg_compression = total_chars / total_tokens
    unknown_rate = unknown_count / total_tokens if total_tokens > 0 else 0
    
    print(f"\n{language} Evaluation:")
    print(f"  Compression: {avg_compression:.2f} chars/token")
    print(f"  Unknown rate: {unknown_rate:.1%}")
    
    # Recommendations
    if avg_compression < 2:
        print("  ⚠️  Low compression - consider multilingual tokenizer")
    elif avg_compression > 5:
        print("  ⚠️  Very high compression - may lose information")
    else:
        print("  ✅ Good compression ratio")
    
    if unknown_rate > 0.05:
        print("  ⚠️  High unknown rate - consider retraining")
    else:
        print("  ✅ Low unknown rate")
 
# Test on your data
my_texts = ["Your sample texts here..."]
evaluate_tokenizer_choice(tokenizer, my_texts)