HuggingFace Tokenizers Library - Complete Learning Guide

# Basic installation
pip install tokenizers
 
# Verify installation
python -c "import tokenizers; print(tokenizers.__version__)"

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import Whitespace
 
# Step 1: Initialize with a model
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
 
# Step 2: Configure pre-tokenizer
tokenizer.pre_tokenizer = Whitespace()
 
# Step 3: Create trainer with special tokens
trainer = BpeTrainer(special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
 
# Step 4: Train on files
files = ["data/train.txt", "data/valid.txt"]
tokenizer.train(files, trainer)
 
# Step 5: Save for later use
tokenizer.save("my-tokenizer.json")
 
# Step 6: Use it!
output = tokenizer.encode("Hello, world!")
print(output.tokens)  # ['Hello', ',', 'world', '!']
print(output.ids)     # [245, 16, 689, 23]

from tokenizers import Tokenizer
 
# From Hugging Face Hub
tokenizer = Tokenizer.from_pretrained("bert-base-uncased")
 
# From local file
tokenizer = Tokenizer.from_file("my-tokenizer.json")
 
# Use it
output = tokenizer.encode("Quick tokenization test")

Raw Text → Normalization → Pre-Tokenization → Model → Post-Processing → Encoding

from tokenizers import Tokenizer
 
# Example text
text = "Hello, y'all! How are you 😁 ?"
 
# Step-by-step pipeline:
# 1. Normalization:     "Hello, y'all! How are you 😁 ?" 
#                       → (lowercase, remove accents, etc.)
#
# 2. Pre-Tokenization:  → ["Hello", ",", "y", "'", "all", "!", "How", "are", "you", "😁", "?"]
#                       (split on whitespace and punctuation)
#
# 3. Model:             → ["Hello", ",", "y", "'", "all", "!", "How", "are", "you", "[UNK]", "?"]
#                       (map to vocabulary)
#
# 4. Post-Processing:   → ["[CLS]", "Hello", ",", "y", "'", "all", "!", "How", "are", "you", "[UNK]", "?", "[SEP]"]
#                       (add special tokens)
 
output = tokenizer.encode(text)

from tokenizers import normalizers
from tokenizers.normalizers import NFD, StripAccents, Lowercase
 
# Single normalizer
tokenizer.normalizer = Lowercase()
 
# Combine multiple normalizers
tokenizer.normalizer = normalizers.Sequence([
    NFD(),              # Unicode normalization
    Lowercase(),        # Lowercase everything
    StripAccents()      # Remove accents
])
 
# Test it
result = tokenizer.normalizer.normalize_str("Héllò HÔW are Ü?")
print(result)  # "hello how are u?"

from tokenizers import pre_tokenizers
from tokenizers.pre_tokenizers import Whitespace, Digits
 
# Single pre-tokenizer
tokenizer.pre_tokenizer = Whitespace()
 
# Combine multiple pre-tokenizers
tokenizer.pre_tokenizer = pre_tokenizers.Sequence([
    Whitespace(),
    Digits(individual_digits=True)  # Split each digit separately
])
 
# Test it
result = tokenizer.pre_tokenizer.pre_tokenize_str("Call 911!")
print(result)
# [("Call", (0, 4)), ("9", (5, 6)), ("1", (6, 7)), ("1", (7, 8)), ("!", (8, 9))]

from tokenizers import Tokenizer
from tokenizers.models import BPE, WordPiece, Unigram, WordLevel
 
# BPE Model
bpe_tokenizer = Tokenizer(BPE(
    unk_token="[UNK]",
    continuing_subword_prefix="",
    end_of_word_suffix=""
))
 
# WordPiece Model (BERT-style)
wordpiece_tokenizer = Tokenizer(WordPiece(
    unk_token="[UNK]",
    max_input_chars_per_word=100
))
 
# Unigram Model
unigram_tokenizer = Tokenizer(Unigram())
 
# WordLevel Model
wordlevel_tokenizer = Tokenizer(WordLevel(
    unk_token="[UNK]"
))

from tokenizers.processors import TemplateProcessing
 
# BERT-style post-processing
tokenizer.post_processor = TemplateProcessing(
    single="[CLS] $A [SEP]",
    pair="[CLS] $A [SEP] $B:1 [SEP]:1",
    special_tokens=[
        ("[CLS]", tokenizer.token_to_id("[CLS]")),
        ("[SEP]", tokenizer.token_to_id("[SEP]")),
    ],
)
 
# Test single sequence
output = tokenizer.encode("Hello world")
print(output.tokens)
# ["[CLS]", "Hello", "world", "[SEP]"]
 
# Test pair of sequences
output = tokenizer.encode("Hello world", "How are you?")
print(output.tokens)
# ["[CLS]", "Hello", "world", "[SEP]", "How", "are", "you", "?", "[SEP]"]
print(output.type_ids)
# [0, 0, 0, 0, 1, 1, 1, 1, 1]

from tokenizers import decoders
 
# WordPiece Decoder (for BERT)
tokenizer.decoder = decoders.WordPiece(prefix="##")
 
# ByteLevel Decoder (for GPT-2)
tokenizer.decoder = decoders.ByteLevel()
 
# Metaspace Decoder (for SentencePiece)
tokenizer.decoder = decoders.Metaspace()
 
# Test decoding
output = tokenizer.encode("Hello, world!")
decoded = tokenizer.decode(output.ids)
print(decoded)  # "Hello, world!"

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.normalizers import NFD, Lowercase, StripAccents
from tokenizers import normalizers
 
# 1. Initialize
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
 
# 2. Configure normalization
tokenizer.normalizer = normalizers.Sequence([NFD(), Lowercase(), StripAccents()])
 
# 3. Configure pre-tokenization
tokenizer.pre_tokenizer = Whitespace()
 
# 4. Create trainer
trainer = BpeTrainer(
    vocab_size=30000,                # Final vocabulary size
    min_frequency=2,                 # Minimum frequency to merge
    special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],
    show_progress=True
)
 
# 5. Train
files = ["data/train.txt", "data/valid.txt", "data/test.txt"]
tokenizer.train(files, trainer)
 
# 6. Save
tokenizer.save("bpe-tokenizer.json")
 
print("Training complete! Vocabulary size:", tokenizer.get_vocab_size())

from tokenizers import Tokenizer
from tokenizers.models import WordPiece
from tokenizers.trainers import WordPieceTrainer
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.normalizers import BertNormalizer
from tokenizers.processors import TemplateProcessing
 
# 1. Initialize with WordPiece
tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
 
# 2. BERT-style normalization
tokenizer.normalizer = BertNormalizer(
    clean_text=True,
    handle_chinese_chars=True,
    strip_accents=True,
    lowercase=True
)
 
# 3. Pre-tokenization
tokenizer.pre_tokenizer = Whitespace()
 
# 4. Create trainer
trainer = WordPieceTrainer(
    vocab_size=30522,  # BERT vocabulary size
    special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],
    continuing_subword_prefix="##"
)
 
# 5. Train
files = ["data/corpus.txt"]
tokenizer.train(files, trainer)
 
# 6. Add post-processing
tokenizer.post_processor = TemplateProcessing(
    single="[CLS] $A [SEP]",
    pair="[CLS] $A [SEP] $B:1 [SEP]:1",
    special_tokens=[
        ("[CLS]", 1),
        ("[SEP]", 2),
    ],
)
 
# 7. Save
tokenizer.save("bert-wordpiece.json")

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
 
# Your data in memory
dataset = [
    "The quick brown fox jumps over the lazy dog.",
    "Machine learning is awesome!",
    "Tokenizers make NLP easier.",
    # ... more sentences
]
 
# Initialize tokenizer
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
trainer = BpeTrainer(special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
 
# Train from iterator
tokenizer.train_from_iterator(
    dataset,
    trainer=trainer,
    length=len(dataset)  # Optional: for progress bar
)
 
print(f"Trained on {len(dataset)} sequences")

from tokenizers import Tokenizer
 
# Load BERT tokenizer
bert_tokenizer = Tokenizer.from_pretrained("bert-base-uncased")
 
# Load GPT-2 tokenizer
gpt2_tokenizer = Tokenizer.from_pretrained("gpt2")
 
# Load with specific revision
tokenizer = Tokenizer.from_pretrained(
    "bert-base-uncased",
    revision="main",
    auth_token="your_token_here"  # For private repos
)

from tokenizers import BertWordPieceTokenizer
 
# Load BERT tokenizer from vocab file
tokenizer = BertWordPieceTokenizer(
    "bert-base-uncased-vocab.txt",
    lowercase=True
)
 
# Download vocab file first:
# wget https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt

output = tokenizer.encode("Hello, y'all! How are you 😁 ?")
 
# Token strings
print(output.tokens)
# ["Hello", ",", "y", "'", "all", "!", "How", "are", "you", "[UNK]", "?"]
 
# Token IDs
print(output.ids)
# [27253, 16, 93, 11, 5097, 5, 7961, 5112, 6218, 0, 35]
 
# Character offsets (alignment tracking)
print(output.offsets)
# [(0, 5), (5, 6), (7, 8), (8, 9), (9, 12), (12, 13), (14, 17), (18, 21), (22, 25), (26, 27), (28, 29)]
 
# Attention mask
print(output.attention_mask)
# [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
 
# Type IDs (for pairs)
print(output.type_ids)
# [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
 
# Special tokens mask
print(output.special_tokens_mask)
# [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
 
# Word IDs (which word each token belongs to)
print(output.word_ids)
# [0, 1, 2, 3, 3, 4, 5, 6, 7, 8, 9]

text = "Hello, y'all! How are you 😁 ?"
output = tokenizer.encode(text)
 
# Token to original characters
token_index = 9  # The "[UNK]" token
start, end = output.offsets[token_index]
print(f"Token '{output.tokens[token_index]}' came from: '{text[start:end]}'")
# Token '[UNK]' came from: '😁'
 
# Character to token
char_pos = 26  # Position of emoji
token_idx = output.char_to_token(char_pos)
print(f"Character at position {char_pos} is in token: {output.tokens[token_idx]}")
# Character at position 26 is in token: [UNK]
 
# Word to tokens
word_idx = 0  # First word
token_range = output.word_to_tokens(word_idx)
print(f"Word {word_idx} maps to tokens: {output.tokens[token_range[0]:token_range[1]]}")

# Encode multiple sequences
sequences = [
    "Hello, y'all!",
    "How are you?",
    "I'm fine, thank you!"
]
 
outputs = tokenizer.encode_batch(sequences)
 
# Each output is an Encoding object
for i, output in enumerate(outputs):
    print(f"Sequence {i}: {output.tokens}")
 
# Encode pairs of sequences
pairs = [
    ("Hello", "How are you?"),
    ("Good morning", "Nice to meet you")
]
 
outputs = tokenizer.encode_batch(pairs)

# Enable padding
tokenizer.enable_padding(
    direction="right",        # or "left"
    pad_id=3,                # ID of [PAD] token
    pad_token="[PAD]",
    length=None,             # Pad to longest in batch
    pad_to_multiple_of=8     # Optional: pad to multiple of 8
)
 
# Encode batch - shorter sequences will be padded
outputs = tokenizer.encode_batch(["Hello", "How are you doing today?"])
 
print(outputs[0].tokens)
# ["[CLS]", "Hello", "[SEP]", "[PAD]", "[PAD]", "[PAD]", "[PAD]"]
 
print(outputs[0].attention_mask)
# [1, 1, 1, 0, 0, 0, 0]
 
# Disable padding
tokenizer.no_padding()

# Enable truncation
tokenizer.enable_truncation(
    max_length=512,
    stride=0,                          # For overflowing tokens
    strategy="longest_first",          # or "only_first", "only_second"
    direction="right"                  # or "left"
)
 
# Encode long text - will be truncated
long_text = "word " * 1000
output = tokenizer.encode(long_text)
print(f"Truncated to {len(output.tokens)} tokens")
 
# Access overflowing tokens
if output.overflowing:
    print(f"Found {len(output.overflowing)} overflow chunks")
 
# Disable truncation
tokenizer.no_truncation()

# Add special tokens (never split)
tokenizer.add_special_tokens(["[CUSTOM]", "[SPECIAL]"])
 
# Add regular tokens
num_added = tokenizer.add_tokens(["tensorflow", "pytorch"])
print(f"Added {num_added} new tokens")
 
# Query token IDs
token_id = tokenizer.token_to_id("[MASK]")
print(f"[MASK] token ID: {token_id}")
 
# Query token from ID
token = tokenizer.id_to_token(0)
print(f"ID 0 maps to token: {token}")
 
# Get full vocabulary
vocab = tokenizer.get_vocab()
print(f"Vocabulary size: {len(vocab)}")

# Decode single sequence
ids = [1, 27253, 16, 93, 11, 5097, 2]
text = tokenizer.decode(ids)
print(text)
# "Hello , y ' all"
 
# Decode without special tokens
text = tokenizer.decode(ids, skip_special_tokens=True)
print(text)
# "Hello , y ' all"
 
# Decode batch
ids_batch = [[1, 27253, 2], [1, 7961, 2]]
texts = tokenizer.decode_batch(ids_batch)
print(texts)
# ["Hello", "How"]

from tokenizers import Tokenizer
from tokenizers.models import WordPiece
from tokenizers.trainers import WordPieceTrainer
from tokenizers.normalizers import BertNormalizer
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.processors import TemplateProcessing
from tokenizers.decoders import WordPiece as WordPieceDecoder
 
def create_bert_tokenizer(vocab_size=30522):
    """Create a complete BERT-style tokenizer"""
    
    # 1. Model
    tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
    
    # 2. Normalizer
    tokenizer.normalizer = BertNormalizer(
        clean_text=True,
        handle_chinese_chars=True,
        strip_accents=True,
        lowercase=True
    )
    
    # 3. Pre-tokenizer
    tokenizer.pre_tokenizer = Whitespace()
    
    # 4. Post-processor
    tokenizer.post_processor = TemplateProcessing(
        single="[CLS] $A [SEP]",
        pair="[CLS] $A [SEP] $B:1 [SEP]:1",
        special_tokens=[
            ("[CLS]", 1),
            ("[SEP]", 2),
        ],
    )
    
    # 5. Decoder
    tokenizer.decoder = WordPieceDecoder(prefix="##")
    
    # 6. Trainer
    trainer = WordPieceTrainer(
        vocab_size=vocab_size,
        special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"],
        continuing_subword_prefix="##"
    )
    
    return tokenizer, trainer
 
# Create and train
tokenizer, trainer = create_bert_tokenizer()
files = ["data/train.txt"]
tokenizer.train(files, trainer)
 
# Test
output = tokenizer.encode("Hello, world!", "How are you?")
print(f"Tokens: {output.tokens}")
print(f"IDs: {output.ids}")
print(f"Type IDs: {output.type_ids}")
 
# Decode
decoded = tokenizer.decode(output.ids)
print(f"Decoded: {decoded}")

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import ByteLevel as ByteLevelPreTokenizer
from tokenizers.decoders import ByteLevel as ByteLevelDecoder
 
def create_gpt2_tokenizer(vocab_size=50257):
    """Create a GPT-2 style tokenizer"""
    
    # 1. Model
    tokenizer = Tokenizer(BPE(unk_token="<|endoftext|>"))
    
    # 2. Pre-tokenizer (ByteLevel)
    tokenizer.pre_tokenizer = ByteLevelPreTokenizer(add_prefix_space=True)
    
    # 3. Decoder
    tokenizer.decoder = ByteLevelDecoder()
    
    # 4. Trainer
    trainer = BpeTrainer(
        vocab_size=vocab_size,
        special_tokens=["<|endoftext|>"],
        show_progress=True
    )
    
    return tokenizer, trainer
 
# Create and train
tokenizer, trainer = create_gpt2_tokenizer()
files = ["data/corpus.txt"]
tokenizer.train(files, trainer)
 
# Test
output = tokenizer.encode("Hello world! 🚀")
print(f"Tokens: {output.tokens}")
print(f"IDs: {output.ids}")

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.normalizers import Lowercase
 
def create_code_tokenizer():
    """Create a tokenizer optimized for code"""
    
    # 1. Model
    tokenizer = Tokenizer(BPE(unk_token="<UNK>"))
    
    # 2. Normalizer - keep case for code
    # tokenizer.normalizer = None  # Or minimal normalization
    
    # 3. Pre-tokenizer
    tokenizer.pre_tokenizer = Whitespace()
    
    # 4. Trainer with code-specific tokens
    trainer = BpeTrainer(
        vocab_size=20000,
        special_tokens=[
            "<UNK>",
            "<PAD>",
            "<BOS>",  # Beginning of sequence
            "<EOS>",  # End of sequence
            "<INDENT>",
            "<DEDENT>",
            "<NEWLINE>"
        ],
        show_progress=True
    )
    
    return tokenizer, trainer
 
# Create and train on code files
tokenizer, trainer = create_code_tokenizer()
code_files = ["data/code1.py", "data/code2.py", "data/code3.js"]
tokenizer.train(code_files, trainer)
 
# Test on code
code = """
def hello_world():
    print("Hello, world!")
    return True
"""
 
output = tokenizer.encode(code)
print(f"Code tokens: {output.tokens}")

from tokenizers import Tokenizer
from tokenizers.models import Unigram
from tokenizers.trainers import UnigramTrainer
from tokenizers.normalizers import NFKC
from tokenizers.pre_tokenizers import Metaspace
 
def create_multilingual_tokenizer():
    """Create a SentencePiece-style multilingual tokenizer"""
    
    # 1. Model (Unigram is often used for multilingual)
    tokenizer = Tokenizer(Unigram())
    
    # 2. Normalizer
    tokenizer.normalizer = NFKC()
    
    # 3. Pre-tokenizer (Metaspace for SentencePiece compatibility)
    tokenizer.pre_tokenizer = Metaspace()
    
    # 4. Trainer
    trainer = UnigramTrainer(
        vocab_size=32000,
        special_tokens=["<pad>", "<s>", "</s>", "<unk>"],
        unk_token="<unk>",
        show_progress=True
    )
    
    return tokenizer, trainer
 
# Create and train on multilingual data
tokenizer, trainer = create_multilingual_tokenizer()
multilingual_files = [
    "data/english.txt",
    "data/spanish.txt",
    "data/chinese.txt",
    "data/arabic.txt"
]
tokenizer.train(multilingual_files, trainer)
 
# Test on different languages
texts = [
    "Hello world",           # English
    "Hola mundo",            # Spanish
    "你好世界",              # Chinese
    "مرحبا بالعالم"          # Arabic
]
 
for text in texts:
    output = tokenizer.encode(text)
    print(f"{text}: {output.tokens}")

# Use BPE when:
# - You want a good balance of speed and quality
# - Working with English or Western languages
# - Need good handling of rare words
bpe_tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
 
# Use WordPiece when:
# - Building BERT-style models
# - Working with compound words (German, etc.)
# - Need compatibility with BERT
wordpiece_tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
 
# Use Unigram when:
# - Working with Asian languages (Japanese, Chinese, Korean)
# - Want probabilistic tokenization
# - Building T5 or XLNet style models
unigram_tokenizer = Tokenizer(Unigram())
 
# Use WordLevel when:
# - Have a small, controlled vocabulary
# - Building simple baseline models
# - Speed is critical, accuracy less so
wordlevel_tokenizer = Tokenizer(WordLevel(unk_token="[UNK]"))

# Small vocabulary (good for limited data)
trainer = BpeTrainer(vocab_size=8000)
 
# Medium vocabulary (general purpose)
trainer = BpeTrainer(vocab_size=30000)  # BERT standard
 
# Large vocabulary (for large diverse datasets)
trainer = BpeTrainer(vocab_size=50000)  # GPT-2
 
# Very large vocabulary (multilingual, massive datasets)
trainer = BpeTrainer(vocab_size=128000)  # XLM-RoBERTa

# Minimum data guidelines:
# - Small vocab (8K):     ~1M tokens  (100KB text)
# - Medium vocab (30K):   ~10M tokens (1MB text)
# - Large vocab (50K):    ~100M tokens (10MB text)
# - Very large (128K):    ~1B tokens (100MB text)
 
# Train from multiple sources
files = [
    "data/books.txt",        # 50MB
    "data/news.txt",         # 30MB
    "data/wikipedia.txt",    # 100MB
    "data/conversations.txt" # 20MB
]
tokenizer.train(files, trainer)

# Minimal special tokens (GPT-2 style)
special_tokens = ["<|endoftext|>"]
 
# Standard special tokens (BERT style)
special_tokens = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
 
# Extended special tokens (task-specific)
special_tokens = [
    "[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]",  # Standard
    "[USER]", "[ASSISTANT]",                        # Chat
    "[PYTHON]", "[JAVASCRIPT]", "[HTML]",           # Code
    "[MATH]", "[TABLE]", "[IMAGE]"                  # Multimodal
]
 
trainer = BpeTrainer(special_tokens=special_tokens)

# Save tokenizer (recommended)
tokenizer.save("tokenizer.json")
 
# Load tokenizer (fast)
tokenizer = Tokenizer.from_file("tokenizer.json")
 
# Save with pretty formatting (for debugging)
tokenizer.save("tokenizer.json", pretty=True)
 
# Export to string (for embedding in code)
json_str = tokenizer.to_str(pretty=False)
 
# Load from string
tokenizer = Tokenizer.from_str(json_str)

# Use batch encoding for multiple sequences
sequences = ["text1", "text2", "text3", ...]
outputs = tokenizer.encode_batch(sequences)  # Much faster than loop
 
# Enable parallel processing when training
trainer = BpeTrainer(
    vocab_size=30000,
    show_progress=True
)
 
# Use train_from_iterator for large datasets
def data_generator():
    with open("large_file.txt") as f:
        for line in f:
            yield line.strip()
 
tokenizer.train_from_iterator(
    data_generator(),
    trainer=trainer,
    length=1000000  # Approximate number of sequences
)

from tokenizers.tools import EncodingVisualizer
 
# Create visualizer
visualizer = EncodingVisualizer(tokenizer)
 
# Visualize tokenization
text = "Hello, how are you today?"
visualizer(text)  # Shows HTML visualization in notebook
 
# Check token alignments
output = tokenizer.encode(text)
for i, token in enumerate(output.tokens):
    start, end = output.offsets[i]
    print(f"Token '{token}' -> Original: '{text[start:end]}'")
 
# Inspect vocabulary
vocab = tokenizer.get_vocab()
print(f"Total tokens: {len(vocab)}")
print(f"First 10 tokens: {list(vocab.items())[:10]}")

def test_tokenizer(tokenizer, test_cases):
    """Test tokenizer on various cases"""
    
    print("=" * 50)
    print("TOKENIZER TESTING")
    print("=" * 50)
    
    for text in test_cases:
        output = tokenizer.encode(text)
        decoded = tokenizer.decode(output.ids)
        
        print(f"\nInput:   {text}")
        print(f"Tokens:  {output.tokens}")
        print(f"IDs:     {output.ids}")
        print(f"Decoded: {decoded}")
        print(f"Match:   {'✓' if text.replace(' ', '') == decoded.replace(' ', '') else '✗'}")
 
# Test cases
test_cases = [
    "Hello, world!",
    "The quick brown fox jumps over the lazy dog.",
    "Machine learning is awesome! 🚀",
    "Code: print('Hello')",
    "Email: test@example.com",
    "URL: https://huggingface.co",
    "Number: 12,345.67",
    "Unicode: café, naïve, résumé",
]
 
test_tokenizer(tokenizer, test_cases)