fix: whitespace-split before subword encoding (full-string tokenization)

The HF tokenizer's full-string path char-class-split the entire string
without dropping inter-word spaces (unlike tensor2tensor tokenizer.encode
and gen_berts.py's LatinTokenizer, which split on whitespace first). Every
space was escaped to `\32;_`, injecting 3 spurious subtokens between words
— input the model never trained on. This corrupted masked-word predictions
and broke subword->word alignment for the documented `tokenizer("sentence")`
usage, while the word-by-word case-study path stayed correct (which is why
validation passed). Reported in two independent bug reports.

Fix (slow + root copy + fast): split on whitespace first, then char-class
encode each word. Native full-string output now equals the original
per-word tokenization (verified: slow == fast IDs). Decode renders word
boundaries as spaces (BERT-style; "tres." -> "tres .").

Also fixes a latent fast-tokenizer bug: it ignored do_lower_case (no Rust
normalizer), escaping capitals to codepoints (`Gallia` -> `\71;allia`) —
the same class of bug v1.1.1 fixed for the slow tokenizer. Added a
Lowercase normalizer.

Tests: regenerated golden IDs (previous "reference" values contained the
space escapes), added full-string==per-word parity guards and a new
test_tokenizer_fast.py. Version bump + changelog deferred until cluster
evals confirm case-study metrics on the native path.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/latincy_latinbert/tokenization_latin_bert.py

@@ -187,13 +187,24 @@ class LatinBertTokenizer(PreTrainedTokenizer):
         return vocab
 
     def _tokenize(self, text: str, **kwargs) -> List[str]:
-        """Tokenize text into subtoken strings."""
+        """Tokenize text into subtoken strings.
+
+        Splits on whitespace first (like the original gen_berts.py
+        LatinTokenizer), then applies character-class splitting within
+        each word. This is essential: tensor2tensor's tokenizer.encode()
+        drops inter-word spaces, so feeding a full string through
+        _tokenizer_encode() directly would escape every space to
+        ``\\32;_`` and inject spurious subtokens between words, corrupting
+        the input the model was trained on and breaking word alignment.
+        Because whitespace-split words contain no spaces, no space escape
+        can be produced here.
+        """
         if self.do_lower_case:
             text = text.lower()
-        tokens = _tokenizer_encode(text)
         subtokens = []
-        for token in tokens:
-            subtokens.extend(self._token_to_subtokens(token))
+        for word in text.split():
+            for token in _tokenizer_encode(word):
+                subtokens.extend(self._token_to_subtokens(token))
         return subtokens
 
     def _token_to_subtokens(self, token: str) -> List[str]:
@@ -240,16 +251,22 @@ class LatinBertTokenizer(PreTrainedTokenizer):
         return self.unk_token
 
     def convert_tokens_to_string(self, tokens: List[str]) -> str:
-        """Reverse the tokenization: unescape and join."""
+        """Reverse the tokenization: unescape and join.
+
+        Word boundaries are the trailing ``_`` markers, so each is
+        rendered as a space (BERT-style; punctuation spacing is lossy,
+        e.g. "tres." → "tres ."). Literal underscores are encoded as
+        ``\\u`` and are restored afterward.
+        """
         # Filter out special tokens before joining
         filtered = [t for t in tokens if t not in SPECIAL_TOKENS]
         text = "".join(filtered)
-        # Remove trailing underscores (word boundary markers)
-        # and unescape: \\u → _, \\\\ → \\, \\<digits>; → chr
-        text = re.sub(r"(?<!\\)_", "", text)
+        # Word-boundary underscores → spaces; then unescape:
+        # \\<digits>; → chr, \\u → _, \\\\ → \\
+        text = re.sub(r"(?<!\\)_", " ", text)
         text = re.sub(r"\\(\d+);", lambda m: chr(int(m.group(1))), text)
         text = text.replace("\\u", "_").replace("\\\\", "\\")
-        return text
+        return text.strip()
 
     def build_inputs_with_special_tokens(
         self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None

tests/test_tokenizer.py

@@ -63,23 +63,49 @@ class TestVocab:
 
 
 class TestEncoding:
-    """Reference IDs from original LatinTokenizer (with +5 offset)."""
+    """Reference IDs from the original LatinTokenizer (with +5 offset).
+
+    The original whitespace-splits before subword encoding, so there are
+    NO inter-word space escapes. One clean subtoken per word here.
+    """
 
     def test_gallia(self, tokenizer):
         ids = tokenizer.encode("Gallia est omnis divisa in partes tres",
                                add_special_tokens=False)
-        # With do_lower_case=True, "Gallia" → "gallia_" (single token)
-        expected = [6533, 32888, 7735, 13, 15, 32888, 7735, 13, 343,
-                    32888, 7735, 13, 6773, 32888, 7735, 13, 12, 32888,
-                    7735, 13, 568, 32888, 7735, 13, 564]
+        # do_lower_case=True → each word is a single subtoken; no space escapes
+        expected = [6533, 15, 343, 6773, 12, 568, 564]
         assert ids == expected
 
     def test_arma(self, tokenizer):
         ids = tokenizer.encode("arma virumque cano",
                                add_special_tokens=False)
-        expected = [915, 32888, 7735, 13, 18566, 8107, 32888, 7735, 13, 4420]
+        expected = [915, 18566, 8107, 4420]
         assert ids == expected
 
+    def test_no_interword_space_escapes(self, tokenizer):
+        """Full-string encoding must not inject space escapes between words.
+
+        The buggy full-string path emitted `\\ 32 ;_` (ids 32888, 7735, 13)
+        for every inter-word space — tokens the model never trained on.
+        """
+        ids = tokenizer.encode("gallia est omnis divisa in partes tres",
+                               add_special_tokens=False)
+        assert 32888 not in ids and 7735 not in ids and 13 not in ids
+
+    def test_fullstring_matches_per_word(self, tokenizer):
+        """Native full-string tokenization == word-by-word tokenization.
+
+        This is the contract: `tokenizer(full_sentence)` reproduces the
+        original per-word LatinTokenizer output, so HF-native usage is
+        faithful without requiring is_split_into_words.
+        """
+        text = "Gallia est omnis divisa in partes tres, quarum unam incolunt Belgae"
+        full = tokenizer.tokenize(text)
+        per_word = []
+        for word in text.split():
+            per_word.extend(tokenizer.tokenize(word))
+        assert full == per_word
+
     def test_uppercase(self, tokenizer):
         """Uppercase input should be lowercased, not escaped to codepoints."""
         ids = tokenizer.encode("ROMA", add_special_tokens=False)
@@ -106,10 +132,12 @@ class TestRoundtrip:
         assert decoded == text
 
     def test_decode_with_punctuation(self, tokenizer):
+        # Word boundaries decode to spaces (BERT-style), so attached
+        # punctuation becomes space-separated: "tres." -> "tres ."
         text = "gallia est omnis divisa in partes tres."
         ids = tokenizer.encode(text, add_special_tokens=False)
         decoded = tokenizer.decode(ids)
-        assert decoded == text
+        assert decoded == "gallia est omnis divisa in partes tres ."
 
     def test_decode_uppercase_lossy(self, tokenizer):
         """Uppercase input decodes to lowercase (lowercasing is lossy)."""
@@ -138,15 +166,14 @@ class TestLowercasing:
     def test_reasonable_expansion_ratio(self, tokenizer):
         """With lowercasing, proper nouns should not explode into codepoint escapes.
 
-        Spaces are escaped by tensor2tensor's design (3 tokens each), so we
-        count only word subtokens, excluding space-escape sequences.
+        Whitespace-splitting means no inter-word space escapes, so every
+        subtoken belongs to a word.
         """
         text = "Cytherea Camenis Roma Gallia"
         tokens = tokenizer.tokenize(text)
-        # Filter out space-escape tokens (\, 32, ;_)
-        word_tokens = [t for t in tokens if t not in ("\\", "32", ";_")]
-        words = text.split()
-        ratio = len(word_tokens) / len(words)
+        # No space-escape tokens should be present at all
+        assert all(t not in ("\\", "32", ";_") for t in tokens)
+        ratio = len(tokens) / len(text.split())
         assert ratio < 2.5, f"Word expansion ratio {ratio:.1f}x is too high"
 
     def test_do_lower_case_false(self):

tests/test_tokenizer_fast.py

@@ -0,0 +1,68 @@
+"""Tests for LatinBertTokenizerFast (Rust-backed, word_ids support).
+
+Guards the two fixes to the fast tokenizer:
+  1. whitespace-split before char-class encoding (no inter-word space
+     escapes; correct word_ids alignment)
+  2. lowercasing normalizer (do_lower_case), so capitalized input is not
+     escaped to codepoints — matching the slow tokenizer.
+"""
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+VOCAB_FILE = str(Path(__file__).parent / "latin.subword.encoder")
+
+
+def _load_fast_module():
+    path = REPO_ROOT / "tokenization_latin_bert_fast.py"
+    spec = importlib.util.spec_from_file_location("tlbf_under_test", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+@pytest.fixture(scope="module")
+def fast_module():
+    return _load_fast_module()
+
+
+@pytest.fixture
+def fast(fast_module):
+    return fast_module.LatinBertTokenizerFast(vocab_file=VOCAB_FILE)
+
+
+@pytest.fixture
+def slow():
+    from latincy_latinbert import LatinBertTokenizer
+    return LatinBertTokenizer(vocab_file=VOCAB_FILE)
+
+
+class TestFastTokenizer:
+    def test_no_interword_space_escapes(self, fast):
+        enc = fast("gallia est omnis divisa in partes tres")
+        toks = fast.convert_ids_to_tokens(enc["input_ids"])
+        assert all(t not in ("\\", "32", ";_") for t in toks)
+
+    def test_word_ids_alignment(self, fast):
+        enc = fast("Gallia est omnis")
+        # [CLS]=None, gallia=0, est=1, omnis=2, [SEP]=None
+        assert enc.word_ids() == [None, 0, 1, 2, None]
+
+    def test_lowercases_like_slow(self, fast):
+        """Capitalized input must not produce codepoint escapes."""
+        enc = fast("Gallia")
+        toks = fast.convert_ids_to_tokens(enc["input_ids"])
+        assert "gallia_" in toks
+        assert all(not t.isdigit() for t in toks)
+
+    def test_matches_slow_ids(self, fast, slow):
+        for text in ["Gallia est omnis divisa in partes tres",
+                     "arma virumque cano",
+                     "ROMA",
+                     "gallia est omnis divisa in partes tres."]:
+            s = slow.encode(text, add_special_tokens=False)
+            f = fast.encode(text, add_special_tokens=False)
+            assert s == f, f"slow/fast mismatch for {text!r}: {s} != {f}"

tokenization_latin_bert.py

@@ -187,13 +187,24 @@ class LatinBertTokenizer(PreTrainedTokenizer):
         return vocab
 
     def _tokenize(self, text: str, **kwargs) -> List[str]:
-        """Tokenize text into subtoken strings."""
+        """Tokenize text into subtoken strings.
+
+        Splits on whitespace first (like the original gen_berts.py
+        LatinTokenizer), then applies character-class splitting within
+        each word. This is essential: tensor2tensor's tokenizer.encode()
+        drops inter-word spaces, so feeding a full string through
+        _tokenizer_encode() directly would escape every space to
+        ``\\32;_`` and inject spurious subtokens between words, corrupting
+        the input the model was trained on and breaking word alignment.
+        Because whitespace-split words contain no spaces, no space escape
+        can be produced here.
+        """
         if self.do_lower_case:
             text = text.lower()
-        tokens = _tokenizer_encode(text)
         subtokens = []
-        for token in tokens:
-            subtokens.extend(self._token_to_subtokens(token))
+        for word in text.split():
+            for token in _tokenizer_encode(word):
+                subtokens.extend(self._token_to_subtokens(token))
         return subtokens
 
     def _token_to_subtokens(self, token: str) -> List[str]:
@@ -240,16 +251,22 @@ class LatinBertTokenizer(PreTrainedTokenizer):
         return self.unk_token
 
     def convert_tokens_to_string(self, tokens: List[str]) -> str:
-        """Reverse the tokenization: unescape and join."""
+        """Reverse the tokenization: unescape and join.
+
+        Word boundaries are the trailing ``_`` markers, so each is
+        rendered as a space (BERT-style; punctuation spacing is lossy,
+        e.g. "tres." → "tres ."). Literal underscores are encoded as
+        ``\\u`` and are restored afterward.
+        """
         # Filter out special tokens before joining
         filtered = [t for t in tokens if t not in SPECIAL_TOKENS]
         text = "".join(filtered)
-        # Remove trailing underscores (word boundary markers)
-        # and unescape: \\u → _, \\\\ → \\, \\<digits>; → chr
-        text = re.sub(r"(?<!\\)_", "", text)
+        # Word-boundary underscores → spaces; then unescape:
+        # \\<digits>; → chr, \\u → _, \\\\ → \\
+        text = re.sub(r"(?<!\\)_", " ", text)
         text = re.sub(r"\\(\d+);", lambda m: chr(int(m.group(1))), text)
         text = text.replace("\\u", "_").replace("\\\\", "\\")
-        return text
+        return text.strip()
 
     def build_inputs_with_special_tokens(
         self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None

tokenization_latin_bert_fast.py

@@ -15,7 +15,7 @@ Usage:
         "latincy/latin-bert", trust_remote_code=True, use_fast=True
     )
     enc = tokenizer("Gallia est omnis", return_tensors="pt")
-    enc.word_ids()  # [None, 0, 0, 1, 2, 2, None]
+    enc.word_ids()  # [None, 0, 1, 2, None]  (each word maps to its subtokens)
 """
 
 import os
@@ -23,7 +23,7 @@ import re
 import unicodedata
 from typing import Dict, List, Optional, Tuple
 
-from tokenizers import Tokenizer, pre_tokenizers, processors
+from tokenizers import Tokenizer, normalizers, pre_tokenizers, processors
 from tokenizers.models import WordPiece
 from tokenizers.pre_tokenizers import PreTokenizer
 from transformers import PreTrainedTokenizerFast
@@ -86,8 +86,11 @@ def _escape_token(token: str, alphabet: set) -> str:
 class _LatinBertPreTokenizer:
     """Custom pre-tokenizer: character-class split + escape + append '_'.
 
-    In is_split_into_words mode, each word is processed individually
-    (char-class split is usually a no-op for single tokens).
+    Applied AFTER a WhitespaceSplit pre-tokenizer (see
+    _build_backend_tokenizer), so it receives one whitespace-delimited
+    word at a time and splits it at alphanumeric/non-alphanumeric
+    boundaries (e.g. "tres," → "tres_", ",_"). It never sees inter-word
+    spaces, so no `\\32;_` space-escapes are produced.
     """
 
     def __init__(self, alphabet: set):
@@ -124,7 +127,7 @@ NUM_SPECIAL = 5
 VOCAB_FILES_NAMES = {"vocab_file": "latin.subword.encoder"}
 
 
-def _build_backend_tokenizer(vocab_file: str) -> Tokenizer:
+def _build_backend_tokenizer(vocab_file: str, do_lower_case: bool = True) -> Tokenizer:
     """Build a tokenizers.Tokenizer from the SubwordTextEncoder vocab."""
     # Load subword vocab
     subtoken_strings = []
@@ -159,10 +162,24 @@ def _build_backend_tokenizer(vocab_file: str) -> Tokenizer:
     )
     tokenizer = Tokenizer(model)
 
-    # Custom pre-tokenizer: char-class split + escape + append '_'
-    tokenizer.pre_tokenizer = PreTokenizer.custom(
-        _LatinBertPreTokenizer(alphabet)
-    )
+    # Normalizer: lowercase (the vocab was trained on lowercased text;
+    # matches the slow tokenizer's do_lower_case=True). Without this,
+    # uppercase chars fall outside the alphabet and get escaped to their
+    # codepoints (e.g. 'G' → `\71;`), inflating tokens and producing
+    # embeddings the model never saw.
+    if do_lower_case:
+        tokenizer.normalizer = normalizers.Lowercase()
+
+    # Pre-tokenizer: whitespace-split FIRST (drops inter-word spaces,
+    # keeps punctuation attached), then char-class split + escape + '_'
+    # per word. Splitting on whitespace first is essential — otherwise
+    # spaces are escaped to `\32;_` and injected between words, corrupting
+    # the model input and breaking word_ids() alignment. WhitespaceSplit
+    # preserves offsets, so word_ids() maps subtokens to the right words.
+    tokenizer.pre_tokenizer = pre_tokenizers.Sequence([
+        pre_tokenizers.WhitespaceSplit(),
+        PreTokenizer.custom(_LatinBertPreTokenizer(alphabet)),
+    ])
 
     # BertProcessing: adds [CLS] at start, [SEP] at end
     tokenizer.post_processor = processors.BertProcessing(
@@ -195,6 +212,7 @@ class LatinBertTokenizerFast(PreTrainedTokenizerFast):
         self,
         vocab_file: Optional[str] = None,
         tokenizer_object: Optional[Tokenizer] = None,
+        do_lower_case: bool = True,
         pad_token: str = "[PAD]",
         unk_token: str = "[UNK]",
         cls_token: str = "[CLS]",
@@ -204,7 +222,10 @@ class LatinBertTokenizerFast(PreTrainedTokenizerFast):
         **kwargs,
     ):
         if tokenizer_object is None and vocab_file is not None:
-            tokenizer_object = _build_backend_tokenizer(vocab_file)
+            tokenizer_object = _build_backend_tokenizer(
+                vocab_file, do_lower_case=do_lower_case
+            )
+        self.do_lower_case = do_lower_case
 
         # PreTrainedTokenizerFast.__init__ does deepcopy(tokenizer_object),
         # which fails for custom Python pre-tokenizers. Bypass by setting
@@ -234,8 +255,13 @@ class LatinBertTokenizerFast(PreTrainedTokenizerFast):
 
     @staticmethod
     def _unescape(text: str) -> str:
-        """Reverse the t2t escape encoding used by the pre-tokenizer."""
-        text = re.sub(r"(?<!\\)_", "", text)
+        """Reverse the t2t escape encoding used by the pre-tokenizer.
+
+        Word-boundary underscores become spaces (BERT-style decode;
+        punctuation spacing is lossy). Literal underscores are encoded
+        as ``\\u`` and restored afterward.
+        """
+        text = re.sub(r"(?<!\\)_", " ", text)
         text = re.sub(r"\\(\d+);", lambda m: chr(int(m.group(1))), text)
         text = text.replace("\\u", "_").replace("\\\\", "\\")
         return text.strip()