comment

itazap · itazap · commit b65fe63535aa · 2026-03-30T14:01:43.000-07:00
diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py
@@ -702,7 +702,7 @@ def from_pretrained(
             else:
                 tokenizer_auto_map = tokenizer_config["auto_map"].get("AutoTokenizer", None)
 
-        # tokenizer_config_class doesn't match the registered tokenizer for this model_type.
+        # if there is a config, we can check that the tokenizer class != than model class.
         # Use the config class if it's a specialized tokenizer, otherwise fall back to TokenizersBackend.
         if (
             tokenizer_auto_map is None
