convert: use token types for BERT special tokens check

Author: o7si

convert_hf_to_gguf.py

@@ -5284,18 +5284,14 @@ def set_vocab(self):
         self.gguf_writer.add_token_type_count(self.hparams.get("type_vocab_size", 1))
 
         # convert to phantom space vocab
-        def phantom(tok):
-            if tok.startswith("[") and tok.endswith("]"):
+        def phantom(tok, toktype):
+            if toktype != gguf.TokenType.NORMAL:
                 return tok
             if tok.startswith("##"):
-                suffix = tok[2:]
-                # If the suffix resembles a special token, keep the original to avoid collision
-                # E.g., preserve "##[1]" to avoid conflict with an existing "[1]" token.
-                if suffix.startswith("[") and suffix.endswith("]"):
-                    return tok
-                return suffix
+                return tok[2:]
             return "\u2581" + tok
-        tokens = list(map(phantom, tokens))
+        assert len(tokens) == len(toktypes)
+        tokens = list(map(phantom, tokens, toktypes))
 
         # add vocab to gguf
         self.gguf_writer.add_tokenizer_model("bert")