Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion conversion/qwen.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from __future__ import annotations

import json

from typing import Any, Callable, Iterable, TYPE_CHECKING

import torch
Expand Down Expand Up @@ -641,7 +643,19 @@ def set_vocab(self):
logger.info(f"DFlash: Using tokenizer from target model: {self.target_model_dir}")
original_dir = self.dir_model
self.dir_model = self.target_model_dir
super().set_vocab()

# Reuse the target model's own vocab handler (e.g. Gemma-4 needs its
# own tokenizer logic, not the Qwen default).
from . import get_model_class
with open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f:
target_arch = json.load(f)["architectures"][0]
target_cls = get_model_class(target_arch)

if target_cls is not type(self):
target_cls.set_vocab(self) # ty: ignore[unresolved-attribute]
else:
super().set_vocab()

self.dir_model = original_dir

mask_token_id = self.hparams.get("dflash_config", {}).get("mask_token_id")
Expand Down
Loading