From 79f294ae792a07331160636a1a1c8065d6557ced Mon Sep 17 00:00:00 2001 From: Hui Kang Date: Fri, 26 Jun 2026 21:47:54 +0800 Subject: [PATCH] [model] Add MiniCPM-V 4.6 model support --- src/mcore_bridge/config/parser.py | 4 +- src/mcore_bridge/model/constant.py | 2 + src/mcore_bridge/model/mm_gpts/__init__.py | 2 +- src/mcore_bridge/model/mm_gpts/minicpmv4_6.py | 91 +++++++++++++++++++ tests/test_mllm.py | 7 +- 5 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 src/mcore_bridge/model/mm_gpts/minicpmv4_6.py diff --git a/src/mcore_bridge/config/parser.py b/src/mcore_bridge/config/parser.py index a97ba70..b5351a5 100644 --- a/src/mcore_bridge/config/parser.py +++ b/src/mcore_bridge/config/parser.py @@ -152,7 +152,7 @@ def hf_to_mcore_config(hf_config: PretrainedConfig) -> Dict[str, Any]: moe_n_hash_layers = res.pop('moe_n_hash_layers', None) rope_scaling = res.get('rope_scaling') or {} if llm_model_type in {'qwen3', 'qwen3_moe', 'qwen3_next'} or hf_model_type in { - 'qwen3_omni_moe', 'qwen3_omni', 'qwen3_vl', 'qwen3_vl_moe', 'qwen3_5', 'qwen3_5_moe', 'llavaonevision1_5' + 'qwen3_omni_moe', 'qwen3_omni', 'qwen3_vl', 'qwen3_vl_moe', 'qwen3_5', 'qwen3_5_moe', 'llavaonevision1_5', 'minicpmv4_6' }: res['qk_layernorm'] = True if llm_model_type in {'qwen2_moe', 'qwen3_moe', 'qwen3_next' @@ -220,7 +220,7 @@ def hf_to_mcore_config(hf_config: PretrainedConfig) -> Dict[str, Any]: res.pop('num_query_groups', None) if llm_model_type == 'glm_moe_dsa': res['experimental_attention_variant'] = 'dsa' - elif llm_model_type == 'qwen3_next' or hf_model_type in {'qwen3_5', 'qwen3_5_moe'}: + elif llm_model_type == 'qwen3_next' or hf_model_type in {'qwen3_5', 'qwen3_5_moe', 'minicpmv4_6'}: use_mcore_gdn = get_env_args('USE_MCORE_GDN', bool, True) res['layernorm_zero_centered_gamma'] = True res['attention_output_gate'] = True diff --git a/src/mcore_bridge/model/constant.py b/src/mcore_bridge/model/constant.py index 02ce638..f1ba4ad 100644 --- a/src/mcore_bridge/model/constant.py +++ b/src/mcore_bridge/model/constant.py @@ -39,6 +39,8 @@ class MLLMModelType: llava_onevision1_5 = 'llava_onevision1_5' + minicpmv4_6 = 'minicpmv4_6' + class ModelType(LLMModelType, MLLMModelType): pass diff --git a/src/mcore_bridge/model/mm_gpts/__init__.py b/src/mcore_bridge/model/mm_gpts/__init__.py index b862ec6..a8950fa 100644 --- a/src/mcore_bridge/model/mm_gpts/__init__.py +++ b/src/mcore_bridge/model/mm_gpts/__init__.py @@ -1,2 +1,2 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -from . import gemma4, glm, internvl, kimi_vl, llama4, llava, qwen, qwen3_5, qwen3_5_gdn, qwen3_asr, qwen3_omni, qwen3_vl +from . import gemma4, glm, internvl, kimi_vl, llama4, llava, minicpmv4_6, qwen, qwen3_5, qwen3_5_gdn, qwen3_asr, qwen3_omni, qwen3_vl diff --git a/src/mcore_bridge/model/mm_gpts/minicpmv4_6.py b/src/mcore_bridge/model/mm_gpts/minicpmv4_6.py new file mode 100644 index 0000000..16b5b73 --- /dev/null +++ b/src/mcore_bridge/model/mm_gpts/minicpmv4_6.py @@ -0,0 +1,91 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import torch +from transformers import PretrainedConfig + +from ..constant import ModelType +from ..gpts.qwen3_next_gdn import Qwen3NextGDNBridgeMixin, Qwen3NextLoader +from ..register import ModelMeta, register_model +from .utils import HuggingFaceVit + + +class MiniCPMV46Vit(HuggingFaceVit): + module_mapping = { + 'model.vision_tower': 'vision_tower', + 'model.merger': 'merger', + } + _vision_tower = ['vision_tower'] + _aligner = ['merger'] + + def prepare_model(self, hf_config: PretrainedConfig): + from transformers.models.minicpmv4_6.modeling_minicpmv4_6 import ( + MiniCPMV4_6VisionModel, + MiniCPMV4_6Merger, + MiniCPMV4_6Model + ) + self.vision_tower = MiniCPMV4_6VisionModel._from_config(hf_config.vision_config) + self.merger = MiniCPMV4_6Merger(hf_config).to(dtype=self.vision_tower.dtype) + self.model_cls = MiniCPMV4_6Model + + def get_inputs_embeds(self, inputs_embeds, **kwargs): + input_ids = kwargs.get('input_ids') + pixel_values = kwargs.get('pixel_values') + pixel_values_videos = kwargs.get('pixel_values_videos') + target_sizes = kwargs.get('target_sizes') + target_sizes_videos = kwargs.get('target_sizes_videos') + hf_config = self.hf_config + + if pixel_values is None and pixel_values_videos is None: + patch_size = hf_config.vision_config.patch_size + dummy_pv = torch.zeros( + 1, 3, 4 * patch_size, 4 * patch_size, + device=inputs_embeds.device, dtype=self.vision_tower.dtype) + dummy_ts = torch.tensor( + [[4, 4]], device=inputs_embeds.device, dtype=torch.int32) + with self.patch_hf_config(): + vision_output = self.model_cls.get_image_features(self, dummy_pv, dummy_ts) + image_embeds = torch.cat(vision_output.pooler_output, dim=0) + inputs_embeds = inputs_embeds + image_embeds.mean() * 0. + else: + if pixel_values is not None: + num_beams = pixel_values.shape[0] + with self.patch_hf_config(): + vision_output = self.model_cls.get_image_features( + self, pixel_values[:1].to(dtype=self.vision_tower.dtype), target_sizes) + image_features = ( + torch.cat(vision_output.pooler_output, dim=0) + .to(device=inputs_embeds.device, dtype=inputs_embeds.dtype) + .repeat(num_beams, 1)) + mask = self.model_cls.get_placeholder_mask( + self, input_ids, inputs_embeds, image_features, hf_config.image_token_id) + inputs_embeds = inputs_embeds.masked_scatter(mask, image_features) + + if pixel_values_videos is not None: + num_beams = pixel_values_videos.shape[0] + with self.patch_hf_config(): + vision_output = self.model_cls.get_video_features( + self, pixel_values_videos[:1].to(dtype=self.vision_tower.dtype), target_sizes_videos) + video_features = ( + torch.cat(vision_output.pooler_output, dim=0) + .to(device=inputs_embeds.device, dtype=inputs_embeds.dtype) + .repeat(num_beams, 1)) + mask = self.model_cls.get_placeholder_mask( + self, input_ids, inputs_embeds, video_features, hf_config.video_token_id) + inputs_embeds = inputs_embeds.masked_scatter(mask, video_features) + + return inputs_embeds + + +class MiniCPMV46Bridge(Qwen3NextGDNBridgeMixin): + hf_layers_prefix = 'model.language_model.layers' + hf_embed_key = 'model.language_model.embed_tokens.weight' + hf_final_layernorm_key = 'model.language_model.norm.weight' + + +register_model( + ModelMeta( + ModelType.minicpmv4_6, + ['minicpmv4_6'], + bridge_cls=MiniCPMV46Bridge, + visual_cls=MiniCPMV46Vit, + loader=Qwen3NextLoader, + )) diff --git a/tests/test_mllm.py b/tests/test_mllm.py index 5eb07d2..e2029be 100644 --- a/tests/test_mllm.py +++ b/tests/test_mllm.py @@ -120,6 +120,10 @@ def test_gemma4(): _test_model('google/gemma-4-E2B-it') +def test_minicpmv4_6(): + _test_model('openbmb/MiniCPM-V-4.6') + + if __name__ == '__main__': # test_qwen2_5_vl() # test_qwen2_vl() @@ -141,4 +145,5 @@ def test_gemma4(): # test_qwen3_5() # test_llava_onevision1_5() # test_qwen3_asr() - test_gemma4() + # test_gemma4() + test_minicpmv4_6()