[model] Add MiniCPM-V 4.6 model support

src/mcore_bridge/config/parser.py

@@ -152,7 +152,7 @@ def hf_to_mcore_config(hf_config: PretrainedConfig) -> Dict[str, Any]:
     moe_n_hash_layers = res.pop('moe_n_hash_layers', None)
     rope_scaling = res.get('rope_scaling') or {}
     if llm_model_type in {'qwen3', 'qwen3_moe', 'qwen3_next'} or hf_model_type in {
-            'qwen3_omni_moe', 'qwen3_omni', 'qwen3_vl', 'qwen3_vl_moe', 'qwen3_5', 'qwen3_5_moe', 'llavaonevision1_5'
+            'qwen3_omni_moe', 'qwen3_omni', 'qwen3_vl', 'qwen3_vl_moe', 'qwen3_5', 'qwen3_5_moe', 'llavaonevision1_5', 'minicpmv4_6'
     }:
         res['qk_layernorm'] = True
     if llm_model_type in {'qwen2_moe', 'qwen3_moe', 'qwen3_next'
@@ -220,7 +220,7 @@ def hf_to_mcore_config(hf_config: PretrainedConfig) -> Dict[str, Any]:
             res.pop('num_query_groups', None)
         if llm_model_type == 'glm_moe_dsa':
             res['experimental_attention_variant'] = 'dsa'
-    elif llm_model_type == 'qwen3_next' or hf_model_type in {'qwen3_5', 'qwen3_5_moe'}:
+    elif llm_model_type == 'qwen3_next' or hf_model_type in {'qwen3_5', 'qwen3_5_moe', 'minicpmv4_6'}:
         use_mcore_gdn = get_env_args('USE_MCORE_GDN', bool, True)
         res['layernorm_zero_centered_gamma'] = True
         res['attention_output_gate'] = True

src/mcore_bridge/model/constant.py

@@ -39,6 +39,8 @@ class MLLMModelType:
 
     llava_onevision1_5 = 'llava_onevision1_5'
 
+    minicpmv4_6 = 'minicpmv4_6'
+
 
 class ModelType(LLMModelType, MLLMModelType):
     pass

src/mcore_bridge/model/mm_gpts/__init__.py

@@ -1,2 +1,2 @@
 # Copyright (c) ModelScope Contributors. All rights reserved.
-from . import gemma4, glm, internvl, kimi_vl, llama4, llava, qwen, qwen3_5, qwen3_5_gdn, qwen3_asr, qwen3_omni, qwen3_vl
+from . import gemma4, glm, internvl, kimi_vl, llama4, llava, minicpmv4_6, qwen, qwen3_5, qwen3_5_gdn, qwen3_asr, qwen3_omni, qwen3_vl

src/mcore_bridge/model/mm_gpts/minicpmv4_6.py

@@ -0,0 +1,91 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import torch
+from transformers import PretrainedConfig
+
+from ..constant import ModelType
+from ..gpts.qwen3_next_gdn import Qwen3NextGDNBridgeMixin, Qwen3NextLoader
+from ..register import ModelMeta, register_model
+from .utils import HuggingFaceVit
+
+
+class MiniCPMV46Vit(HuggingFaceVit):
+    module_mapping = {
+        'model.vision_tower': 'vision_tower',
+        'model.merger': 'merger',
+    }
+    _vision_tower = ['vision_tower']
+    _aligner = ['merger']
+
+    def prepare_model(self, hf_config: PretrainedConfig):
+        from transformers.models.minicpmv4_6.modeling_minicpmv4_6 import (
+            MiniCPMV4_6VisionModel,
+            MiniCPMV4_6Merger,
+            MiniCPMV4_6Model
+        )
+        self.vision_tower = MiniCPMV4_6VisionModel._from_config(hf_config.vision_config)
+        self.merger = MiniCPMV4_6Merger(hf_config).to(dtype=self.vision_tower.dtype)
+        self.model_cls = MiniCPMV4_6Model
+
+    def get_inputs_embeds(self, inputs_embeds, **kwargs):
+        input_ids = kwargs.get('input_ids')
+        pixel_values = kwargs.get('pixel_values')
+        pixel_values_videos = kwargs.get('pixel_values_videos')
+        target_sizes = kwargs.get('target_sizes')
+        target_sizes_videos = kwargs.get('target_sizes_videos')
+        hf_config = self.hf_config
+
+        if pixel_values is None and pixel_values_videos is None:
+            patch_size = hf_config.vision_config.patch_size
+            dummy_pv = torch.zeros(
+                1, 3, 4 * patch_size, 4 * patch_size,
+                device=inputs_embeds.device, dtype=self.vision_tower.dtype)
+            dummy_ts = torch.tensor(
+                [[4, 4]], device=inputs_embeds.device, dtype=torch.int32)
+            with self.patch_hf_config():
+                vision_output = self.model_cls.get_image_features(self, dummy_pv, dummy_ts)
+            image_embeds = torch.cat(vision_output.pooler_output, dim=0)
+            inputs_embeds = inputs_embeds + image_embeds.mean() * 0.
+        else:
+            if pixel_values is not None:
+                num_beams = pixel_values.shape[0]
+                with self.patch_hf_config():
+                    vision_output = self.model_cls.get_image_features(
+                        self, pixel_values[:1].to(dtype=self.vision_tower.dtype), target_sizes)
+                    image_features = (
+                        torch.cat(vision_output.pooler_output, dim=0)
+                        .to(device=inputs_embeds.device, dtype=inputs_embeds.dtype)
+                        .repeat(num_beams, 1))
+                    mask = self.model_cls.get_placeholder_mask(
+                        self, input_ids, inputs_embeds, image_features, hf_config.image_token_id)
+                inputs_embeds = inputs_embeds.masked_scatter(mask, image_features)
+
+            if pixel_values_videos is not None:
+                num_beams = pixel_values_videos.shape[0]
+                with self.patch_hf_config():
+                    vision_output = self.model_cls.get_video_features(
+                        self, pixel_values_videos[:1].to(dtype=self.vision_tower.dtype), target_sizes_videos)
+                    video_features = (
+                        torch.cat(vision_output.pooler_output, dim=0)
+                        .to(device=inputs_embeds.device, dtype=inputs_embeds.dtype)
+                        .repeat(num_beams, 1))
+                    mask = self.model_cls.get_placeholder_mask(
+                        self, input_ids, inputs_embeds, video_features, hf_config.video_token_id)
+                inputs_embeds = inputs_embeds.masked_scatter(mask, video_features)
+
+        return inputs_embeds
+
+
+class MiniCPMV46Bridge(Qwen3NextGDNBridgeMixin):
+    hf_layers_prefix = 'model.language_model.layers'
+    hf_embed_key = 'model.language_model.embed_tokens.weight'
+    hf_final_layernorm_key = 'model.language_model.norm.weight'
+
+
+register_model(
+    ModelMeta(
+        ModelType.minicpmv4_6,
+        ['minicpmv4_6'],
+        bridge_cls=MiniCPMV46Bridge,
+        visual_cls=MiniCPMV46Vit,
+        loader=Qwen3NextLoader,
+    ))

tests/test_mllm.py

@@ -120,6 +120,10 @@ def test_gemma4():
     _test_model('google/gemma-4-E2B-it')
 
 
+def test_minicpmv4_6():
+    _test_model('openbmb/MiniCPM-V-4.6')
+
+
 if __name__ == '__main__':
     # test_qwen2_5_vl()
     # test_qwen2_vl()
@@ -141,4 +145,5 @@ def test_gemma4():
     # test_qwen3_5()
     # test_llava_onevision1_5()
     # test_qwen3_asr()
-    test_gemma4()
+    # test_gemma4()
+    test_minicpmv4_6()