mtmd: add Eagle2-VL vision and projector support

convert_hf_to_gguf.py

@@ -3848,6 +3848,212 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
         return [] # skip other tensors
 
 
+@ModelBase.register("Eagle2_VLForConditionalGeneration","Eagle2_5_VLForConditionalGeneration")
+class Eagle2VLVisionModel(MmprojModel):
+    """
+    Dedicated Eagle2-VL mmproj converter.
+
+    Responsibilities:
+    - Emit a distinct projector_type (eagle2vl) and vision metadata (image/patch size, mean/std, eps, block_count, merge size).
+    - Perform Eagle2-specific layout normalization during conversion in modify_tensors (in a later follow-up).
+      The C++ runtime will assume canonical layout and must not include Eagle2-specific transposes or hacks.
+    """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.hparams_vision is not None:
+            # Prefer overrides from vision_config when present
+            vc = self.get_vision_config() or {}
+            if "image_size" in vc and "image_size" not in self.hparams_vision:
+                self.hparams_vision["image_size"] = vc["image_size"]
+            if "patch_size" in vc and "patch_size" not in self.hparams_vision:
+                self.hparams_vision["patch_size"] = vc["patch_size"]
+            if "image_mean" in vc and "image_mean" not in self.hparams_vision:
+                self.hparams_vision["image_mean"] = vc["image_mean"]
+            if "image_std" in vc and "image_std" not in self.hparams_vision:
+                self.hparams_vision["image_std"] = vc["image_std"]
+
+            # Normalize common aliases
+            if "num_heads" in self.hparams_vision and "num_attention_heads" not in self.hparams_vision:
+                self.hparams_vision["num_attention_heads"] = self.hparams_vision.get("num_heads")
+            if "depth" in self.hparams_vision and "num_hidden_layers" not in self.hparams_vision:
+                self.hparams_vision["num_hidden_layers"] = self.hparams_vision.get("depth")
+
+    def set_gguf_parameters(self):
+        # Base writes general vision fields (embedding/feed-forward/heads when available)
+        super().set_gguf_parameters()
+
+        # Projector type: Eagle2-VL
+        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.EAGLE2VL)
+
+        # Vision attention layernorm eps: use config if available, else 1e-6
+        vc = self.get_vision_config() or {}
+        eps = 1e-6        
+        if isinstance(vc, dict) and "layer_norm_eps" in vc:
+            eps = vc["layer_norm_eps"]
+        self.gguf_writer.add_vision_attention_layernorm_eps(eps)
+
+        # 2x2 spatial merge
+        self.gguf_writer.add_vision_spatial_merge_size(2)
+
+        # Mirror Qwen2VL-style image metadata if provided; do not guess
+        hpv = self.hparams_vision or {}
+
+        img_sz = hpv.get("image_size")
+        if isinstance(img_sz, (list, tuple)) and len(img_sz) > 0:
+            img_sz = img_sz[0]
+        if isinstance(img_sz, int):
+            self.gguf_writer.add_vision_image_size(img_sz)
+
+        patch_sz = hpv.get("patch_size")
+        if isinstance(patch_sz, (list, tuple)) and len(patch_sz) > 0:
+            patch_sz = patch_sz[0]
+        if isinstance(patch_sz, int):
+            self.gguf_writer.add_vision_patch_size(patch_sz)
+
+        blk_cnt = hpv.get("num_hidden_layers")
+        if isinstance(blk_cnt, int):
+            self.gguf_writer.add_vision_block_count(blk_cnt)
+
+        img_mean = hpv.get("image_mean")
+        if isinstance(img_mean, (list, tuple)) and len(img_mean) > 0:
+            self.gguf_writer.add_vision_image_mean(list(img_mean))
+
+        img_std = hpv.get("image_std")
+        if isinstance(img_std, (list, tuple)) and len(img_std) > 0:
+            self.gguf_writer.add_vision_image_std(list(img_std))
+
+    # Note:
+    # Eagle2-specific tensor layout normalization (mlp1 → mm.*, QKV split)
+    # will live here in Eagle2VLVisionModel.modify_tensors() in a follow-up.
+    # The C++ builder will assume canonical weights and perform zero ad-hoc transposes.
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        del bid  # unused
+
+        # 1) Name prefix normalization
+        if name.startswith("vision_model.vision_model."):
+            name = name.replace("vision_model.vision_model.", "model.vision_model.", 1)
+        # Eagle2 HF projector tensors often live under "multi_modal_projector.*"
+        # Normalize that prefix away so we can match on "mlp1.*" below
+        if name.startswith("multi_modal_projector."):
+            name = name.replace("multi_modal_projector.", "", 1)
+
+        # 2) Skip clearly non-vision towers
+        if name.startswith("audio") or name.startswith("talker") or name.startswith("token2wav"):
+            return []
+
+        # 2.1) Skip classification / pooling head tensors under vision model head
+        # Example tensors to skip:
+        # - vision_model.vision_model.head.attention.in_proj_weight
+        # - vision_model.vision_model.head.mlp.fc1.weight
+        # - vision_model.vision_model.head.probe
+        if ".head." in name:
+            return []
+
+        # 3) Projector MLP remap: map Eagle2-VL mlp1.* -> mm_input_norm/mm.0/mm.2
+        mlp_pos = name.find("mlp1.")
+        if mlp_pos != -1:
+            mlp_suffix = name[mlp_pos + len("mlp1."):]
+            # Map Eagle2-VL projector LayerNorm:
+            # mlp1.0.{weight,bias} correspond to the input LayerNorm of the projector
+            # (structure: LayerNorm → Linear → GELU → Linear).
+            # The C++ runtime applies ggml_norm + scale/shift, so we store γ/β as mm_input_norm_w/b.
+            if mlp_suffix.startswith("0."):
+                if mlp_suffix.endswith("weight"):
+                    return [("mm_input_norm_w", data_torch)]
+                if mlp_suffix.endswith("bias"):
+                    return [("mm_input_norm_b", data_torch)]
+                # any other subfield under mlp1.0.* (rare) -> skip
+                return []
+
+            # Map first Linear (mlp1.1.*) -> mm.0.*
+            if mlp_suffix.startswith("1."):
+                new_name = "mm.0." + mlp_suffix[2:]
+                if new_name.endswith(".weight"):
+                    # Canonicalize to [n_in, n_out]; detect and transpose only if needed.
+                    if data_torch.ndim == 2:
+                        d0, d1 = int(data_torch.shape[0]), int(data_torch.shape[1])
+                        # Expected input width after 2x2 merge = vit_hidden * 4 (if available)
+                        vit_hidden = (self.hparams_vision or {}).get("hidden_size")
+                        expected_in = vit_hidden * 4 if isinstance(vit_hidden, int) and vit_hidden > 0 else None
+                        expected_out = getattr(self, "n_embd_text", None)
+
+                        def _should_transpose_weight(d0, d1, expected_in, expected_out):
+                            # Both expected_in and expected_out are known
+                            if isinstance(expected_in, int) and expected_in > 0 and isinstance(expected_out, int) and expected_out > 0:
+                                # If shape matches [out, in], transpose to [in, out]
+                                if d0 == expected_out and d1 == expected_in:
+                                    return True
+                                # If already [in, out], no transpose
+                                if d0 == expected_in and d1 == expected_out:
+                                    return False
+                                # Otherwise, fall through to heuristics
+                            # Only expected_in is known
+                            if isinstance(expected_in, int) and expected_in > 0:
+                                if d0 == expected_in:
+                                    return False
+                                if d1 == expected_in:
+                                    return True
+                                # Otherwise, fall through to fallback
+                            # Fallback: choose orientation that puts larger dim on axis 0
+                            # or, if dims are equal, assume already canonical
+                            return d0 < d1
+
+                        if _should_transpose_weight(d0, d1, expected_in, expected_out):
+                            data_torch = data_torch.transpose(-1, -2)
+                    return [(new_name, data_torch)]
+                return [(new_name, data_torch)]
+            # Map second Linear (mlp1.3.*) -> mm.2.*
+            # Note: In the HuggingFace model, the projector is implemented as a Sequential module:
+            #   [0]: LayerNorm, [1]: Linear, [2]: GELU, [3]: Linear
+            # So the second Linear layer is at index 3 (mlp1.3.*), not 2, because GELU occupies index 2.
+            if mlp_suffix.startswith("3."):
+                new_name = "mm.2." + mlp_suffix[2:]
+                if new_name.endswith(".weight"):
+                    # Canonicalize to [n_in, n_out] for the second layer.
+                    # For mm.2, the output dimension should match the text embedding dimension (expected), while the input dimension should match mm.0's output (the intermediate dimension). If ambiguous (often square), leave as-is.
+                    if data_torch.ndim == 2:
+                        d0, d1 = int(data_torch.shape[0]), int(data_torch.shape[1])
+                        expected = getattr(self, "n_embd_text", None)
+                        if isinstance(expected, int) and expected > 0:
+                            if d0 == expected and d1 == expected:
+                                pass  # already canonical
+                            elif d1 == expected and d0 != expected:
+                                logging.warning(
+                                    f"mm.2 weight shape {data_torch.shape} for {name}: d1 matches expected ({expected}), d0={d0} does not. Transposing, but please verify correctness."
+                                )
+                                data_torch = data_torch.transpose(-1, -2)
+                            else:
+                                logging.warning(
+                                    f"mm.2 weight shape {data_torch.shape} for {name}: neither dimension matches expected ({expected}). Leaving as-is, but this may be incorrect."
+                                )
+                    return [(new_name, data_torch)]
+                return [(new_name, data_torch)]
+            # Unknown mlp1 component -> skip
+            return []
+
+        # 4) Fused QKV split
+        if ".qkv." in name:
+            # Determine split size from leading dimension
+            c3 = data_torch.shape[0]
+            assert c3 % 3 == 0, f"qkv tensor leading dim must be divisible by 3, got {c3} for {name}"
+            c = c3 // 3
+            wq = data_torch[:c]
+            wk = data_torch[c: 2 * c]
+            wv = data_torch[2 * c:]
+            return [
+                (self.map_tensor_name(name.replace("qkv", "q")), wq),
+                (self.map_tensor_name(name.replace("qkv", "k")), wk),
+                (self.map_tensor_name(name.replace("qkv", "v")), wv),
+            ]
+
+        # 5) Default mapping for remaining Eagle2 vision tensors
+        if name.startswith("vision_model.") or name.startswith("model.vision_model.") or name.startswith("visual."):
+            return [(self.map_tensor_name(name), data_torch)]
+
+        # Not an Eagle2 vision tensor -> skip
+        return []
+
 @ModelBase.register("Qwen2_5OmniModel")
 class Qwen25OmniModel(Qwen2VLVisionModel):
     has_vision_encoder = True

gguf-py/gguf/constants.py

@@ -3414,6 +3414,7 @@ class VisionProjectorType:
     LIGHTONOCR = "lightonocr"
     COGVLM = "cogvlm"
     JANUS_PRO = "janus_pro"
+    EAGLE2VL = "eagle2vl"
     LFM2A = "lfm2a" # audio
     GLM4V = "glm4v"
 

tools/mtmd/clip-impl.h

@@ -169,6 +169,7 @@ enum projector_type {
     PROJECTOR_TYPE_GLM_EDGE,
     PROJECTOR_TYPE_QWEN2VL,
     PROJECTOR_TYPE_QWEN3VL,
+    PROJECTOR_TYPE_EAGLE2VL,
     PROJECTOR_TYPE_GEMMA3,
     PROJECTOR_TYPE_IDEFICS3,
     PROJECTOR_TYPE_PIXTRAL,
@@ -199,6 +200,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_QWEN2VL,   "qwen2vl_merger"},
     { PROJECTOR_TYPE_QWEN25VL,  "qwen2.5vl_merger"},
     { PROJECTOR_TYPE_QWEN3VL,   "qwen3vl_merger"},
+    { PROJECTOR_TYPE_EAGLE2VL,  "eagle2vl"},
     { PROJECTOR_TYPE_GEMMA3,    "gemma3"},
     { PROJECTOR_TYPE_IDEFICS3,  "idefics3"},
     { PROJECTOR_TYPE_PIXTRAL,   "pixtral"},

tools/mtmd/clip.cpp

@@ -802,6 +802,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
             {
                 builder = std::make_unique<clip_graph_qwen3vl>(ctx, img);
             } break;
+        case PROJECTOR_TYPE_EAGLE2VL:
+            {
+                res = graph.build_eagle2vl();
+            } break;
         case PROJECTOR_TYPE_MINICPMV:
             {
                 builder = std::make_unique<clip_graph_minicpmv>(ctx, img);
@@ -1092,6 +1096,15 @@ struct clip_model_loader {
 
             // model-specific params
             switch (model.proj_type) {
+                case PROJECTOR_TYPE_EAGLE2VL:
+                    {
+                        // spatial merge (default 2), allow override from metadata if present
+                        hparams.n_merge = 2;
+                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
+                        // set reasonable token limits and warmup like qwen2vl
+                        hparams.set_limit_image_tokens(8, 4096);
+                        hparams.set_warmup_n_tokens(46*46);
+                    } break;
                 case PROJECTOR_TYPE_MINICPMV:
                     {
                         if (hparams.minicpmv_version == 0) {
@@ -1405,6 +1418,17 @@ struct clip_model_loader {
                     }
                     model.image_newline = get_tensor(TN_IMAGE_NEWLINE, false);
                 } break;
+            case PROJECTOR_TYPE_EAGLE2VL:
+                {
+                    // projector input LayerNorm (mlp1.0.{weight,bias})
+                    model.mm_input_norm_w = get_tensor("mm_input_norm_w", false);
+                    model.mm_input_norm_b = get_tensor("mm_input_norm_b", false);
+                    // 2-layer MLP projector using mm.0 and mm.2
+                    model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
+                    model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"),   false);
+                    model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
+                    model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"),   false);
+                } break;
             case PROJECTOR_TYPE_LDP:
                 {
                     // MobileVLM projection
@@ -2736,6 +2760,7 @@ bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, str
 
         case PROJECTOR_TYPE_GLM_EDGE:
         case PROJECTOR_TYPE_GEMMA3:
+        case PROJECTOR_TYPE_EAGLE2VL:
         case PROJECTOR_TYPE_INTERNVL: // TODO @ngxson : support dynamic resolution
             {
                 clip_image_u8 resized_image;
@@ -2998,6 +3023,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
                 n_patches = x_patch * y_patch;
             } break;
         case PROJECTOR_TYPE_GEMMA3:
+        case PROJECTOR_TYPE_EAGLE2VL:
         case PROJECTOR_TYPE_IDEFICS3:
         case PROJECTOR_TYPE_INTERNVL:
         case PROJECTOR_TYPE_LLAMA4:
@@ -3396,6 +3422,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
                 set_input_i32("patches", patches);
             } break;
         case PROJECTOR_TYPE_GEMMA3:
+        case PROJECTOR_TYPE_EAGLE2VL:
         case PROJECTOR_TYPE_IDEFICS3:
         case PROJECTOR_TYPE_INTERNVL:
         case PROJECTOR_TYPE_QWEN2A:
@@ -3540,6 +3567,9 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
             return ctx->model.mm_2_w->ne[1];
         case PROJECTOR_TYPE_COGVLM:
             return ctx->model.mm_4h_to_h_w->ne[1];
+        case PROJECTOR_TYPE_EAGLE2VL:
+            // final projector output dim
+            return ctx->model.mm_2_w->ne[1];
         case PROJECTOR_TYPE_LFM2A:
             return ctx->model.position_embeddings->ne[0];
         case PROJECTOR_TYPE_GLM4V: