feat(minimax-h3): single-file truncated int8-convrot Qwen3-VL text encoder (PR 7)

invokeai/app/invocations/minimax_h3_model_loader.py

@@ -37,16 +37,17 @@ class MiniMaxH3ModelLoaderOutput(BaseInvocationOutput):
     title="Main Model - MiniMax H3",
     tags=["model", "minimax", "video"],
     category="model",
-    version="1.1.0",
+    version="1.2.0",
     classification=Classification.Prototype,
 )
 class MiniMaxH3ModelLoaderInvocation(BaseInvocation):
     """Loads a MiniMax H3 (FL2VA) model, outputting its submodels.
 
     All six submodels (transformer, text encoder, tokenizer, processor, video VAE, audio VAE)
     come from the one diffusers-layout install. Optionally, a single-file transformer checkpoint
-    (e.g. the pruned int8 repack) replaces the folder's transformer while the encoders and VAEs
-    keep coming from the folder install.
+    (e.g. the pruned int8 repack) replaces the folder's transformer, and/or a single-file
+    truncated Qwen3-VL encoder (e.g. the int8 repack) replaces the folder's text encoder, while
+    everything else keeps coming from the folder install.
     """
 
     model: ModelIdentifierField = InputField(
@@ -67,6 +68,17 @@ class MiniMaxH3ModelLoaderInvocation(BaseInvocation):
         ui_model_format=ModelFormat.Checkpoint,
         title="Transformer (single file)",
     )
+    text_encoder_model: Optional[ModelIdentifierField] = InputField(
+        default=None,
+        description="Optional single-file MiniMax H3 Qwen3-VL text encoder (e.g. the truncated int8 "
+        "repack) used in place of the main model's text encoder. The tokenizer and processor still "
+        "come from the main model.",
+        input=Input.Direct,
+        ui_model_base=BaseModelType.MiniMaxH3,
+        ui_model_type=ModelType.Qwen3VLEncoder,
+        ui_model_format=ModelFormat.Checkpoint,
+        title="Text Encoder (single file)",
+    )
 
     def invoke(self, context: InvocationContext) -> MiniMaxH3ModelLoaderOutput:
         if not context.models.exists(self.model.key):
@@ -80,7 +92,12 @@ def invoke(self, context: InvocationContext) -> MiniMaxH3ModelLoaderOutput:
             transformer = self.model.model_copy(update={"submodel_type": SubModelType.Transformer})
         tokenizer = self.model.model_copy(update={"submodel_type": SubModelType.Tokenizer})
         processor = self.model.model_copy(update={"submodel_type": SubModelType.Processor})
-        text_encoder = self.model.model_copy(update={"submodel_type": SubModelType.TextEncoder})
+        if self.text_encoder_model is not None:
+            if not context.models.exists(self.text_encoder_model.key):
+                raise ValueError(f"Unknown text encoder model: {self.text_encoder_model.key}")
+            text_encoder = self.text_encoder_model.model_copy(update={"submodel_type": SubModelType.TextEncoder})
+        else:
+            text_encoder = self.model.model_copy(update={"submodel_type": SubModelType.TextEncoder})
         vae = self.model.model_copy(update={"submodel_type": SubModelType.VAE})
         audio_vae = self.model.model_copy(update={"submodel_type": SubModelType.AudioVAE})
 

invokeai/app/services/workflow_records/default_workflows/First Frame to Video - MiniMax H3.json

@@ -72,7 +72,7 @@
       "type": "invocation",
       "data": {
         "id": "9a1e6c7b-1d2f-4b3c-8e1a-2f3d4c5b6a01",
-        "version": "1.1.0",
+        "version": "1.2.0",
         "nodePack": "invokeai",
         "label": "",
         "notes": "",

invokeai/app/services/workflow_records/default_workflows/Text to Video - MiniMax H3.json

@@ -42,7 +42,7 @@
       "type": "invocation",
       "data": {
         "id": "9a1e6c7b-1d2f-4b3c-8e1a-2f3d4c5b6a01",
-        "version": "1.1.0",
+        "version": "1.2.0",
         "nodePack": "invokeai",
         "label": "",
         "notes": "",

invokeai/backend/minimax_h3/qwen3vl_32b_h3_text_encoder_config.json

@@ -0,0 +1,62 @@
+{
+  "architectures": [
+    "Qwen3VLForConditionalGeneration"
+  ],
+  "image_token_id": 151655,
+  "model_type": "qwen3_vl",
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 151643,
+    "dtype": "bfloat16",
+    "eos_token_id": 151645,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 5120,
+    "initializer_range": 0.02,
+    "intermediate_size": 25600,
+    "max_position_embeddings": 262144,
+    "model_type": "qwen3_vl_text",
+    "num_attention_heads": 64,
+    "num_hidden_layers": 64,
+    "num_key_value_heads": 8,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": {
+      "mrope_interleaved": true,
+      "mrope_section": [
+        24,
+        20,
+        20
+      ],
+      "rope_type": "default"
+    },
+    "rope_theta": 5000000,
+    "use_cache": true,
+    "vocab_size": 151936
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.0.dev0",
+  "video_token_id": 151656,
+  "vision_config": {
+    "deepstack_visual_indexes": [
+      8,
+      16,
+      24
+    ],
+    "depth": 27,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "in_channels": 3,
+    "initializer_range": 0.02,
+    "intermediate_size": 4304,
+    "model_type": "qwen3_vl",
+    "num_heads": 16,
+    "num_position_embeddings": 2304,
+    "out_hidden_size": 5120,
+    "patch_size": 16,
+    "spatial_merge_size": 2,
+    "temporal_patch_size": 2
+  },
+  "vision_end_token_id": 151653,
+  "vision_start_token_id": 151652
+}

invokeai/backend/minimax_h3/text_conditioning.py

@@ -16,6 +16,29 @@
 )
 
 
+def validate_text_encoder_depth(text_encoder) -> None:
+    """Reject encoders whose ``hidden_states[MINIMAX_H3_TEXT_ENCODER_LAYER]`` is not H3's conditioning.
+
+    In a full stack that entry is mid-stack and always unnormalized. In a stack truncated to
+    exactly that many layers, transformers appends the post-final-norm output at that index
+    instead - which is only the right tensor when the final norm is an Identity, as in the
+    purpose-truncated H3 single-file encoders (their files ship no final norm and the loader
+    installs an Identity in its place).
+    """
+    num_layers = text_encoder.config.text_config.num_hidden_layers
+    truncated_ok = num_layers == MINIMAX_H3_TEXT_ENCODER_LAYER and isinstance(
+        text_encoder.model.language_model.norm, torch.nn.Identity
+    )
+    if num_layers <= MINIMAX_H3_TEXT_ENCODER_LAYER and not truncated_ok:
+        raise ValueError(
+            f"MiniMax H3 conditions on hidden_states[{MINIMAX_H3_TEXT_ENCODER_LAYER}] of its Qwen3-VL "
+            f"conditioner, which needs more than {MINIMAX_H3_TEXT_ENCODER_LAYER} decoder layers; the "
+            f"selected text encoder has {num_layers}. A truncated stack's last hidden state is post-norm "
+            "and is not the conditioning MiniMax H3 expects (unless the final norm is an Identity, as in "
+            "the H3-truncated single-file encoders)."
+        )
+
+
 def encode_prompt(
     text_encoder,
     tokenizer,
@@ -40,14 +63,7 @@ def encode_prompt(
         ``(prompt_embeds, text_token_tags)``: the ``(1, num_text_tokens, text_dim)`` hidden
         states and the per-row modality tags (vision-block rows are tagged as video).
     """
-    num_layers = text_encoder.config.text_config.num_hidden_layers
-    if num_layers <= MINIMAX_H3_TEXT_ENCODER_LAYER:
-        raise ValueError(
-            f"MiniMax H3 conditions on hidden_states[{MINIMAX_H3_TEXT_ENCODER_LAYER}] of its Qwen3-VL "
-            f"conditioner, which needs more than {MINIMAX_H3_TEXT_ENCODER_LAYER} decoder layers; the "
-            f"selected text encoder has {num_layers}. A truncated stack's last hidden state is post-norm "
-            "and is not the conditioning MiniMax H3 expects."
-        )
+    validate_text_encoder_depth(text_encoder)
 
     pixel_values, image_grid_thw = None, None
     token_ids: list[int] = []

invokeai/backend/model_manager/configs/factory.py

@@ -117,6 +117,7 @@
 )
 from invokeai.backend.model_manager.configs.qwen3_vl_encoder import (
     Qwen3VLEncoder_Checkpoint_Config,
+    Qwen3VLEncoder_Checkpoint_MiniMaxH3_Config,
     Qwen3VLEncoder_Qwen3VLEncoder_Config,
 )
 from invokeai.backend.model_manager.configs.qwen_vl_encoder import (
@@ -375,6 +376,10 @@ def has_model_export(module: Any, name: Any, expected_bases: tuple[type, ...]) -
         # Qwen3-VL Encoder (Qwen3-VL multimodal encoder for Krea-2) - checked BEFORE the text-only Qwen3
         # encoder so single-file VL checkpoints (which also carry generic model.layers.* keys) are not
         # misclassified as the Z-Image Qwen3 encoder. The VL probe requires the visual tower.
+        # MiniMax H3's truncated 32B conditioning encoder goes first: it matches on explicit
+        # safetensors metadata without reading tensors, and the Krea-2 config below is locked to
+        # the 4B shape so neither can claim the other's files.
+        Annotated[Qwen3VLEncoder_Checkpoint_MiniMaxH3_Config, Qwen3VLEncoder_Checkpoint_MiniMaxH3_Config.get_tag()],
         Annotated[Qwen3VLEncoder_Checkpoint_Config, Qwen3VLEncoder_Checkpoint_Config.get_tag()],
         Annotated[Qwen3VLEncoder_Qwen3VLEncoder_Config, Qwen3VLEncoder_Qwen3VLEncoder_Config.get_tag()],
         # Qwen3 Encoder

invokeai/backend/model_manager/configs/qwen3_vl_encoder.py

@@ -193,3 +193,65 @@ def from_model_on_disk(cls, mod: ModelOnDisk, override_fields: dict[str, Any]) -
         _validate_krea2_qwen3_vl_checkpoint_shape(state_dict)
 
         return cls(**override_fields)
+
+
+_MINIMAX_H3_TE_METADATA_KEY = "minimax_h3_te"
+_MINIMAX_H3_TE_HIDDEN_SIZE = 5120
+
+
+class Qwen3VLEncoder_Checkpoint_MiniMaxH3_Config(Checkpoint_Config_Base, Config_Base):
+    """Configuration for MiniMax H3's truncated Qwen3-VL-32B conditioning encoder single files
+    (Comfy-Org ``qwen3vl_32b_minimax_h3_*.safetensors`` and mirrors).
+
+    These are NOT complete Qwen3-VL-32B checkpoints: the language stack is truncated to the 50
+    layers H3 conditions on (the file's ``minimax_h3_te`` metadata records the contract:
+    "unnormalized_hidden_after_layer_50"), the final norm and LM head are omitted, and the
+    bf16/int8-convrot repacks quantize only the 50 language layers (vision tower stays bf16).
+
+    Identified primarily by the explicit ``minimax_h3_te`` safetensors metadata; a structural
+    fallback covers metadata-stripped re-uploads. Krea-2's ``Qwen3VLEncoder_Checkpoint_Config``
+    is locked to the 4B shape (hidden 2560), so neither config can claim the other's files.
+
+    The nvfp4 repacks share this layout and are accepted here, but the loader rejects their
+    quantization format early (header-only check) with a clear error - mirroring how the H3
+    transformer checkpoint config treats fp8_scaled files.
+    """
+
+    base: Literal[BaseModelType.MiniMaxH3] = Field(default=BaseModelType.MiniMaxH3)
+    type: Literal[ModelType.Qwen3VLEncoder] = Field(default=ModelType.Qwen3VLEncoder)
+    format: Literal[ModelFormat.Checkpoint] = Field(default=ModelFormat.Checkpoint)
+
+    @classmethod
+    def from_model_on_disk(cls, mod: ModelOnDisk, override_fields: dict[str, Any]) -> Self:
+        raise_if_not_file(mod)
+
+        raise_for_override_fields(cls, override_fields)
+
+        if mod.path.suffix.lower() != ".safetensors":
+            raise NotAMatchError(f"expected a .safetensors file, got {mod.path.suffix or '(no suffix)'}")
+
+        state_dict = mod.load_state_dict()
+        # The structural minimum holds on BOTH paths: a re-tagged arbitrary file must not install
+        # on the strength of its metadata alone and fail only after the ~25 GiB load.
+        if (
+            "model.layers.0.self_attn.q_proj.weight" not in state_dict
+            or "visual.blocks.0.attn.qkv.weight" not in state_dict
+        ):
+            raise NotAMatchError("state dict does not look like a MiniMax H3 Qwen3-VL-32B encoder")
+
+        if _MINIMAX_H3_TE_METADATA_KEY not in mod.metadata():
+            # Structural fallback for metadata-stripped re-uploads: additionally require the
+            # 32B hidden size and no layer beyond the H3 truncation point. (A full 64-layer 32B
+            # encoder is deliberately NOT matched - H3 conditioning requires the truncated stack
+            # or the diffusers folder install.)
+            embed = state_dict.get("model.embed_tokens.weight")
+            shape = getattr(embed, "shape", ())
+            if len(shape) < 2 or shape[1] != _MINIMAX_H3_TE_HIDDEN_SIZE:
+                raise NotAMatchError("state dict does not look like a MiniMax H3 Qwen3-VL-32B encoder")
+            if any(isinstance(key, str) and ".layers.50." in key for key in state_dict):
+                raise NotAMatchError(
+                    "state dict looks like a full (untruncated) Qwen3-VL-32B - MiniMax H3 requires the "
+                    "50-layer truncated conditioning encoder"
+                )
+
+        return cls(**override_fields)