From 90c984e00a1ffe0a913aa9001357a248a19c7027 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Fri, 7 Aug 2026 07:44:16 +0000 Subject: [PATCH 01/10] [NVBug: 6563509] Don't abort model load when the meta-device skeleton fails get_model() builds a throwaway skeleton under init_empty_weights(include_buffers=True) purely to size the model for infer_auto_device_map. include_buffers=True makes accelerate push a global torch.device("meta") context, so every tensor constructed in __init__ lands on meta -- not just parameters and buffers. Remote-code checkpoints written before Transformers v5 routinely derive scalar hyperparameters from real tensors there (Phi-4-multimodal's conformer subsampling does int(torch.tensor(...))), which raises "Tensor.item() cannot be called on meta tensors" and killed the whole run before from_pretrained was ever reached. Retry the skeleton without the global meta context, and if that also fails, warn and skip the memory estimate instead of aborting -- from_pretrained can still map the model on its own. Losing the estimate only costs the automatic max_memory shrink, which --use_seq_device_map and --gpu_max_memory_percentage already cover. Note this does not by itself make Phi-4-multimodal-instruct loadable: its remote code additionally needs transformers <4.52 (Phi4MMModel relies on PreTrainedModel inheriting GenerationMixin, which peft's get_peft_model calls into) and declares _tied_weights_keys as a list, which Transformers 5.x rejects. Both are outside ModelOpt. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- examples/hf_ptq/example_utils.py | 45 ++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index 740a09b2267..b3842c4181a 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -873,17 +873,40 @@ def has_pack_quantized_config(config): hf_config, auto_model_module, ckpt_path, config_kwargs ) - with init_empty_weights(include_buffers=True): - # When computing the device_map, assuming bfloat16 precision by default, - # unless specified by the hf_config. - config_dtype = _get_config_dtype(config_for_init) - model_kwargs2 = _apply_dtype_to_config( - model_kwargs, config_dtype, architecture, apply_config_dtype=True + # When computing the device_map, assuming bfloat16 precision by default, + # unless specified by the hf_config. + config_dtype = _get_config_dtype(config_for_init) + model_kwargs2 = _apply_dtype_to_config( + model_kwargs, config_dtype, architecture, apply_config_dtype=True + ) + if auto_model_module not in [AutoModelForCausalLM, AutoModel]: + model_kwargs2.pop("trust_remote_code", None) + model_kwargs2.pop("max_memory", None) + + # This skeleton is only a sizing aid for ``infer_auto_device_map``; it is thrown + # away right after. ``include_buffers=True`` makes accelerate push a global + # ``torch.device("meta")`` context, so *every* tensor built in ``__init__`` lands + # on meta -- not just parameters and buffers. Remote-code checkpoints written + # before Transformers v5 often compute scalar hyperparameters from real tensors + # there (Phi-4-multimodal's conformer subsampling does ``int(torch.tensor(...))``), + # which raises on meta. Retry without the global context, then give up on the + # estimate rather than failing a load that ``from_pretrained`` can still do. + model = None + skeleton_errors = [] + for include_buffers in (True, False): + try: + with init_empty_weights(include_buffers=include_buffers): + model = from_config(config_for_init, **model_kwargs2) + break + except Exception as e: + skeleton_errors.append(f"include_buffers={include_buffers}: {e!r}") + if model is None: + warnings.warn( + f"Could not build a meta-device skeleton of {architecture} " + f"({'; '.join(skeleton_errors)}). Skipping the device-map memory estimate " + "and letting from_pretrained map the model; if you hit GPU OOM, pass " + "--use_seq_device_map or lower --gpu_max_memory_percentage." ) - if auto_model_module not in [AutoModelForCausalLM, AutoModel]: - model_kwargs2.pop("trust_remote_code", None) - model_kwargs2.pop("max_memory", None) - model = from_config(config_for_init, **model_kwargs2) max_memory = get_max_memory() @@ -903,7 +926,7 @@ def has_pack_quantized_config(config): f"Offload folder: {offload_folder}\n" "Weights exceeding GPU+CPU budgets will be streamed from disk." ) - else: + elif model is not None: inferred_device_map = infer_auto_device_map(model, max_memory=max_memory) if "cpu" in inferred_device_map.values(): for _device in max_memory: From d94911be1feef3bfcc245f3f040090b7371c6688 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Fri, 7 Aug 2026 20:38:47 +0000 Subject: [PATCH 02/10] [NVBug: 6563509] Drop Phi-3-vision / Phi-4-multimodal PTQ support Both ship remote code that predates Transformers v5 and no longer loads on any version this repo supports (transformers>=4.57,<5.15): - Phi-4-multimodal needs transformers<4.52. Its __init__ calls peft.get_peft_model on Phi4MMModel, which reads prepare_inputs_for_generation -- present only while PreTrainedModel still inherited GenerationMixin. Verified loading at 4.48.2 / 4.49.0 / 4.50.0 / 4.51.3, failing at 4.53.3 / 4.56.2 / 4.57.1 with AttributeError. - Both declare _tied_weights_keys as a list; Transformers 5.x calls .keys() on it in post_init and raises AttributeError. - Phi-4-multimodal additionally computes int(torch.tensor(...)) in __init__, which Transformers 5.x's meta-device from_pretrained cannot evaluate. The model card pins transformers==4.48.2 / peft==0.13.2, so there is no overlap with our floor and nothing on our side can bridge it. Removes the support-matrix row, the phi4mm model type, the multimodal-detection heuristics that only ever matched these two (vision_lora, audio_processor, embd_layer.image_embd_layer), the Phi3Image/PhiImage embedding-export exclusions, and modelopt_recipes/huggingface/phi4mm/. Text-only Phi-3/Phi-4 and Phi-3.5-MoE are natively supported by transformers and are untouched. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 3 ++ examples/hf_ptq/README.md | 8 ++++- examples/hf_ptq/example_utils.py | 6 ---- examples/hf_ptq/hf_ptq.py | 3 -- modelopt/torch/export/layer_utils.py | 7 +--- modelopt/torch/export/model_utils.py | 15 -------- .../huggingface/phi4mm/ptq/README.md | 13 ------- .../phi4mm/ptq/disabled_quantizers.yaml | 34 ------------------ .../phi4mm/ptq/nvfp4-kv_fp8_cast.yaml | 36 ------------------- modelopt_recipes/ptq.md | 6 ++-- 10 files changed, 13 insertions(+), 118 deletions(-) delete mode 100644 modelopt_recipes/huggingface/phi4mm/ptq/README.md delete mode 100644 modelopt_recipes/huggingface/phi4mm/ptq/disabled_quantizers.yaml delete mode 100644 modelopt_recipes/huggingface/phi4mm/ptq/nvfp4-kv_fp8_cast.yaml diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 632b5532ebf..a7d522d213b 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,9 +19,12 @@ Changelog **Deprecations** +- Drop PTQ support for **Phi-3-vision** and **Phi-4-multimodal**. Their bundled remote code predates Transformers v5 and no longer loads on the versions this repo requires (``transformers>=4.57``): Phi-4-multimodal needs ``transformers<4.52`` because it reaches ``prepare_inputs_for_generation`` through ``peft``, which requires ``PreTrainedModel`` to still inherit ``GenerationMixin``, and both models declare ``_tied_weights_keys`` as a list, which Transformers 5.x rejects. Removes the ``phi4mm`` model type, its multimodal-detection heuristics (``vision_lora`` / ``audio_processor`` / ``embd_layer.image_embd_layer``), the ``Phi3Image`` / ``PhiImage`` embedding-export exclusions, and the ``modelopt_recipes/huggingface/phi4mm/`` recipes. + **Bug Fixes** - Fix EAGLE-3 training with context parallelism (``--cp_size > 1`` in ``examples/speculative_decoding``), which failed to start on ``accelerate >= 1.13`` and then raised ``got mixed torch.Tensor and DTensor``. +- ``examples/hf_ptq/hf_ptq.py`` no longer aborts the whole load when the throwaway meta-device skeleton it builds to size ``infer_auto_device_map`` cannot be constructed. ``init_empty_weights(include_buffers=True)`` pushes a global ``torch.device("meta")`` context, so remote-code checkpoints that derive scalar hyperparameters from real tensors in ``__init__`` raised ``Tensor.item() cannot be called on meta tensors``. The skeleton is now retried without the global meta context, and if that also fails the memory estimate is skipped with a warning instead of failing the run. 0.46 (2026-08-17) ^^^^^^^^^^^^^^^^^ diff --git a/examples/hf_ptq/README.md b/examples/hf_ptq/README.md index e80926bea91..903d708de9b 100755 --- a/examples/hf_ptq/README.md +++ b/examples/hf_ptq/README.md @@ -119,13 +119,19 @@ Please reference our [framework scripts](#framework-scripts) and our [docs](http | Whisper9 | ✅ | ❌ | ❌ | ❌ | - | | Nemotron-3 | ✅ | ❌ | ❌ | ❌ | ✅ | | Llava (VLM)11 | ✅ | ✅12 | ✅ | ✅ | - | -| Phi-3-vision, Phi-4-multimodal (VLM)11 | ✅ | ✅12 | ✅ | ✅ | ✅ | | Qwen2, 2.5-VL (VLM)11 | ✅ | ✅12 | ✅ | ✅ | ✅ | | Gemma 3 (VLM)11 | ✅ | - | - | - | - | | Nemotron VL (VLM)11,13 | ✅ | - | - | - | ✅ | > *This is a subset of the models supported. For the full list please check the [TensorRT-LLM support matrix](https://nvidia.github.io/TensorRT-LLM/reference/precision.html#support-matrix)* +> *Phi-3-vision and Phi-4-multimodal were dropped from this matrix: their bundled +> remote code predates Transformers v5 and no longer loads on the versions this repo +> requires (`transformers>=4.57`). Phi-4-multimodal needs `transformers<4.52` — it +> reaches `prepare_inputs_for_generation` through `peft`, which requires +> `PreTrainedModel` to still inherit `GenerationMixin` — and both models declare +> `_tied_weights_keys` as a list, which Transformers 5.x rejects.* + > *1.The w4a8_awq is an experimental quantization scheme that may result in a higher accuracy penalty.* \ > *2.For some models, there is only support for exporting quantized checkpoints.* \ > *3.W4A8_AWQ is only available on some models but not all* \ diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index b3842c4181a..6f9fe51063e 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -177,12 +177,6 @@ def _is_multimodal_config(config): """Check if a config indicates a multimodal model (config-only version of is_multimodal_model).""" return ( hasattr(config, "vision_config") # Standard vision config (e.g., Qwen2.5-VL) - or getattr(config, "model_type", "") == "phi4mm" # Phi-4 multimodal - or hasattr(config, "vision_lora") # Vision LoRA configurations - or hasattr(config, "audio_processor") # Audio processing capabilities - or ( - hasattr(config, "embd_layer") and hasattr(config.embd_layer, "image_embd_layer") - ) # Image embedding layers or getattr(config, "is_encoder_decoder", False) # Encoder-decoder VL models or any( # Architecture-based detection for custom VL models (e.g., Nemotron-Parse) "conditionalgeneration" in arch.lower() for arch in getattr(config, "architectures", []) diff --git a/examples/hf_ptq/hf_ptq.py b/examples/hf_ptq/hf_ptq.py index 7a2328d10f7..0790f644308 100755 --- a/examples/hf_ptq/hf_ptq.py +++ b/examples/hf_ptq/hf_ptq.py @@ -705,9 +705,6 @@ def load_model(args: argparse.Namespace): # Left padding usually provides better calibration result. tokenizer.padding_side = "left" - if model_type == "phi4mm": - warnings.warn("Please set the default input_mode to InputMode.LANGUAGE before quantizing.") - return ( full_model, language_model, diff --git a/modelopt/torch/export/layer_utils.py b/modelopt/torch/export/layer_utils.py index d5f1fb2330d..de136fcd378 100755 --- a/modelopt/torch/export/layer_utils.py +++ b/modelopt/torch/export/layer_utils.py @@ -222,12 +222,7 @@ def is_conv(module: nn.Module) -> bool: def is_embedding(module: nn.Module) -> bool: """Returns whether the module is an embedding layer.""" module_type_name = type(module).__name__ - return ( - "Embedding" in module_type_name - and "Rotary" not in module_type_name - and "PhiImage" not in module_type_name - and "Phi3Image" not in module_type_name - ) + return "Embedding" in module_type_name and "Rotary" not in module_type_name def build_embedding_config(module: nn.Module, normalization_constant: float = 1) -> EmbeddingConfig: diff --git a/modelopt/torch/export/model_utils.py b/modelopt/torch/export/model_utils.py index 307ea9aac51..1729dbfffcf 100755 --- a/modelopt/torch/export/model_utils.py +++ b/modelopt/torch/export/model_utils.py @@ -44,7 +44,6 @@ "phi3small": "phi3small", "phi3": "phi3", "PhiMoEForCausalLM": "phi3", - "Phi4MMForCausalLM": "phi4mm", "phi": "phi", "TLGv4ForCausalLM": "phi", "MixtralForCausalLM": "llama", @@ -88,10 +87,6 @@ def is_multimodal_model(model): This function detects various multimodal model architectures by checking for: - Standard vision configurations (vision_config) - Language model attributes (language_model) - - Specific multimodal model types (phi4mm) - - Vision LoRA configurations - - Audio processing capabilities - - Image embedding layers - Nemotron-Parse conditional generation models Args: @@ -104,10 +99,6 @@ def is_multimodal_model(model): >>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-VL-7B-Instruct") >>> is_multimodal_model(model) True - - >>> model = AutoModelForCausalLM.from_pretrained("microsoft/Phi-4-multimodal-instruct") - >>> is_multimodal_model(model) - True """ config = model.config @@ -118,12 +109,6 @@ def is_multimodal_model(model): return ( hasattr(config, "vision_config") # Standard vision config (e.g., Qwen2.5-VL) or hasattr(model, "language_model") # Language model attribute (e.g., LLaVA) - or getattr(config, "model_type", "") == "phi4mm" # Phi-4 multimodal - or hasattr(config, "vision_lora") # Vision LoRA configurations - or hasattr(config, "audio_processor") # Audio processing capabilities - or ( - hasattr(config, "embd_layer") and hasattr(config.embd_layer, "image_embd_layer") - ) # Image embedding layers or is_nemotron_parse # Nemotron-Parse conditional generation model ) diff --git a/modelopt_recipes/huggingface/phi4mm/ptq/README.md b/modelopt_recipes/huggingface/phi4mm/ptq/README.md deleted file mode 100644 index bedaf1fcb6b..00000000000 --- a/modelopt_recipes/huggingface/phi4mm/ptq/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Phi-4-Multimodal PTQ recipes - -Phi-4-Multimodal is a multimodal model. Quantization should be applied only to -the language model; the speech, audio, image, and vision branches are kept in -full precision to avoid accuracy regressions on those modalities. - -| File | What's model-specific | -|------|-----------------------| -| `disabled_quantizers.yaml` | Reusable unit (`QuantizerCfgListConfig`). Merges the standard `default_disabled_quantizers` exclusions with Phi-4-MM ones (`*speech*`, `*audio*`, `*image*`, `*vision*`). Imported by recipes below as the single `disabled_quantizers` slot so they don't pull in two disabled-quantizer sets. | -| `nvfp4-kv_fp8_cast.yaml` | NVFP4 W4A4 model quantization + FP8 KV-cache cast (constant amax, no KV calibration). Identical numerics to the general `nvfp4` preset / `kv_fp8_cast` unit; what makes it model-specific is that it imports `disabled_quantizers.yaml` from this folder to skip the non-language branches. | - -Additional `-kv_fp8_cast.yaml` recipes can be generated for other formats -if needed; only `nvfp4-kv_fp8_cast.yaml` is shipped by default. diff --git a/modelopt_recipes/huggingface/phi4mm/ptq/disabled_quantizers.yaml b/modelopt_recipes/huggingface/phi4mm/ptq/disabled_quantizers.yaml deleted file mode 100644 index 1c6089f087f..00000000000 --- a/modelopt_recipes/huggingface/phi4mm/ptq/disabled_quantizers.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# QuantizerCfgList snippet of disabled quantizers for Phi-4-Multimodal. -# Splices in the standard `default_disabled_quantizers` exclusions and appends -# Phi-4-MM-specific ones so that only the language model is quantized; -# speech/audio/image/vision branches are skipped. Recipes that import this -# should NOT also import `default_disabled_quantizers`. - -# modelopt-schema: modelopt.torch.quantization.config.QuantizerCfgListConfig -imports: - default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers ---- - - $import: default_disabled_quantizers - - quantizer_name: '*speech*' - enable: false - - quantizer_name: '*audio*' - enable: false - - quantizer_name: '*image*' - enable: false - - quantizer_name: '*vision*' - enable: false diff --git a/modelopt_recipes/huggingface/phi4mm/ptq/nvfp4-kv_fp8_cast.yaml b/modelopt_recipes/huggingface/phi4mm/ptq/nvfp4-kv_fp8_cast.yaml deleted file mode 100644 index dfb1be1778d..00000000000 --- a/modelopt_recipes/huggingface/phi4mm/ptq/nvfp4-kv_fp8_cast.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Phi-4-Multimodal-specific PTQ recipe for the `nvfp4` quantization format. -# Equivalent to the general `nvfp4` preset with quantization disabled -# on non-language branches. - -imports: - base_disable_all: configs/ptq/units/base_disable_all - w4a4_nvfp4_nvfp4: configs/ptq/units/w4a4_nvfp4_nvfp4 - disabled_quantizers: huggingface/phi4mm/ptq/disabled_quantizers - kv_fp8_cast: configs/ptq/units/kv_fp8_cast - -metadata: - recipe_type: ptq - description: 'Phi-4-Multimodal PTQ recipe (nvfp4): same numerics as the general nvfp4 preset, applied to the language model only (speech, audio, image, - and vision branches are skipped).' -quantize: - algorithm: max - quant_cfg: - - $import: base_disable_all - - $import: w4a4_nvfp4_nvfp4 - - $import: kv_fp8_cast - - $import: disabled_quantizers diff --git a/modelopt_recipes/ptq.md b/modelopt_recipes/ptq.md index 54ff6511489..3d56c761adf 100644 --- a/modelopt_recipes/ptq.md +++ b/modelopt_recipes/ptq.md @@ -234,7 +234,7 @@ that baseline. The deviations come in four kinds: |------|-------------------------------------|----------| | **Architecture-aware `quant_cfg`** | Per-sub-module format choices a single wildcard scheme can't express | `minimax_m3_vl`, `qwen3_5`, `qwen3_5_moe`, `vit`, `nemotron_llama` | | **Algorithm override** | Same numerics & scope, but the *calibration algorithm* is tweaked because the default breaks or regresses | `gemma`, `gemma4`, `mpt` | -| **Extra exclusions** | Adds disabled-quantizer patterns so non-language branches stay full precision | `nemotron_vl`, `phi4mm`, `diffusion_gemma` | +| **Extra exclusions** | Adds disabled-quantizer patterns so non-language branches stay full precision | `nemotron_vl`, `diffusion_gemma` | | **Checkpoint mirror** | A mixed-precision map reproducing one published checkpoint exactly | `models/nvidia/Nemotron-3-*`, `models/nvidia/Mistral-Medium-3.5-128B-NVFP4` | The numerics and standard exclusions are still inherited from `configs/` @@ -314,7 +314,7 @@ These quantize the **same layers** as the general recipes; only the *Why special:* identical scope/numerics to a general scheme, but a general recipe's default algorithm would overflow or regress here. -### Extra exclusions — `nemotron_vl`, `phi4mm`, `diffusion_gemma` +### Extra exclusions — `nemotron_vl`, `diffusion_gemma` Each of these is **numerically identical** to a general recipe. What makes them special is a model-local `disabled_quantizers.yaml` unit that *extends* the @@ -324,8 +324,6 @@ standard exclusions so a model-specific branch stays in full precision: `nvfp4_default-kv_fp8_cast` numerics, adding `*vision*`, `*image*`, `*radio*`, `*visual*`, `*encoder*`, `*model_encoder*` so only the language decoder is quantized. -- **`phi4mm`** (Phi-4-Multimodal) — general `nvfp4_default-kv_fp8_cast` - numerics, adding `*speech*`, `*audio*`, `*image*`, `*vision*`. - **`diffusion_gemma`** (block-diffusion encoder-decoder text LLM on a Gemma4 MoE backbone) — general `nvfp4_experts_only-kv_fp8_cast` numerics, adding `*self_conditioning*`: the self-conditioning network is text-only and never From 876dd8c6f75a99f0e8aa1b02c5ecae6022df898c Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Fri, 7 Aug 2026 22:09:32 +0000 Subject: [PATCH 03/10] Move NVBug 6563509 changelog entries into 0.46 0.46 is the release these land in, not the still-open 0.47 section. The Phi drop goes under Backward Breaking Changes next to the VILA / NVILA entry, which removed model support for the same reason (bundled remote code pinned below our transformers floor); the skeleton fallback goes under Bug Fixes. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a7d522d213b..69e501580ac 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,12 +19,9 @@ Changelog **Deprecations** -- Drop PTQ support for **Phi-3-vision** and **Phi-4-multimodal**. Their bundled remote code predates Transformers v5 and no longer loads on the versions this repo requires (``transformers>=4.57``): Phi-4-multimodal needs ``transformers<4.52`` because it reaches ``prepare_inputs_for_generation`` through ``peft``, which requires ``PreTrainedModel`` to still inherit ``GenerationMixin``, and both models declare ``_tied_weights_keys`` as a list, which Transformers 5.x rejects. Removes the ``phi4mm`` model type, its multimodal-detection heuristics (``vision_lora`` / ``audio_processor`` / ``embd_layer.image_embd_layer``), the ``Phi3Image`` / ``PhiImage`` embedding-export exclusions, and the ``modelopt_recipes/huggingface/phi4mm/`` recipes. - **Bug Fixes** - Fix EAGLE-3 training with context parallelism (``--cp_size > 1`` in ``examples/speculative_decoding``), which failed to start on ``accelerate >= 1.13`` and then raised ``got mixed torch.Tensor and DTensor``. -- ``examples/hf_ptq/hf_ptq.py`` no longer aborts the whole load when the throwaway meta-device skeleton it builds to size ``infer_auto_device_map`` cannot be constructed. ``init_empty_weights(include_buffers=True)`` pushes a global ``torch.device("meta")`` context, so remote-code checkpoints that derive scalar hyperparameters from real tensors in ``__init__`` raised ``Tensor.item() cannot be called on meta tensors``. The skeleton is now retried without the global meta context, and if that also fails the memory estimate is skipped with a warning instead of failing the run. 0.46 (2026-08-17) ^^^^^^^^^^^^^^^^^ @@ -86,6 +83,7 @@ Changelog - Remove the deprecated ``examples/llm_autodeploy`` example (deprecated in 0.45). Use TensorRT-LLM's `AutoDeploy `_ directly together with ModelOpt PTQ in ``examples/hf_ptq``. - Remove the deprecated ``examples/llm_qad`` Megatron-LM QAD example (deprecated in 0.45). Use the `megatron_bridge QAD example `_ instead, which provides a simpler Python-based interface and better model coverage. - Dropped VILA / NVILA vision-language model support in ``examples/hf_ptq``. VILA's modeling code requires ``transformers<=4.50.0``, which conflicts with ModelOpt's minimum supported ``transformers`` version. The VILA-specific bootstrap (repo clone, ``requirements-vila.txt``) and loading paths in ``example_utils.py`` have been removed. +- Dropped **Phi-3-vision** and **Phi-4-multimodal** PTQ support in ``examples/hf_ptq`` (NVBug 6563509). Their bundled remote code predates Transformers v5 and no longer loads on any version ModelOpt supports (``transformers>=4.57``): Phi-4-multimodal requires ``transformers<4.52`` because it reaches ``prepare_inputs_for_generation`` through ``peft``, which needs ``PreTrainedModel`` to still inherit ``GenerationMixin``, and both models declare ``_tied_weights_keys`` as a list, which Transformers 5.x rejects. The support-matrix row, the ``phi4mm`` model type, the multimodal-detection heuristics that only ever matched these two (``vision_lora`` / ``audio_processor`` / ``embd_layer.image_embd_layer``), the ``Phi3Image`` / ``PhiImage`` embedding-export exclusions, and the ``modelopt_recipes/huggingface/phi4mm/`` recipes have been removed. Text-only Phi-3/Phi-4 and Phi-3.5-MoE are unaffected. **Deprecations** @@ -108,6 +106,7 @@ Changelog - Fix ``examples/vllm_serve`` serving shared experts uncalibrated: their ``gate_proj``/``up_proj`` quantizer keys were not merged into ``gate_up_proj`` on reload, so they matched no module and were dropped. - Fix Qwen3-VL MoE PTQ failing on ``transformers>=5.12`` with ``AttributeError: 'QuantQwen3VLMoeTextExperts' object has no attribute 'hidden_size'`` (NVBug 6518551). transformers 5.12 moved ``Qwen3VLMoeTextExperts`` onto the standard ``@use_experts_implementation`` fused layout (``hidden_size``/``expert_dim`` renamed to ``hidden_dim``/``intermediate_dim``, ``gate_up_proj`` transposed to ``(num_experts, 2*intermediate_dim, hidden_dim)``, two ``F.linear`` calls per expert), but the legacy ``_QuantQwen3VLMoeTextExperts`` wrapper stayed statically registered and shadowed on-the-fly detection. The new layout is now left to ``register_fused_experts_on_the_fly``, which claims it with the generic ``_QuantFusedExperts``; the legacy wrapper is still registered on ``transformers<5.12``, whose ``torch.bmm``-based forward the generic wrapper cannot intercept. - Fix ``examples/hf_ptq`` multi-node FSDP2 export (``--use_fsdp2``) failing with ``RuntimeError: Cannot set version_counter for inference tensor``. ``export_quantized`` wrapped its whole body in ``torch.inference_mode()``, so the full params gathered by ``get_model_state_dict(full_state_dict=True)`` were inference tensors and the subsequent ``state_dict()`` -> ``param.detach()`` could not set their version counter. The export context is now ``torch.no_grad()``, which still disables autograd but keeps the gathered params as normal tensors. +- Fix ``examples/hf_ptq`` aborting the whole model load when the throwaway meta-device skeleton it builds to size ``infer_auto_device_map`` cannot be constructed (NVBug 6563509). ``init_empty_weights(include_buffers=True)`` pushes a global ``torch.device("meta")`` context, so remote-code checkpoints that derive scalar hyperparameters from real tensors in ``__init__`` failed with ``Tensor.item() cannot be called on meta tensors`` before ``from_pretrained`` was ever reached. The skeleton is now retried without the global meta context, and if that also fails the memory estimate is skipped with a warning instead of failing the run; ``--use_seq_device_map`` and ``--gpu_max_memory_percentage`` cover the lost heuristic. - Fix unified HF export of multimodal models whose vision tower carries its own ``PrefixChange`` conversion (``LlavaForConditionalGeneration`` on ``transformers>=5.12`` — NVBug 6525511). transformers collects conversion mappings recursively and scopes each sub-model's transforms to that sub-module via ``scope_prefix``, matching only keys under that prefix. ModelOpt's quant-aware reverse conversion read the raw patterns and ignored ``scope_prefix``, so the vision tower's "add a ``vision_model.`` prefix" rule was applied to *every* key in the state dict: an exported llava-1.5-13b checkpoint had all 758 tensors moved under a bogus top-level ``vision_model.`` namespace (``vision_model.language_model.*``, ``vision_model.lm_head.*``), and vLLM rejected it with ``ValueError: There is no module or parameter named 'vision_model' in LlavaForConditionalGeneration``. Reverse rename rules now carry their scope and are applied only to keys under it, matching transformers' own ``WeightTransform._scoped_match`` semantics. ``Gemma3ForConditionalGeneration`` was affected identically and is fixed by the same change. 0.45 (2026-07-02) From 8444c11f6bba343e25830658571aefe53fc7672e Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Sat, 8 Aug 2026 05:55:59 +0000 Subject: [PATCH 04/10] Address PR review: extract skeleton helper, fix OOM hint, drop README note - Extract the two-attempt skeleton build into `_build_meta_skeleton()` next to the other `get_model` helpers (@cjluo-nv). `get_model` now reads as one line, and the fallback logic is unit-testable in isolation. - Fix the OOM hint in the fallback warning. It named `--gpu_max_memory_percentage`, which does not exist -- the flag is `--gpu_max_mem_percentage` -- and on its own that flag has no effect when the skeleton fails: `model_kwargs["max_memory"]` is only set by the `_disk_offload` and `use_seq_device_map` paths, so the skipped `infer_auto_device_map` branch never applies it. Point at `--use_seq_device_map` (which does apply the percentage) and `--batch_size` instead (@coderabbitai). Not adopting the suggested unconditional cap: the original code only shrinks `max_memory` when `infer_auto_device_map` reports a CPU spill, so capping whenever the skeleton fails would force CPU offload onto models that currently fit entirely on GPU. - Drop the support-matrix explanation note; the CHANGELOG covers it (@cjluo-nv). Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- examples/hf_ptq/README.md | 7 ---- examples/hf_ptq/example_utils.py | 57 ++++++++++++++++++-------------- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/examples/hf_ptq/README.md b/examples/hf_ptq/README.md index 903d708de9b..dc852408c7a 100755 --- a/examples/hf_ptq/README.md +++ b/examples/hf_ptq/README.md @@ -125,13 +125,6 @@ Please reference our [framework scripts](#framework-scripts) and our [docs](http > *This is a subset of the models supported. For the full list please check the [TensorRT-LLM support matrix](https://nvidia.github.io/TensorRT-LLM/reference/precision.html#support-matrix)* -> *Phi-3-vision and Phi-4-multimodal were dropped from this matrix: their bundled -> remote code predates Transformers v5 and no longer loads on the versions this repo -> requires (`transformers>=4.57`). Phi-4-multimodal needs `transformers<4.52` — it -> reaches `prepare_inputs_for_generation` through `peft`, which requires -> `PreTrainedModel` to still inherit `GenerationMixin` — and both models declare -> `_tied_weights_keys` as a list, which Transformers 5.x rejects.* - > *1.The w4a8_awq is an experimental quantization scheme that may result in a higher accuracy penalty.* \ > *2.For some models, there is only support for exporting quantized checkpoints.* \ > *3.W4A8_AWQ is only available on some models but not all* \ diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index 6f9fe51063e..cb02fbb3408 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -657,6 +657,36 @@ def _resolve_init_config(hf_config, auto_model_module, ckpt_path, config_kwargs) return hf_config +def _build_meta_skeleton(from_config, config_for_init, model_kwargs, architecture): + """Build a throwaway meta-device model used only to size ``infer_auto_device_map``. + + ``include_buffers=True`` makes accelerate push a global ``torch.device("meta")`` + context, so *every* tensor built in ``__init__`` lands on meta -- not just parameters + and buffers. Remote-code checkpoints written before Transformers v5 routinely derive + scalar hyperparameters from real tensors there, which raises on meta, so fall back to + the parameter-only patching of ``include_buffers=False``. + + Returns ``None`` (after warning) when neither attempt works; the caller then skips the + memory estimate rather than failing a load ``from_pretrained`` can still do. + """ + skeleton_errors = [] + for include_buffers in (True, False): + try: + with init_empty_weights(include_buffers=include_buffers): + return from_config(config_for_init, **model_kwargs) + except Exception as e: # noqa: PERF203 -- at most two attempts, error path only + skeleton_errors.append(f"include_buffers={include_buffers}: {e!r}") + + warnings.warn( + f"Could not build a meta-device skeleton of {architecture} " + f"({'; '.join(skeleton_errors)}). Skipping the device-map memory estimate and " + "letting from_pretrained map the model. If you hit GPU OOM, rerun with " + "--use_seq_device_map (which applies --gpu_max_mem_percentage) or lower " + "--batch_size." + ) + return None + + def _get_config_dtype(config): config_dtype = ( getattr(config, "dtype", None) or getattr(config, "torch_dtype", None) or torch.bfloat16 @@ -877,30 +907,9 @@ def has_pack_quantized_config(config): model_kwargs2.pop("trust_remote_code", None) model_kwargs2.pop("max_memory", None) - # This skeleton is only a sizing aid for ``infer_auto_device_map``; it is thrown - # away right after. ``include_buffers=True`` makes accelerate push a global - # ``torch.device("meta")`` context, so *every* tensor built in ``__init__`` lands - # on meta -- not just parameters and buffers. Remote-code checkpoints written - # before Transformers v5 often compute scalar hyperparameters from real tensors - # there (Phi-4-multimodal's conformer subsampling does ``int(torch.tensor(...))``), - # which raises on meta. Retry without the global context, then give up on the - # estimate rather than failing a load that ``from_pretrained`` can still do. - model = None - skeleton_errors = [] - for include_buffers in (True, False): - try: - with init_empty_weights(include_buffers=include_buffers): - model = from_config(config_for_init, **model_kwargs2) - break - except Exception as e: - skeleton_errors.append(f"include_buffers={include_buffers}: {e!r}") - if model is None: - warnings.warn( - f"Could not build a meta-device skeleton of {architecture} " - f"({'; '.join(skeleton_errors)}). Skipping the device-map memory estimate " - "and letting from_pretrained map the model; if you hit GPU OOM, pass " - "--use_seq_device_map or lower --gpu_max_memory_percentage." - ) + # Only a sizing aid for ``infer_auto_device_map`` below; ``None`` when the model + # cannot be built on meta, in which case the estimate is skipped. + model = _build_meta_skeleton(from_config, config_for_init, model_kwargs2, architecture) max_memory = get_max_memory() From 1b062da365e3029dae96a876683f9177956487b3 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Sat, 8 Aug 2026 22:15:10 +0000 Subject: [PATCH 05/10] Simplify sizing skeleton to a single include_buffers=False build The probe exists only to answer one boolean -- will this model spill to CPU, so should max_memory get the gpu_max_mem_percentage haircut. `infer_auto_device_map` gets there via `compute_module_sizes`, which reads `tensor.numel()` and `tensor.dtype` and never touches storage. So the probe only has to reproduce the module tree, and it must be no stricter than the loader it predicts. `include_buffers=True` was stricter than both loaders: - Transformers 4.x `from_pretrained` builds under `init_empty_weights()`, whose `include_buffers` defaults to False (modeling_utils.py:4378). - Transformers 5.x builds under `torch.device("meta")` plus `meta_device_safe_creation_ops()`, which redirects `torch.linspace` to CPU precisely so pre-v5 remote code that derives scalars in `__init__` keeps working. accelerate implements `include_buffers=True` as a bare global `torch.device("meta")` context, which also captures scratch arithmetic in `__init__` that has nothing to do with weights. A model doing `int(torch.tensor(...))` there loads fine through `from_pretrained` on both 4.x and 5.x but died in our probe -- an optimization killing runs it was only meant to speed up. Dropping to a single `include_buffers=False` build removes the retry loop, the PERF203 waiver, and the two-attempt error accumulation. Measured cost on real checkpoints is nil: Qwen3-8B and DeepSeek-R1-Distill-Llama-70B both report 0.0 MiB retained and identical sized totals (15.256 / 131.417 GiB) either way, for +0.09s and +0.19s respectively. Adds the coverage requested in review: the probe uses permissive patching, it survives a meta-hostile `__init__`, it returns None and warns on failure, and `get_model` skips `infer_auto_device_map` while still calling `from_pretrained` without inventing a max_memory cap. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 2 +- examples/hf_ptq/example_utils.py | 53 ++++++------ tests/examples/hf_ptq/test_example_utils.py | 90 +++++++++++++++++++++ 3 files changed, 120 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 69e501580ac..c3c3976e51a 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -106,7 +106,7 @@ Changelog - Fix ``examples/vllm_serve`` serving shared experts uncalibrated: their ``gate_proj``/``up_proj`` quantizer keys were not merged into ``gate_up_proj`` on reload, so they matched no module and were dropped. - Fix Qwen3-VL MoE PTQ failing on ``transformers>=5.12`` with ``AttributeError: 'QuantQwen3VLMoeTextExperts' object has no attribute 'hidden_size'`` (NVBug 6518551). transformers 5.12 moved ``Qwen3VLMoeTextExperts`` onto the standard ``@use_experts_implementation`` fused layout (``hidden_size``/``expert_dim`` renamed to ``hidden_dim``/``intermediate_dim``, ``gate_up_proj`` transposed to ``(num_experts, 2*intermediate_dim, hidden_dim)``, two ``F.linear`` calls per expert), but the legacy ``_QuantQwen3VLMoeTextExperts`` wrapper stayed statically registered and shadowed on-the-fly detection. The new layout is now left to ``register_fused_experts_on_the_fly``, which claims it with the generic ``_QuantFusedExperts``; the legacy wrapper is still registered on ``transformers<5.12``, whose ``torch.bmm``-based forward the generic wrapper cannot intercept. - Fix ``examples/hf_ptq`` multi-node FSDP2 export (``--use_fsdp2``) failing with ``RuntimeError: Cannot set version_counter for inference tensor``. ``export_quantized`` wrapped its whole body in ``torch.inference_mode()``, so the full params gathered by ``get_model_state_dict(full_state_dict=True)`` were inference tensors and the subsequent ``state_dict()`` -> ``param.detach()`` could not set their version counter. The export context is now ``torch.no_grad()``, which still disables autograd but keeps the gathered params as normal tensors. -- Fix ``examples/hf_ptq`` aborting the whole model load when the throwaway meta-device skeleton it builds to size ``infer_auto_device_map`` cannot be constructed (NVBug 6563509). ``init_empty_weights(include_buffers=True)`` pushes a global ``torch.device("meta")`` context, so remote-code checkpoints that derive scalar hyperparameters from real tensors in ``__init__`` failed with ``Tensor.item() cannot be called on meta tensors`` before ``from_pretrained`` was ever reached. The skeleton is now retried without the global meta context, and if that also fails the memory estimate is skipped with a warning instead of failing the run; ``--use_seq_device_map`` and ``--gpu_max_memory_percentage`` cover the lost heuristic. +- Fix ``examples/hf_ptq`` aborting the whole model load when the throwaway meta-device skeleton it builds to size ``infer_auto_device_map`` cannot be constructed (NVBug 6563509). The skeleton used ``init_empty_weights(include_buffers=True)``, which accelerate implements as a bare global ``torch.device("meta")`` context: *every* tensor built in ``__init__`` lands on meta, so remote-code checkpoints that derive scalar hyperparameters from real tensors there failed with ``Tensor.item() cannot be called on meta tensors`` before ``from_pretrained`` was ever reached. That made the probe stricter than the loader it predicts -- Transformers 4.x builds under ``include_buffers=False`` and Transformers 5.x under a meta context patched by ``meta_device_safe_creation_ops()`` -- so it could kill loads that would otherwise have succeeded. The skeleton now uses ``include_buffers=False``, which patches only ``nn.Module.register_parameter`` and leaves ``__init__`` arithmetic on a real device (module sizes are unchanged: ``compute_module_sizes`` reads shape and dtype, not storage). If it still cannot be built, the memory estimate is skipped with a warning instead of failing the run; ``--use_seq_device_map`` (which applies ``--gpu_max_mem_percentage``) and ``--batch_size`` cover the lost heuristic. - Fix unified HF export of multimodal models whose vision tower carries its own ``PrefixChange`` conversion (``LlavaForConditionalGeneration`` on ``transformers>=5.12`` — NVBug 6525511). transformers collects conversion mappings recursively and scopes each sub-model's transforms to that sub-module via ``scope_prefix``, matching only keys under that prefix. ModelOpt's quant-aware reverse conversion read the raw patterns and ignored ``scope_prefix``, so the vision tower's "add a ``vision_model.`` prefix" rule was applied to *every* key in the state dict: an exported llava-1.5-13b checkpoint had all 758 tensors moved under a bogus top-level ``vision_model.`` namespace (``vision_model.language_model.*``, ``vision_model.lm_head.*``), and vLLM rejected it with ``ValueError: There is no module or parameter named 'vision_model' in LlavaForConditionalGeneration``. Reverse rename rules now carry their scope and are applied only to keys under it, matching transformers' own ``WeightTransform._scoped_match`` semantics. ``Gemma3ForConditionalGeneration`` was affected identically and is fixed by the same change. 0.45 (2026-07-02) diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index cb02fbb3408..d14784f8dcd 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -660,31 +660,36 @@ def _resolve_init_config(hf_config, auto_model_module, ckpt_path, config_kwargs) def _build_meta_skeleton(from_config, config_for_init, model_kwargs, architecture): """Build a throwaway meta-device model used only to size ``infer_auto_device_map``. - ``include_buffers=True`` makes accelerate push a global ``torch.device("meta")`` - context, so *every* tensor built in ``__init__`` lands on meta -- not just parameters - and buffers. Remote-code checkpoints written before Transformers v5 routinely derive - scalar hyperparameters from real tensors there, which raises on meta, so fall back to - the parameter-only patching of ``include_buffers=False``. - - Returns ``None`` (after warning) when neither attempt works; the caller then skips the - memory estimate rather than failing a load ``from_pretrained`` can still do. + ``compute_module_sizes`` needs shapes and dtypes, never values or storage, so this + probe only has to reproduce the module tree. It must also be *no stricter than the + loader it predicts*, or it kills runs ``from_pretrained`` would have completed: + + - Transformers 4.x builds under ``init_empty_weights()``, i.e. ``include_buffers=False``. + - Transformers 5.x builds under ``torch.device("meta")`` plus + ``meta_device_safe_creation_ops()``, which redirects ``torch.linspace`` to CPU so + remote code that derives scalars from it in ``__init__`` keeps working. + + ``include_buffers=True`` is stricter than both: accelerate implements it as a bare + global ``torch.device("meta")`` context, so *every* tensor built in ``__init__`` lands + on meta and any ``int(torch.tensor(...))`` raises. ``include_buffers=False`` patches + only ``nn.Module.register_parameter``, leaving ``__init__`` arithmetic on a real + device; buffers stay materialized, which costs nothing here because their shape and + dtype size the same either way. + + Returns ``None`` (after warning) if the model cannot be built at all; the caller then + skips the estimate rather than failing a load ``from_pretrained`` can still do. """ - skeleton_errors = [] - for include_buffers in (True, False): - try: - with init_empty_weights(include_buffers=include_buffers): - return from_config(config_for_init, **model_kwargs) - except Exception as e: # noqa: PERF203 -- at most two attempts, error path only - skeleton_errors.append(f"include_buffers={include_buffers}: {e!r}") - - warnings.warn( - f"Could not build a meta-device skeleton of {architecture} " - f"({'; '.join(skeleton_errors)}). Skipping the device-map memory estimate and " - "letting from_pretrained map the model. If you hit GPU OOM, rerun with " - "--use_seq_device_map (which applies --gpu_max_mem_percentage) or lower " - "--batch_size." - ) - return None + try: + with init_empty_weights(include_buffers=False): + return from_config(config_for_init, **model_kwargs) + except Exception as e: + warnings.warn( + f"Could not build a meta-device skeleton of {architecture} ({e!r}). " + "Skipping the device-map memory estimate and letting from_pretrained map the " + "model. If you hit GPU OOM, rerun with --use_seq_device_map (which applies " + "--gpu_max_mem_percentage) or lower --batch_size." + ) + return None def _get_config_dtype(config): diff --git a/tests/examples/hf_ptq/test_example_utils.py b/tests/examples/hf_ptq/test_example_utils.py index 4b9120b0441..6175066f5c2 100644 --- a/tests/examples/hf_ptq/test_example_utils.py +++ b/tests/examples/hf_ptq/test_example_utils.py @@ -471,3 +471,93 @@ def from_pretrained(*args, **kwargs): example_utils.get_model("checkpoint", device="cpu", trust_remote_code=trust_remote_code) assert used["path"] == ("bundled" if expect_bundled_code else "builtin") + + +# ---------- _build_meta_skeleton ---------------------------------------------- + + +def test_build_meta_skeleton_uses_permissive_patching(): + """The probe must be no stricter than the loader it predicts. + + ``include_buffers=True`` is a bare global ``torch.device("meta")`` context, which + also captures scratch arithmetic in ``__init__``; ``from_pretrained`` never does + that, so the probe must not either. + """ + seen = {} + + def fake_init_empty_weights(include_buffers): + seen["include_buffers"] = include_buffers + return nullcontext() + + sentinel = object() + with patch.object(example_utils, "init_empty_weights", fake_init_empty_weights): + out = example_utils._build_meta_skeleton( + lambda cfg, **kw: sentinel, "cfg", {"dtype": torch.bfloat16}, "Arch" + ) + + assert out is sentinel + assert seen["include_buffers"] is False + + +def test_build_meta_skeleton_survives_meta_hostile_init(): + """Remote code that reads a real scalar in ``__init__`` must still build.""" + + def from_config(cfg, **kwargs): + # Mirrors Phi-4-multimodal's conformer: int() on a freshly built tensor. This + # raises under a global meta context but works with parameter-only patching. + return SimpleNamespace(width=int(torch.tensor(80.0))) + + model = example_utils._build_meta_skeleton(from_config, "cfg", {}, "Arch") + + assert model.width == 80 + + +def test_build_meta_skeleton_returns_none_and_warns_on_failure(): + def from_config(cfg, **kwargs): + raise RuntimeError("boom") + + with pytest.warns(UserWarning, match="Could not build a meta-device skeleton of Arch"): + assert example_utils._build_meta_skeleton(from_config, "cfg", {}, "Arch") is None + + +def test_get_model_skips_device_map_estimate_when_skeleton_fails(monkeypatch): + """A failed probe must not abort the load: skip sizing, still call from_pretrained.""" + calls = {} + hf_config = SimpleNamespace( + architectures=["LlamaForCausalLM"], + dtype=torch.float16, + model_type="llama", + torch_dtype=torch.bfloat16, + ) + + class FakeModel: + def eval(self): + calls["eval"] = True + + class FakeLlamaForCausalLM: + @staticmethod + def _from_config(config, **kwargs): + raise RuntimeError("Tensor.item() cannot be called on meta tensors") + + @staticmethod + def from_pretrained(*args, **kwargs): + calls["from_pretrained"] = kwargs + return FakeModel() + + def _boom_infer(model, max_memory): + raise AssertionError("infer_auto_device_map must be skipped without a skeleton") + + monkeypatch.setattr(example_utils.AutoConfig, "from_pretrained", lambda *a, **kw: hf_config) + monkeypatch.setattr(example_utils.transformers, "LlamaForCausalLM", FakeLlamaForCausalLM) + monkeypatch.setattr(example_utils, "is_nemotron_vl", lambda config: False) + monkeypatch.setattr(example_utils, "is_speculative", lambda config: False) + monkeypatch.setattr(example_utils, "get_max_memory", lambda: {0: 1024}) + monkeypatch.setattr(example_utils, "infer_auto_device_map", _boom_infer) + + with pytest.warns(UserWarning, match="Skipping the device-map memory estimate"): + model = example_utils.get_model("checkpoint", device="cpu", trust_remote_code=True) + + assert isinstance(model, FakeModel) + assert calls["eval"] + # The estimate is the only thing lost; no memory cap is invented for the load. + assert "max_memory" not in calls["from_pretrained"] From 17dfd3b7a412dcc7e3b38d39b63fa0fcd8f214e6 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 10 Aug 2026 16:46:09 +0000 Subject: [PATCH 06/10] Clarify in the changelog why Phi-3-vision is dropped Phi-4-multimodal is dropped because it cannot load on any supported transformers version. Phi-3-vision is dropped because it is the superseded predecessor in the same family -- with the successor unsupportable there is no reason to keep the older model -- not because an equivalent 4.57 repro exists for it. The previous wording ran both models' rationale together and implied Phi-3-vision had been shown unloadable across the whole supported range, which overstates what was verified: it shares the list-valued _tied_weights_keys defect (so is broken on Transformers 5.x) but does not hit the peft/GenerationMixin blocker. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c3c3976e51a..4876cf837a0 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -83,7 +83,7 @@ Changelog - Remove the deprecated ``examples/llm_autodeploy`` example (deprecated in 0.45). Use TensorRT-LLM's `AutoDeploy `_ directly together with ModelOpt PTQ in ``examples/hf_ptq``. - Remove the deprecated ``examples/llm_qad`` Megatron-LM QAD example (deprecated in 0.45). Use the `megatron_bridge QAD example `_ instead, which provides a simpler Python-based interface and better model coverage. - Dropped VILA / NVILA vision-language model support in ``examples/hf_ptq``. VILA's modeling code requires ``transformers<=4.50.0``, which conflicts with ModelOpt's minimum supported ``transformers`` version. The VILA-specific bootstrap (repo clone, ``requirements-vila.txt``) and loading paths in ``example_utils.py`` have been removed. -- Dropped **Phi-3-vision** and **Phi-4-multimodal** PTQ support in ``examples/hf_ptq`` (NVBug 6563509). Their bundled remote code predates Transformers v5 and no longer loads on any version ModelOpt supports (``transformers>=4.57``): Phi-4-multimodal requires ``transformers<4.52`` because it reaches ``prepare_inputs_for_generation`` through ``peft``, which needs ``PreTrainedModel`` to still inherit ``GenerationMixin``, and both models declare ``_tied_weights_keys`` as a list, which Transformers 5.x rejects. The support-matrix row, the ``phi4mm`` model type, the multimodal-detection heuristics that only ever matched these two (``vision_lora`` / ``audio_processor`` / ``embd_layer.image_embd_layer``), the ``Phi3Image`` / ``PhiImage`` embedding-export exclusions, and the ``modelopt_recipes/huggingface/phi4mm/`` recipes have been removed. Text-only Phi-3/Phi-4 and Phi-3.5-MoE are unaffected. +- Dropped **Phi-4-multimodal** PTQ support in ``examples/hf_ptq`` (NVBug 6563509). Its bundled remote code predates Transformers v5 and no longer loads on any version ModelOpt supports (``transformers>=4.57``): it requires ``transformers<4.52`` because it reaches ``prepare_inputs_for_generation`` through ``peft``, which needs ``PreTrainedModel`` to still inherit ``GenerationMixin``, and it declares ``_tied_weights_keys`` as a list, which Transformers 5.x rejects. **Phi-3-vision** is dropped alongside it: it is the older, superseded model in the same family, so with its successor unsupportable there is no reason to keep carrying the predecessor. (Phi-3-vision shares the list-valued ``_tied_weights_keys`` defect and so is likewise broken on Transformers 5.x, though it does not hit the ``peft`` blocker.) The support-matrix row, the ``phi4mm`` model type, the multimodal-detection heuristics that only ever matched these two (``vision_lora`` / ``audio_processor`` / ``embd_layer.image_embd_layer``), the ``Phi3Image`` / ``PhiImage`` embedding-export exclusions, and the ``modelopt_recipes/huggingface/phi4mm/`` recipes have been removed. Text-only Phi-3/Phi-4 and Phi-3.5-MoE are unaffected. **Deprecations** From 4e0157d0f8ea26e7468352bdfc74e33be0be73b2 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 10 Aug 2026 19:04:49 +0000 Subject: [PATCH 07/10] Size the device map from the checkpoint instead of a constructed model get_model builds a device map to leave GPU headroom for calibration activations, and needs one boolean to do it: will the weights fit on the GPUs? It answered that by constructing a throwaway meta-device model and running infer_auto_device_map on it, then discarding the map and keeping only `"cpu" in .values()`. Constructing a model to learn its size means running the checkpoint's __init__, which is where this bug lives: remote code that derives a scalar from a real tensor (Phi-4-multimodal's conformer does int(torch.tensor(...))) cannot run under a meta device context. Both Transformers 4.x and 5.x load those models fine, so an optimization was killing loads it existed only to speed up. The number is already on disk. `metadata.total_size` in the safetensors index is the same sum `compute_module_sizes` computes -- exactly, verified on Qwen3-8B (16381470720 B = 15.256 GiB) and DeepSeek-R1-Distill-Llama-70B (141107412992 B = 131.417 GiB), matching what the meta model reported. `_checkpoint_size_bytes` reads it, falling back to summing shard sizes, and returns None when the layout is unreadable (no cap applied, as before). This removes the failure mode rather than tolerating it, and with it the retry/fallback machinery it needed: `_build_meta_skeleton`, its warning path, the `del` that kept the probe's real buffers from overlapping the checkpoint load, the disk-offload special case, `_resolve_init_config` and the `from_config` resolution that fed it, and the accelerate `init_empty_weights` / `infer_auto_device_map` imports. Net 74 lines lighter. config_dtype now comes from hf_config rather than the re-derived built-in config; both read the same config.json dtype field. Tests cover the index-metadata path, the shard-sum fallback, a malformed index, capping only when the weights exceed the GPU budget, and no cap when the size is unknown. Verified end to end that Phi-4-multimodal now sizes at 10.383 GiB and reaches from_pretrained without a meta-tensor error. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 2 +- examples/hf_ptq/example_utils.py | 107 ++++------- tests/examples/hf_ptq/test_example_utils.py | 201 ++++++++------------ 3 files changed, 118 insertions(+), 192 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4876cf837a0..33f61eae4d5 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -106,7 +106,7 @@ Changelog - Fix ``examples/vllm_serve`` serving shared experts uncalibrated: their ``gate_proj``/``up_proj`` quantizer keys were not merged into ``gate_up_proj`` on reload, so they matched no module and were dropped. - Fix Qwen3-VL MoE PTQ failing on ``transformers>=5.12`` with ``AttributeError: 'QuantQwen3VLMoeTextExperts' object has no attribute 'hidden_size'`` (NVBug 6518551). transformers 5.12 moved ``Qwen3VLMoeTextExperts`` onto the standard ``@use_experts_implementation`` fused layout (``hidden_size``/``expert_dim`` renamed to ``hidden_dim``/``intermediate_dim``, ``gate_up_proj`` transposed to ``(num_experts, 2*intermediate_dim, hidden_dim)``, two ``F.linear`` calls per expert), but the legacy ``_QuantQwen3VLMoeTextExperts`` wrapper stayed statically registered and shadowed on-the-fly detection. The new layout is now left to ``register_fused_experts_on_the_fly``, which claims it with the generic ``_QuantFusedExperts``; the legacy wrapper is still registered on ``transformers<5.12``, whose ``torch.bmm``-based forward the generic wrapper cannot intercept. - Fix ``examples/hf_ptq`` multi-node FSDP2 export (``--use_fsdp2``) failing with ``RuntimeError: Cannot set version_counter for inference tensor``. ``export_quantized`` wrapped its whole body in ``torch.inference_mode()``, so the full params gathered by ``get_model_state_dict(full_state_dict=True)`` were inference tensors and the subsequent ``state_dict()`` -> ``param.detach()`` could not set their version counter. The export context is now ``torch.no_grad()``, which still disables autograd but keeps the gathered params as normal tensors. -- Fix ``examples/hf_ptq`` aborting the whole model load when the throwaway meta-device skeleton it builds to size ``infer_auto_device_map`` cannot be constructed (NVBug 6563509). The skeleton used ``init_empty_weights(include_buffers=True)``, which accelerate implements as a bare global ``torch.device("meta")`` context: *every* tensor built in ``__init__`` lands on meta, so remote-code checkpoints that derive scalar hyperparameters from real tensors there failed with ``Tensor.item() cannot be called on meta tensors`` before ``from_pretrained`` was ever reached. That made the probe stricter than the loader it predicts -- Transformers 4.x builds under ``include_buffers=False`` and Transformers 5.x under a meta context patched by ``meta_device_safe_creation_ops()`` -- so it could kill loads that would otherwise have succeeded. The skeleton now uses ``include_buffers=False``, which patches only ``nn.Module.register_parameter`` and leaves ``__init__`` arithmetic on a real device (module sizes are unchanged: ``compute_module_sizes`` reads shape and dtype, not storage). If it still cannot be built, the memory estimate is skipped with a warning instead of failing the run; ``--use_seq_device_map`` (which applies ``--gpu_max_mem_percentage``) and ``--batch_size`` cover the lost heuristic. +- Fix ``examples/hf_ptq`` aborting the whole model load while deciding a device map (NVBug 6563509). To leave GPU headroom for calibration activations, ``get_model`` needs to know whether the weights fit on the GPUs. It answered that by constructing a throwaway meta-device model and running ``infer_auto_device_map`` on it, under ``init_empty_weights(include_buffers=True)`` -- which accelerate implements as a bare global ``torch.device("meta")`` context, so *every* tensor built in ``__init__`` lands on meta. Remote-code checkpoints that derive scalar hyperparameters from real tensors there (Phi-4-multimodal's conformer does ``int(torch.tensor(...))``) failed with ``Tensor.item() cannot be called on meta tensors`` before ``from_pretrained`` was ever reached, even though both Transformers 4.x and 5.x load them fine. The size is now read from the checkpoint on disk (``metadata.total_size`` in the safetensors index, falling back to summing shard sizes) instead of from a constructed model, so no checkpoint ``__init__`` runs during device-map selection and this class of failure is unreachable rather than merely survivable. The recorded total matches the previous computation exactly (verified on Qwen3-8B and DeepSeek-R1-Distill-Llama-70B). When the size cannot be determined, no cap is applied and ``from_pretrained`` maps the model as before. - Fix unified HF export of multimodal models whose vision tower carries its own ``PrefixChange`` conversion (``LlavaForConditionalGeneration`` on ``transformers>=5.12`` — NVBug 6525511). transformers collects conversion mappings recursively and scopes each sub-model's transforms to that sub-module via ``scope_prefix``, matching only keys under that prefix. ModelOpt's quant-aware reverse conversion read the raw patterns and ignored ``scope_prefix``, so the vision tower's "add a ``vision_model.`` prefix" rule was applied to *every* key in the state dict: an exported llava-1.5-13b checkpoint had all 758 tensors moved under a bogus top-level ``vision_model.`` namespace (``vision_model.language_model.*``, ``vision_model.lm_head.*``), and vLLM rejected it with ``ValueError: There is no module or parameter named 'vision_model' in LlavaForConditionalGeneration``. Reverse rename rules now carry their scope and are applied only to keys under it, matching transformers' own ``WeightTransform._scoped_match`` semantics. ``Gemma3ForConditionalGeneration`` was affected identically and is fixed by the same change. 0.45 (2026-07-02) diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index d14784f8dcd..39df7322e7f 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -33,7 +33,6 @@ import torch import transformers import yaml -from accelerate import infer_auto_device_map, init_empty_weights from accelerate.utils import get_max_memory from safetensors import safe_open from transformers import ( @@ -638,57 +637,37 @@ def get_original_hf_quant_method(config) -> str | None: return None -def _resolve_init_config(hf_config, auto_model_module, ckpt_path, config_kwargs): - """Re-derive a built-in config when a remote-code config is used with a built-in model - class, so it matches the model definition's version; fall back to hf_config otherwise. - """ - if auto_model_module in [AutoModelForCausalLM, AutoModel]: - return hf_config - if not type(hf_config).__module__.startswith("transformers_modules"): - return hf_config - builtin_config_kwargs = {k: v for k, v in config_kwargs.items() if k != "trust_remote_code"} - try: - return AutoConfig.from_pretrained(ckpt_path, **builtin_config_kwargs) - except Exception as e: - warnings.warn( - f"Could not re-derive a built-in config for {ckpt_path} ({e}); using the " - "remote-code config for device-map inference." - ) - return hf_config +def _checkpoint_size_bytes(ckpt_path) -> int | None: + """Total on-disk size of a checkpoint's weights, or ``None`` if it cannot be read. + Used to decide whether the weights will fit on the GPUs. The obvious alternative -- + building a meta-device model and summing ``numel * dtype_size`` -- runs the + checkpoint's ``__init__``, which for remote-code models can fail outright and abort a + load that would otherwise have succeeded (NVBug 6563509). The safetensors index + already carries that sum: ``metadata.total_size`` matched ``compute_module_sizes`` + exactly on Qwen3-8B (15.256 GiB) and DeepSeek-R1-Distill-Llama-70B (131.417 GiB). -def _build_meta_skeleton(from_config, config_for_init, model_kwargs, architecture): - """Build a throwaway meta-device model used only to size ``infer_auto_device_map``. - - ``compute_module_sizes`` needs shapes and dtypes, never values or storage, so this - probe only has to reproduce the module tree. It must also be *no stricter than the - loader it predicts*, or it kills runs ``from_pretrained`` would have completed: - - - Transformers 4.x builds under ``init_empty_weights()``, i.e. ``include_buffers=False``. - - Transformers 5.x builds under ``torch.device("meta")`` plus - ``meta_device_safe_creation_ops()``, which redirects ``torch.linspace`` to CPU so - remote code that derives scalars from it in ``__init__`` keeps working. - - ``include_buffers=True`` is stricter than both: accelerate implements it as a bare - global ``torch.device("meta")`` context, so *every* tensor built in ``__init__`` lands - on meta and any ``int(torch.tensor(...))`` raises. ``include_buffers=False`` patches - only ``nn.Module.register_parameter``, leaving ``__init__`` arithmetic on a real - device; buffers stay materialized, which costs nothing here because their shape and - dtype size the same either way. - - Returns ``None`` (after warning) if the model cannot be built at all; the caller then - skips the estimate rather than failing a load ``from_pretrained`` can still do. + On-disk size is the checkpoint's own dtype, which is what gets loaded under the + default ``dtype="auto"``. """ + path = Path(ckpt_path) + if not path.is_dir(): + return None + + index_file = path / "model.safetensors.index.json" + if index_file.is_file(): + try: + with open(index_file) as f: + total_size = json.load(f).get("metadata", {}).get("total_size") + if isinstance(total_size, int): + return total_size + except (OSError, ValueError): + pass # fall through to summing the shards + + shards = sorted(path.glob("*.safetensors")) or sorted(path.glob("*.bin")) try: - with init_empty_weights(include_buffers=False): - return from_config(config_for_init, **model_kwargs) - except Exception as e: - warnings.warn( - f"Could not build a meta-device skeleton of {architecture} ({e!r}). " - "Skipping the device-map memory estimate and letting from_pretrained map the " - "model. If you hit GPU OOM, rerun with --use_seq_device_map (which applies " - "--gpu_max_mem_percentage) or lower --batch_size." - ) + return sum(shard.stat().st_size for shard in shards) or None + except OSError: return None @@ -893,28 +872,11 @@ def has_pack_quantized_config(config): auto_model_module = AutoModel else: auto_model_module = AutoModelForCausalLM - from_config = auto_model_module.from_config else: auto_model_module = getattr(transformers, architecture) - from_config = auto_model_module._from_config - config_for_init = _resolve_init_config( - hf_config, auto_model_module, ckpt_path, config_kwargs - ) - - # When computing the device_map, assuming bfloat16 precision by default, - # unless specified by the hf_config. - config_dtype = _get_config_dtype(config_for_init) - model_kwargs2 = _apply_dtype_to_config( - model_kwargs, config_dtype, architecture, apply_config_dtype=True - ) - if auto_model_module not in [AutoModelForCausalLM, AutoModel]: - model_kwargs2.pop("trust_remote_code", None) - model_kwargs2.pop("max_memory", None) - - # Only a sizing aid for ``infer_auto_device_map`` below; ``None`` when the model - # cannot be built on meta, in which case the estimate is skipped. - model = _build_meta_skeleton(from_config, config_for_init, model_kwargs2, architecture) + # Assume bfloat16 precision by default, unless specified by the hf_config. + config_dtype = _get_config_dtype(hf_config) max_memory = get_max_memory() @@ -934,9 +896,14 @@ def has_pack_quantized_config(config): f"Offload folder: {offload_folder}\n" "Weights exceeding GPU+CPU budgets will be streamed from disk." ) - elif model is not None: - inferred_device_map = infer_auto_device_map(model, max_memory=max_memory) - if "cpu" in inferred_device_map.values(): + else: + # Weights that do not fit on the GPUs leave no room for calibration + # activations, because device_map="auto" packs them to ~100% of free memory. + # Cap the budget in that case. Sized from the checkpoint on disk rather than + # from a constructed model, so no checkpoint ``__init__`` runs here. + checkpoint_bytes = _checkpoint_size_bytes(ckpt_path) + gpu_budget = sum(size for dev, size in max_memory.items() if isinstance(dev, int)) + if checkpoint_bytes is not None and checkpoint_bytes > gpu_budget: for _device in max_memory: if isinstance(_device, int): max_memory[_device] *= gpu_mem_percentage diff --git a/tests/examples/hf_ptq/test_example_utils.py b/tests/examples/hf_ptq/test_example_utils.py index 6175066f5c2..f4e80e1096f 100644 --- a/tests/examples/hf_ptq/test_example_utils.py +++ b/tests/examples/hf_ptq/test_example_utils.py @@ -19,9 +19,7 @@ """ import json -from contextlib import nullcontext from types import SimpleNamespace -from unittest.mock import patch import pytest import torch @@ -199,59 +197,15 @@ def test_get_original_hf_quant_method_none_for_unquantized(): ) -# ---------- _resolve_init_config --------------------------------------------- - - -def _remote_config(): - # Config whose class module lives under "transformers_modules" (remote code). - cls = type("_RemoteConfig", (), {"__module__": "transformers_modules.ckpt.config"}) - return cls() - - -def test_resolve_init_config_rederives_for_remote_config(): - builtin_cfg = SimpleNamespace() - with patch.object( - example_utils.AutoConfig, "from_pretrained", return_value=builtin_cfg - ) as mock: - out = example_utils._resolve_init_config( - _remote_config(), object, "/ckpt", {"trust_remote_code": True} - ) - assert out is builtin_cfg - mock.assert_called_once_with("/ckpt") # trust_remote_code stripped - - -def test_resolve_init_config_keeps_non_remote_config(): - cfg = SimpleNamespace() # module is "types", not remote - with patch.object(example_utils.AutoConfig, "from_pretrained") as mock: - assert example_utils._resolve_init_config(cfg, object, "/ckpt", {}) is cfg - mock.assert_not_called() - - -def test_resolve_init_config_falls_back_when_rederive_raises(): - cfg = _remote_config() - with patch.object(example_utils.AutoConfig, "from_pretrained", side_effect=ValueError()): - assert example_utils._resolve_init_config(cfg, object, "/ckpt", {}) is cfg - - @pytest.mark.parametrize( - ( - "architecture", - "model_class_name", - "expected_config_dtype_kwarg", - "unexpected_config_dtype_kwarg", - ), + ("architecture", "model_class_name"), [ - ("DeciLMForCausalLM", "AutoModelForCausalLM", "torch_dtype", "dtype"), - ("LlamaForCausalLM", "LlamaForCausalLM", "dtype", "torch_dtype"), + # DeciLM takes the legacy ``torch_dtype`` kwarg; everything else takes ``dtype``. + ("DeciLMForCausalLM", "AutoModelForCausalLM"), + ("LlamaForCausalLM", "LlamaForCausalLM"), ], ) -def test_get_model_uses_expected_dtype_kwarg( - monkeypatch, - architecture, - model_class_name, - expected_config_dtype_kwarg, - unexpected_config_dtype_kwarg, -): +def test_get_model_uses_expected_dtype_kwarg(monkeypatch, architecture, model_class_name): calls = {} hf_config = SimpleNamespace( architectures=[architecture], @@ -267,12 +221,7 @@ def eval(self): class FakeAutoModelForCausalLM: @staticmethod def from_config(config, **kwargs): - calls["from_config"] = kwargs - assert config is hf_config - assert kwargs[expected_config_dtype_kwarg] is torch.float16 - assert unexpected_config_dtype_kwarg not in kwargs - assert "max_memory" not in kwargs - return FakeModel() + raise AssertionError("get_model must not construct a model to size the load") @staticmethod def from_pretrained(*args, **kwargs): @@ -303,18 +252,12 @@ def from_pretrained(*args, **kwargs): monkeypatch.setattr(example_utils.transformers, model_class_name, FakeLlamaForCausalLM) monkeypatch.setattr(example_utils, "is_nemotron_vl", lambda config: False) monkeypatch.setattr(example_utils, "is_speculative", lambda config: False) - monkeypatch.setattr(example_utils, "init_empty_weights", lambda include_buffers: nullcontext()) monkeypatch.setattr(example_utils, "get_max_memory", lambda: {0: 1024}) - monkeypatch.setattr(example_utils, "infer_auto_device_map", lambda model, max_memory: {"": 0}) model = example_utils.get_model("checkpoint", device="cpu", trust_remote_code=True) assert isinstance(model, FakeModel) assert calls["eval"] - if expected_config_dtype_kwarg == "torch_dtype": - assert calls["from_config"]["trust_remote_code"] is True - else: - assert "trust_remote_code" not in calls["from_config"] assert calls["from_pretrained"]["trust_remote_code"] is True @@ -369,9 +312,7 @@ def from_pretrained(*args, **kwargs): monkeypatch.setattr(example_utils.transformers, architecture, FakeArchitecture, raising=False) monkeypatch.setattr(example_utils, "is_nemotron_vl", lambda config: False) monkeypatch.setattr(example_utils, "is_speculative", lambda config: False) - monkeypatch.setattr(example_utils, "init_empty_weights", lambda include_buffers: nullcontext()) monkeypatch.setattr(example_utils, "get_max_memory", lambda: {0: 1024}) - monkeypatch.setattr(example_utils, "infer_auto_device_map", lambda model, max_memory: {"": 0}) monkeypatch.setattr(torch.cuda, "device_count", lambda: device_count) example_utils.get_model("checkpoint", device="cuda", trust_remote_code=True) @@ -464,100 +405,118 @@ def from_pretrained(*args, **kwargs): ) monkeypatch.setattr(example_utils, "is_nemotron_vl", lambda config: False) monkeypatch.setattr(example_utils, "is_speculative", lambda config: False) - monkeypatch.setattr(example_utils, "init_empty_weights", lambda include_buffers: nullcontext()) monkeypatch.setattr(example_utils, "get_max_memory", lambda: {0: 1024}) - monkeypatch.setattr(example_utils, "infer_auto_device_map", lambda model, max_memory: {"": 0}) example_utils.get_model("checkpoint", device="cpu", trust_remote_code=trust_remote_code) assert used["path"] == ("bundled" if expect_bundled_code else "builtin") -# ---------- _build_meta_skeleton ---------------------------------------------- - +def _patch_get_model_deps(monkeypatch, hf_config, model_class): + monkeypatch.setattr(example_utils.AutoConfig, "from_pretrained", lambda *a, **kw: hf_config) + monkeypatch.setattr(example_utils.transformers, "LlamaForCausalLM", model_class) + monkeypatch.setattr(example_utils, "is_nemotron_vl", lambda config: False) + monkeypatch.setattr(example_utils, "is_speculative", lambda config: False) -def test_build_meta_skeleton_uses_permissive_patching(): - """The probe must be no stricter than the loader it predicts. - ``include_buffers=True`` is a bare global ``torch.device("meta")`` context, which - also captures scratch arithmetic in ``__init__``; ``from_pretrained`` never does - that, so the probe must not either. - """ - seen = {} +def _llama_config(): + return SimpleNamespace( + architectures=["LlamaForCausalLM"], + dtype=torch.float16, + model_type="llama", + torch_dtype=torch.bfloat16, + ) - def fake_init_empty_weights(include_buffers): - seen["include_buffers"] = include_buffers - return nullcontext() - sentinel = object() - with patch.object(example_utils, "init_empty_weights", fake_init_empty_weights): - out = example_utils._build_meta_skeleton( - lambda cfg, **kw: sentinel, "cfg", {"dtype": torch.bfloat16}, "Arch" - ) +# ---------- _checkpoint_size_bytes / GPU-budget sizing ------------------------- - assert out is sentinel - assert seen["include_buffers"] is False +def test_checkpoint_size_bytes_prefers_index_metadata(tmp_path): + (tmp_path / "model.safetensors.index.json").write_text( + json.dumps({"metadata": {"total_size": 16381470720}, "weight_map": {}}) + ) + # A shard is present but must be ignored in favour of the recorded total. + (tmp_path / "model-00001-of-00001.safetensors").write_bytes(b"\0" * 32) -def test_build_meta_skeleton_survives_meta_hostile_init(): - """Remote code that reads a real scalar in ``__init__`` must still build.""" + assert example_utils._checkpoint_size_bytes(tmp_path) == 16381470720 - def from_config(cfg, **kwargs): - # Mirrors Phi-4-multimodal's conformer: int() on a freshly built tensor. This - # raises under a global meta context but works with parameter-only patching. - return SimpleNamespace(width=int(torch.tensor(80.0))) - model = example_utils._build_meta_skeleton(from_config, "cfg", {}, "Arch") +def test_checkpoint_size_bytes_sums_shards_without_index(tmp_path): + (tmp_path / "model-00001-of-00002.safetensors").write_bytes(b"\0" * 100) + (tmp_path / "model-00002-of-00002.safetensors").write_bytes(b"\0" * 40) - assert model.width == 80 + assert example_utils._checkpoint_size_bytes(tmp_path) == 140 -def test_build_meta_skeleton_returns_none_and_warns_on_failure(): - def from_config(cfg, **kwargs): - raise RuntimeError("boom") +def test_checkpoint_size_bytes_falls_back_to_shards_on_bad_index(tmp_path): + (tmp_path / "model.safetensors.index.json").write_text("{not json") + (tmp_path / "model.safetensors").write_bytes(b"\0" * 77) - with pytest.warns(UserWarning, match="Could not build a meta-device skeleton of Arch"): - assert example_utils._build_meta_skeleton(from_config, "cfg", {}, "Arch") is None + assert example_utils._checkpoint_size_bytes(tmp_path) == 77 -def test_get_model_skips_device_map_estimate_when_skeleton_fails(monkeypatch): - """A failed probe must not abort the load: skip sizing, still call from_pretrained.""" - calls = {} - hf_config = SimpleNamespace( - architectures=["LlamaForCausalLM"], - dtype=torch.float16, - model_type="llama", - torch_dtype=torch.bfloat16, +@pytest.mark.parametrize( + ("total_size", "expect_capped"), + [ + # 3 GiB of weights against a 4 GiB GPU budget: fits, so no cap is applied. + (3 * 1024**3, False), + # 6 GiB against the same budget: will spill, so leave activation headroom. + (6 * 1024**3, True), + ], +) +def test_get_model_caps_gpu_budget_only_when_weights_do_not_fit( + monkeypatch, tmp_path, total_size, expect_capped +): + (tmp_path / "model.safetensors.index.json").write_text( + json.dumps({"metadata": {"total_size": total_size}, "weight_map": {}}) ) + calls = {} class FakeModel: def eval(self): - calls["eval"] = True + pass class FakeLlamaForCausalLM: @staticmethod def _from_config(config, **kwargs): - raise RuntimeError("Tensor.item() cannot be called on meta tensors") + raise AssertionError("get_model must not construct a model to size the load") @staticmethod def from_pretrained(*args, **kwargs): calls["from_pretrained"] = kwargs return FakeModel() - def _boom_infer(model, max_memory): - raise AssertionError("infer_auto_device_map must be skipped without a skeleton") + _patch_get_model_deps(monkeypatch, _llama_config(), FakeLlamaForCausalLM) + monkeypatch.setattr(example_utils, "get_max_memory", lambda: {0: 2 * 1024**3, 1: 2 * 1024**3}) - monkeypatch.setattr(example_utils.AutoConfig, "from_pretrained", lambda *a, **kw: hf_config) - monkeypatch.setattr(example_utils.transformers, "LlamaForCausalLM", FakeLlamaForCausalLM) - monkeypatch.setattr(example_utils, "is_nemotron_vl", lambda config: False) - monkeypatch.setattr(example_utils, "is_speculative", lambda config: False) + example_utils.get_model(str(tmp_path), device="cpu", trust_remote_code=True) + + if expect_capped: + assert calls["from_pretrained"]["max_memory"] == { + 0: 2 * 1024**3 * 0.8, + 1: 2 * 1024**3 * 0.8, + } + else: + assert "max_memory" not in calls["from_pretrained"] + + +def test_get_model_skips_cap_when_checkpoint_size_unknown(monkeypatch, tmp_path): + """An unreadable checkpoint layout must not invent a memory cap.""" + calls = {} + + class FakeModel: + def eval(self): + pass + + class FakeLlamaForCausalLM: + @staticmethod + def from_pretrained(*args, **kwargs): + calls["from_pretrained"] = kwargs + return FakeModel() + + _patch_get_model_deps(monkeypatch, _llama_config(), FakeLlamaForCausalLM) monkeypatch.setattr(example_utils, "get_max_memory", lambda: {0: 1024}) - monkeypatch.setattr(example_utils, "infer_auto_device_map", _boom_infer) - with pytest.warns(UserWarning, match="Skipping the device-map memory estimate"): - model = example_utils.get_model("checkpoint", device="cpu", trust_remote_code=True) + example_utils.get_model(str(tmp_path), device="cpu", trust_remote_code=True) - assert isinstance(model, FakeModel) - assert calls["eval"] - # The estimate is the only thing lost; no memory cap is invented for the load. assert "max_memory" not in calls["from_pretrained"] From 1b6a79d99dc8eece8f1ad3cc3f27c9c4cdb14986 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 10 Aug 2026 21:07:27 +0000 Subject: [PATCH 08/10] Stop auto-capping GPU memory for device_map="auto" get_model built a throwaway meta-device model, ran infer_auto_device_map on it, and -- if the weights would spill to CPU -- shrank max_memory by gpu_max_mem_percentage to leave room for calibration activations. Answering "will the weights fit" requires constructing the model, and that is the whole of NVBug 6563509: remote code that derives a scalar from a real tensor in __init__ (Phi-4-multimodal's conformer does int(torch.tensor(...))) cannot run under the global torch.device("meta") context accelerate uses for include_buffers=True. Both Transformers 4.x and 5.x load such models fine, so the probe was killing loads it existed only to tune. The probe is gone rather than repaired. Sizing it from the checkpoint on disk instead was accurate but not free: it silently skipped Hugging Face model IDs (`--pyt_ckpt_path` accepts a model card, and the README recommends it), and an aggregate byte count cannot reproduce per-device placement, so it could report a false fit near capacity. Neither is worth solving, because the cap is only wanted in the offload case. There it is close to free -- the run is already streaming weights, and the headroom is required, since at 100% packed calibration OOMs immediately. Applied to a model that does fit, it would convert a fully resident load into an offloaded one: a large, certain slowdown introduced where there was none. That asymmetry is why the old condition fired only on spill, and it means the check must not be biased in either direction -- which rules out a conservative threshold as well. So device_map="auto" now sizes itself; from_pretrained runs its own infer_auto_device_map on the real model regardless. --use_seq_device_map is the supported answer to GPU OOM, and --gpu_max_mem_percentage applies there and to --offload_folder, matching what its help text already documented ("when device_map is set to sequential"). Verified Phi-4-multimodal now reaches from_pretrained with no meta-tensor error, and no model is constructed during device-map selection. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 2 +- examples/hf_ptq/example_utils.py | 61 ++----------- tests/examples/hf_ptq/test_example_utils.py | 94 --------------------- 3 files changed, 7 insertions(+), 150 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 33f61eae4d5..82e680f564a 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -83,6 +83,7 @@ Changelog - Remove the deprecated ``examples/llm_autodeploy`` example (deprecated in 0.45). Use TensorRT-LLM's `AutoDeploy `_ directly together with ModelOpt PTQ in ``examples/hf_ptq``. - Remove the deprecated ``examples/llm_qad`` Megatron-LM QAD example (deprecated in 0.45). Use the `megatron_bridge QAD example `_ instead, which provides a simpler Python-based interface and better model coverage. - Dropped VILA / NVILA vision-language model support in ``examples/hf_ptq``. VILA's modeling code requires ``transformers<=4.50.0``, which conflicts with ModelOpt's minimum supported ``transformers`` version. The VILA-specific bootstrap (repo clone, ``requirements-vila.txt``) and loading paths in ``example_utils.py`` have been removed. +- ``examples/hf_ptq`` no longer auto-caps GPU memory for ``device_map="auto"``. It used to build a throwaway meta-device model, run ``infer_auto_device_map`` on it, and -- if the weights would spill to CPU -- shrink ``max_memory`` by ``--gpu_max_mem_percentage`` to leave room for calibration activations. Deciding that requires constructing the model, and remote-code checkpoints that derive scalar hyperparameters from real tensors in ``__init__`` (Phi-4-multimodal's conformer does ``int(torch.tensor(...))``) cannot be built under the global ``torch.device("meta")`` context accelerate uses, so the run died with ``Tensor.item() cannot be called on meta tensors`` before ``from_pretrained`` was ever reached (NVBug 6563509) -- even though both Transformers 4.x and 5.x load those models fine. ``device_map="auto"`` now sizes itself, as ``from_pretrained`` runs its own ``infer_auto_device_map`` regardless. Pass ``--use_seq_device_map`` if quantization hits GPU OOM; ``--gpu_max_mem_percentage`` applies there and to ``--offload_folder``, matching what its help text already documented. - Dropped **Phi-4-multimodal** PTQ support in ``examples/hf_ptq`` (NVBug 6563509). Its bundled remote code predates Transformers v5 and no longer loads on any version ModelOpt supports (``transformers>=4.57``): it requires ``transformers<4.52`` because it reaches ``prepare_inputs_for_generation`` through ``peft``, which needs ``PreTrainedModel`` to still inherit ``GenerationMixin``, and it declares ``_tied_weights_keys`` as a list, which Transformers 5.x rejects. **Phi-3-vision** is dropped alongside it: it is the older, superseded model in the same family, so with its successor unsupportable there is no reason to keep carrying the predecessor. (Phi-3-vision shares the list-valued ``_tied_weights_keys`` defect and so is likewise broken on Transformers 5.x, though it does not hit the ``peft`` blocker.) The support-matrix row, the ``phi4mm`` model type, the multimodal-detection heuristics that only ever matched these two (``vision_lora`` / ``audio_processor`` / ``embd_layer.image_embd_layer``), the ``Phi3Image`` / ``PhiImage`` embedding-export exclusions, and the ``modelopt_recipes/huggingface/phi4mm/`` recipes have been removed. Text-only Phi-3/Phi-4 and Phi-3.5-MoE are unaffected. **Deprecations** @@ -106,7 +107,6 @@ Changelog - Fix ``examples/vllm_serve`` serving shared experts uncalibrated: their ``gate_proj``/``up_proj`` quantizer keys were not merged into ``gate_up_proj`` on reload, so they matched no module and were dropped. - Fix Qwen3-VL MoE PTQ failing on ``transformers>=5.12`` with ``AttributeError: 'QuantQwen3VLMoeTextExperts' object has no attribute 'hidden_size'`` (NVBug 6518551). transformers 5.12 moved ``Qwen3VLMoeTextExperts`` onto the standard ``@use_experts_implementation`` fused layout (``hidden_size``/``expert_dim`` renamed to ``hidden_dim``/``intermediate_dim``, ``gate_up_proj`` transposed to ``(num_experts, 2*intermediate_dim, hidden_dim)``, two ``F.linear`` calls per expert), but the legacy ``_QuantQwen3VLMoeTextExperts`` wrapper stayed statically registered and shadowed on-the-fly detection. The new layout is now left to ``register_fused_experts_on_the_fly``, which claims it with the generic ``_QuantFusedExperts``; the legacy wrapper is still registered on ``transformers<5.12``, whose ``torch.bmm``-based forward the generic wrapper cannot intercept. - Fix ``examples/hf_ptq`` multi-node FSDP2 export (``--use_fsdp2``) failing with ``RuntimeError: Cannot set version_counter for inference tensor``. ``export_quantized`` wrapped its whole body in ``torch.inference_mode()``, so the full params gathered by ``get_model_state_dict(full_state_dict=True)`` were inference tensors and the subsequent ``state_dict()`` -> ``param.detach()`` could not set their version counter. The export context is now ``torch.no_grad()``, which still disables autograd but keeps the gathered params as normal tensors. -- Fix ``examples/hf_ptq`` aborting the whole model load while deciding a device map (NVBug 6563509). To leave GPU headroom for calibration activations, ``get_model`` needs to know whether the weights fit on the GPUs. It answered that by constructing a throwaway meta-device model and running ``infer_auto_device_map`` on it, under ``init_empty_weights(include_buffers=True)`` -- which accelerate implements as a bare global ``torch.device("meta")`` context, so *every* tensor built in ``__init__`` lands on meta. Remote-code checkpoints that derive scalar hyperparameters from real tensors there (Phi-4-multimodal's conformer does ``int(torch.tensor(...))``) failed with ``Tensor.item() cannot be called on meta tensors`` before ``from_pretrained`` was ever reached, even though both Transformers 4.x and 5.x load them fine. The size is now read from the checkpoint on disk (``metadata.total_size`` in the safetensors index, falling back to summing shard sizes) instead of from a constructed model, so no checkpoint ``__init__`` runs during device-map selection and this class of failure is unreachable rather than merely survivable. The recorded total matches the previous computation exactly (verified on Qwen3-8B and DeepSeek-R1-Distill-Llama-70B). When the size cannot be determined, no cap is applied and ``from_pretrained`` maps the model as before. - Fix unified HF export of multimodal models whose vision tower carries its own ``PrefixChange`` conversion (``LlavaForConditionalGeneration`` on ``transformers>=5.12`` — NVBug 6525511). transformers collects conversion mappings recursively and scopes each sub-model's transforms to that sub-module via ``scope_prefix``, matching only keys under that prefix. ModelOpt's quant-aware reverse conversion read the raw patterns and ignored ``scope_prefix``, so the vision tower's "add a ``vision_model.`` prefix" rule was applied to *every* key in the state dict: an exported llava-1.5-13b checkpoint had all 758 tensors moved under a bogus top-level ``vision_model.`` namespace (``vision_model.language_model.*``, ``vision_model.lm_head.*``), and vLLM rejected it with ``ValueError: There is no module or parameter named 'vision_model' in LlavaForConditionalGeneration``. Reverse rename rules now carry their scope and are applied only to keys under it, matching transformers' own ``WeightTransform._scoped_match`` semantics. ``Gemma3ForConditionalGeneration`` was affected identically and is fixed by the same change. 0.45 (2026-07-02) diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index 39df7322e7f..9018d267e02 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -637,40 +637,6 @@ def get_original_hf_quant_method(config) -> str | None: return None -def _checkpoint_size_bytes(ckpt_path) -> int | None: - """Total on-disk size of a checkpoint's weights, or ``None`` if it cannot be read. - - Used to decide whether the weights will fit on the GPUs. The obvious alternative -- - building a meta-device model and summing ``numel * dtype_size`` -- runs the - checkpoint's ``__init__``, which for remote-code models can fail outright and abort a - load that would otherwise have succeeded (NVBug 6563509). The safetensors index - already carries that sum: ``metadata.total_size`` matched ``compute_module_sizes`` - exactly on Qwen3-8B (15.256 GiB) and DeepSeek-R1-Distill-Llama-70B (131.417 GiB). - - On-disk size is the checkpoint's own dtype, which is what gets loaded under the - default ``dtype="auto"``. - """ - path = Path(ckpt_path) - if not path.is_dir(): - return None - - index_file = path / "model.safetensors.index.json" - if index_file.is_file(): - try: - with open(index_file) as f: - total_size = json.load(f).get("metadata", {}).get("total_size") - if isinstance(total_size, int): - return total_size - except (OSError, ValueError): - pass # fall through to summing the shards - - shards = sorted(path.glob("*.safetensors")) or sorted(path.glob("*.bin")) - try: - return sum(shard.stat().st_size for shard in shards) or None - except OSError: - return None - - def _get_config_dtype(config): config_dtype = ( getattr(config, "dtype", None) or getattr(config, "torch_dtype", None) or torch.bfloat16 @@ -878,9 +844,13 @@ def has_pack_quantized_config(config): # Assume bfloat16 precision by default, unless specified by the hf_config. config_dtype = _get_config_dtype(hf_config) - max_memory = get_max_memory() - + # device_map="auto" is left to size itself. Capping it here needs to know whether + # the weights fit, which needs a constructed model, whose __init__ some remote-code + # checkpoints cannot survive on a meta device (NVBug 6563509) -- and capping when + # they *do* fit would introduce CPU offload that was not there. --use_seq_device_map + # is the supported answer to GPU OOM, and --gpu_max_mem_percentage applies there. if _disk_offload: + max_memory = get_max_memory() for _k in max_memory: if isinstance(_k, int): if max_gpu_memory_gb is not None: @@ -896,25 +866,6 @@ def has_pack_quantized_config(config): f"Offload folder: {offload_folder}\n" "Weights exceeding GPU+CPU budgets will be streamed from disk." ) - else: - # Weights that do not fit on the GPUs leave no room for calibration - # activations, because device_map="auto" packs them to ~100% of free memory. - # Cap the budget in that case. Sized from the checkpoint on disk rather than - # from a constructed model, so no checkpoint ``__init__`` runs here. - checkpoint_bytes = _checkpoint_size_bytes(ckpt_path) - gpu_budget = sum(size for dev, size in max_memory.items() if isinstance(dev, int)) - if checkpoint_bytes is not None and checkpoint_bytes > gpu_budget: - for _device in max_memory: - if isinstance(_device, int): - max_memory[_device] *= gpu_mem_percentage - - print( - "Model does not fit to the GPU mem. " - f"We apply the following memory limit for calibration: \n{max_memory}\n" - "If you hit GPU OOM issue, please adjust `gpu_mem_percentage` or " - "reduce the calibration `batch_size` manually." - ) - model_kwargs["max_memory"] = max_memory model_kwargs2 = _apply_dtype_to_config(model_kwargs, config_dtype, architecture) if _disk_offload: diff --git a/tests/examples/hf_ptq/test_example_utils.py b/tests/examples/hf_ptq/test_example_utils.py index f4e80e1096f..66b853e3ba5 100644 --- a/tests/examples/hf_ptq/test_example_utils.py +++ b/tests/examples/hf_ptq/test_example_utils.py @@ -426,97 +426,3 @@ def _llama_config(): model_type="llama", torch_dtype=torch.bfloat16, ) - - -# ---------- _checkpoint_size_bytes / GPU-budget sizing ------------------------- - - -def test_checkpoint_size_bytes_prefers_index_metadata(tmp_path): - (tmp_path / "model.safetensors.index.json").write_text( - json.dumps({"metadata": {"total_size": 16381470720}, "weight_map": {}}) - ) - # A shard is present but must be ignored in favour of the recorded total. - (tmp_path / "model-00001-of-00001.safetensors").write_bytes(b"\0" * 32) - - assert example_utils._checkpoint_size_bytes(tmp_path) == 16381470720 - - -def test_checkpoint_size_bytes_sums_shards_without_index(tmp_path): - (tmp_path / "model-00001-of-00002.safetensors").write_bytes(b"\0" * 100) - (tmp_path / "model-00002-of-00002.safetensors").write_bytes(b"\0" * 40) - - assert example_utils._checkpoint_size_bytes(tmp_path) == 140 - - -def test_checkpoint_size_bytes_falls_back_to_shards_on_bad_index(tmp_path): - (tmp_path / "model.safetensors.index.json").write_text("{not json") - (tmp_path / "model.safetensors").write_bytes(b"\0" * 77) - - assert example_utils._checkpoint_size_bytes(tmp_path) == 77 - - -@pytest.mark.parametrize( - ("total_size", "expect_capped"), - [ - # 3 GiB of weights against a 4 GiB GPU budget: fits, so no cap is applied. - (3 * 1024**3, False), - # 6 GiB against the same budget: will spill, so leave activation headroom. - (6 * 1024**3, True), - ], -) -def test_get_model_caps_gpu_budget_only_when_weights_do_not_fit( - monkeypatch, tmp_path, total_size, expect_capped -): - (tmp_path / "model.safetensors.index.json").write_text( - json.dumps({"metadata": {"total_size": total_size}, "weight_map": {}}) - ) - calls = {} - - class FakeModel: - def eval(self): - pass - - class FakeLlamaForCausalLM: - @staticmethod - def _from_config(config, **kwargs): - raise AssertionError("get_model must not construct a model to size the load") - - @staticmethod - def from_pretrained(*args, **kwargs): - calls["from_pretrained"] = kwargs - return FakeModel() - - _patch_get_model_deps(monkeypatch, _llama_config(), FakeLlamaForCausalLM) - monkeypatch.setattr(example_utils, "get_max_memory", lambda: {0: 2 * 1024**3, 1: 2 * 1024**3}) - - example_utils.get_model(str(tmp_path), device="cpu", trust_remote_code=True) - - if expect_capped: - assert calls["from_pretrained"]["max_memory"] == { - 0: 2 * 1024**3 * 0.8, - 1: 2 * 1024**3 * 0.8, - } - else: - assert "max_memory" not in calls["from_pretrained"] - - -def test_get_model_skips_cap_when_checkpoint_size_unknown(monkeypatch, tmp_path): - """An unreadable checkpoint layout must not invent a memory cap.""" - calls = {} - - class FakeModel: - def eval(self): - pass - - class FakeLlamaForCausalLM: - @staticmethod - def from_pretrained(*args, **kwargs): - calls["from_pretrained"] = kwargs - return FakeModel() - - _patch_get_model_deps(monkeypatch, _llama_config(), FakeLlamaForCausalLM) - monkeypatch.setattr(example_utils, "get_max_memory", lambda: {0: 1024}) - - example_utils.get_model(str(tmp_path), device="cpu", trust_remote_code=True) - - assert "max_memory" not in calls["from_pretrained"] From 701180ed681e175588009eedd75c7a272397e095 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 10 Aug 2026 21:29:34 +0000 Subject: [PATCH 09/10] Revert all device-map sizing changes; keep only the Phi deprecation examples/hf_ptq/example_utils.py and its tests are restored to main except for the Phi-specific multimodal-detection heuristics (vision_lora, audio_processor, embd_layer.image_embd_layer, model_type == "phi4mm"), which only ever matched the two models being dropped. The meta-device skeleton, infer_auto_device_map, and the gpu_max_mem_percentage cap keep their original behavior. The cap is wanted exactly where it already fires -- when the model is already offloading to CPU, where it is close to free and the headroom is required. Every alternative explored here (a permissive skeleton, sizing from the checkpoint index, an unconditional budget) either changed behavior for models that fit today or moved the guard somewhere it does not belong, for a crash that only ever affected checkpoints this PR removes. NVBug 6563509 is therefore resolved by dropping Phi-4-multimodal, not by changing the loader. A remote-code checkpoint that computes scalars from real tensors in __init__ will still fail the meta-device build; that is a separate question if it ever affects a supported model. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 1 - examples/hf_ptq/example_utils.py | 61 ++++++++++++-- tests/examples/hf_ptq/test_example_utils.py | 89 ++++++++++++++++----- 3 files changed, 120 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 82e680f564a..c1e207c2bb4 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -83,7 +83,6 @@ Changelog - Remove the deprecated ``examples/llm_autodeploy`` example (deprecated in 0.45). Use TensorRT-LLM's `AutoDeploy `_ directly together with ModelOpt PTQ in ``examples/hf_ptq``. - Remove the deprecated ``examples/llm_qad`` Megatron-LM QAD example (deprecated in 0.45). Use the `megatron_bridge QAD example `_ instead, which provides a simpler Python-based interface and better model coverage. - Dropped VILA / NVILA vision-language model support in ``examples/hf_ptq``. VILA's modeling code requires ``transformers<=4.50.0``, which conflicts with ModelOpt's minimum supported ``transformers`` version. The VILA-specific bootstrap (repo clone, ``requirements-vila.txt``) and loading paths in ``example_utils.py`` have been removed. -- ``examples/hf_ptq`` no longer auto-caps GPU memory for ``device_map="auto"``. It used to build a throwaway meta-device model, run ``infer_auto_device_map`` on it, and -- if the weights would spill to CPU -- shrink ``max_memory`` by ``--gpu_max_mem_percentage`` to leave room for calibration activations. Deciding that requires constructing the model, and remote-code checkpoints that derive scalar hyperparameters from real tensors in ``__init__`` (Phi-4-multimodal's conformer does ``int(torch.tensor(...))``) cannot be built under the global ``torch.device("meta")`` context accelerate uses, so the run died with ``Tensor.item() cannot be called on meta tensors`` before ``from_pretrained`` was ever reached (NVBug 6563509) -- even though both Transformers 4.x and 5.x load those models fine. ``device_map="auto"`` now sizes itself, as ``from_pretrained`` runs its own ``infer_auto_device_map`` regardless. Pass ``--use_seq_device_map`` if quantization hits GPU OOM; ``--gpu_max_mem_percentage`` applies there and to ``--offload_folder``, matching what its help text already documented. - Dropped **Phi-4-multimodal** PTQ support in ``examples/hf_ptq`` (NVBug 6563509). Its bundled remote code predates Transformers v5 and no longer loads on any version ModelOpt supports (``transformers>=4.57``): it requires ``transformers<4.52`` because it reaches ``prepare_inputs_for_generation`` through ``peft``, which needs ``PreTrainedModel`` to still inherit ``GenerationMixin``, and it declares ``_tied_weights_keys`` as a list, which Transformers 5.x rejects. **Phi-3-vision** is dropped alongside it: it is the older, superseded model in the same family, so with its successor unsupportable there is no reason to keep carrying the predecessor. (Phi-3-vision shares the list-valued ``_tied_weights_keys`` defect and so is likewise broken on Transformers 5.x, though it does not hit the ``peft`` blocker.) The support-matrix row, the ``phi4mm`` model type, the multimodal-detection heuristics that only ever matched these two (``vision_lora`` / ``audio_processor`` / ``embd_layer.image_embd_layer``), the ``Phi3Image`` / ``PhiImage`` embedding-export exclusions, and the ``modelopt_recipes/huggingface/phi4mm/`` recipes have been removed. Text-only Phi-3/Phi-4 and Phi-3.5-MoE are unaffected. **Deprecations** diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index 9018d267e02..34fa5a975aa 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -33,6 +33,7 @@ import torch import transformers import yaml +from accelerate import infer_auto_device_map, init_empty_weights from accelerate.utils import get_max_memory from safetensors import safe_open from transformers import ( @@ -637,6 +638,25 @@ def get_original_hf_quant_method(config) -> str | None: return None +def _resolve_init_config(hf_config, auto_model_module, ckpt_path, config_kwargs): + """Re-derive a built-in config when a remote-code config is used with a built-in model + class, so it matches the model definition's version; fall back to hf_config otherwise. + """ + if auto_model_module in [AutoModelForCausalLM, AutoModel]: + return hf_config + if not type(hf_config).__module__.startswith("transformers_modules"): + return hf_config + builtin_config_kwargs = {k: v for k, v in config_kwargs.items() if k != "trust_remote_code"} + try: + return AutoConfig.from_pretrained(ckpt_path, **builtin_config_kwargs) + except Exception as e: + warnings.warn( + f"Could not re-derive a built-in config for {ckpt_path} ({e}); using the " + "remote-code config for device-map inference." + ) + return hf_config + + def _get_config_dtype(config): config_dtype = ( getattr(config, "dtype", None) or getattr(config, "torch_dtype", None) or torch.bfloat16 @@ -838,19 +858,30 @@ def has_pack_quantized_config(config): auto_model_module = AutoModel else: auto_model_module = AutoModelForCausalLM + from_config = auto_model_module.from_config else: auto_model_module = getattr(transformers, architecture) + from_config = auto_model_module._from_config - # Assume bfloat16 precision by default, unless specified by the hf_config. - config_dtype = _get_config_dtype(hf_config) + config_for_init = _resolve_init_config( + hf_config, auto_model_module, ckpt_path, config_kwargs + ) + + with init_empty_weights(include_buffers=True): + # When computing the device_map, assuming bfloat16 precision by default, + # unless specified by the hf_config. + config_dtype = _get_config_dtype(config_for_init) + model_kwargs2 = _apply_dtype_to_config( + model_kwargs, config_dtype, architecture, apply_config_dtype=True + ) + if auto_model_module not in [AutoModelForCausalLM, AutoModel]: + model_kwargs2.pop("trust_remote_code", None) + model_kwargs2.pop("max_memory", None) + model = from_config(config_for_init, **model_kwargs2) + + max_memory = get_max_memory() - # device_map="auto" is left to size itself. Capping it here needs to know whether - # the weights fit, which needs a constructed model, whose __init__ some remote-code - # checkpoints cannot survive on a meta device (NVBug 6563509) -- and capping when - # they *do* fit would introduce CPU offload that was not there. --use_seq_device_map - # is the supported answer to GPU OOM, and --gpu_max_mem_percentage applies there. if _disk_offload: - max_memory = get_max_memory() for _k in max_memory: if isinstance(_k, int): if max_gpu_memory_gb is not None: @@ -866,6 +897,20 @@ def has_pack_quantized_config(config): f"Offload folder: {offload_folder}\n" "Weights exceeding GPU+CPU budgets will be streamed from disk." ) + else: + inferred_device_map = infer_auto_device_map(model, max_memory=max_memory) + if "cpu" in inferred_device_map.values(): + for _device in max_memory: + if isinstance(_device, int): + max_memory[_device] *= gpu_mem_percentage + + print( + "Model does not fit to the GPU mem. " + f"We apply the following memory limit for calibration: \n{max_memory}\n" + "If you hit GPU OOM issue, please adjust `gpu_mem_percentage` or " + "reduce the calibration `batch_size` manually." + ) + model_kwargs["max_memory"] = max_memory model_kwargs2 = _apply_dtype_to_config(model_kwargs, config_dtype, architecture) if _disk_offload: diff --git a/tests/examples/hf_ptq/test_example_utils.py b/tests/examples/hf_ptq/test_example_utils.py index 66b853e3ba5..4b9120b0441 100644 --- a/tests/examples/hf_ptq/test_example_utils.py +++ b/tests/examples/hf_ptq/test_example_utils.py @@ -19,7 +19,9 @@ """ import json +from contextlib import nullcontext from types import SimpleNamespace +from unittest.mock import patch import pytest import torch @@ -197,15 +199,59 @@ def test_get_original_hf_quant_method_none_for_unquantized(): ) +# ---------- _resolve_init_config --------------------------------------------- + + +def _remote_config(): + # Config whose class module lives under "transformers_modules" (remote code). + cls = type("_RemoteConfig", (), {"__module__": "transformers_modules.ckpt.config"}) + return cls() + + +def test_resolve_init_config_rederives_for_remote_config(): + builtin_cfg = SimpleNamespace() + with patch.object( + example_utils.AutoConfig, "from_pretrained", return_value=builtin_cfg + ) as mock: + out = example_utils._resolve_init_config( + _remote_config(), object, "/ckpt", {"trust_remote_code": True} + ) + assert out is builtin_cfg + mock.assert_called_once_with("/ckpt") # trust_remote_code stripped + + +def test_resolve_init_config_keeps_non_remote_config(): + cfg = SimpleNamespace() # module is "types", not remote + with patch.object(example_utils.AutoConfig, "from_pretrained") as mock: + assert example_utils._resolve_init_config(cfg, object, "/ckpt", {}) is cfg + mock.assert_not_called() + + +def test_resolve_init_config_falls_back_when_rederive_raises(): + cfg = _remote_config() + with patch.object(example_utils.AutoConfig, "from_pretrained", side_effect=ValueError()): + assert example_utils._resolve_init_config(cfg, object, "/ckpt", {}) is cfg + + @pytest.mark.parametrize( - ("architecture", "model_class_name"), + ( + "architecture", + "model_class_name", + "expected_config_dtype_kwarg", + "unexpected_config_dtype_kwarg", + ), [ - # DeciLM takes the legacy ``torch_dtype`` kwarg; everything else takes ``dtype``. - ("DeciLMForCausalLM", "AutoModelForCausalLM"), - ("LlamaForCausalLM", "LlamaForCausalLM"), + ("DeciLMForCausalLM", "AutoModelForCausalLM", "torch_dtype", "dtype"), + ("LlamaForCausalLM", "LlamaForCausalLM", "dtype", "torch_dtype"), ], ) -def test_get_model_uses_expected_dtype_kwarg(monkeypatch, architecture, model_class_name): +def test_get_model_uses_expected_dtype_kwarg( + monkeypatch, + architecture, + model_class_name, + expected_config_dtype_kwarg, + unexpected_config_dtype_kwarg, +): calls = {} hf_config = SimpleNamespace( architectures=[architecture], @@ -221,7 +267,12 @@ def eval(self): class FakeAutoModelForCausalLM: @staticmethod def from_config(config, **kwargs): - raise AssertionError("get_model must not construct a model to size the load") + calls["from_config"] = kwargs + assert config is hf_config + assert kwargs[expected_config_dtype_kwarg] is torch.float16 + assert unexpected_config_dtype_kwarg not in kwargs + assert "max_memory" not in kwargs + return FakeModel() @staticmethod def from_pretrained(*args, **kwargs): @@ -252,12 +303,18 @@ def from_pretrained(*args, **kwargs): monkeypatch.setattr(example_utils.transformers, model_class_name, FakeLlamaForCausalLM) monkeypatch.setattr(example_utils, "is_nemotron_vl", lambda config: False) monkeypatch.setattr(example_utils, "is_speculative", lambda config: False) + monkeypatch.setattr(example_utils, "init_empty_weights", lambda include_buffers: nullcontext()) monkeypatch.setattr(example_utils, "get_max_memory", lambda: {0: 1024}) + monkeypatch.setattr(example_utils, "infer_auto_device_map", lambda model, max_memory: {"": 0}) model = example_utils.get_model("checkpoint", device="cpu", trust_remote_code=True) assert isinstance(model, FakeModel) assert calls["eval"] + if expected_config_dtype_kwarg == "torch_dtype": + assert calls["from_config"]["trust_remote_code"] is True + else: + assert "trust_remote_code" not in calls["from_config"] assert calls["from_pretrained"]["trust_remote_code"] is True @@ -312,7 +369,9 @@ def from_pretrained(*args, **kwargs): monkeypatch.setattr(example_utils.transformers, architecture, FakeArchitecture, raising=False) monkeypatch.setattr(example_utils, "is_nemotron_vl", lambda config: False) monkeypatch.setattr(example_utils, "is_speculative", lambda config: False) + monkeypatch.setattr(example_utils, "init_empty_weights", lambda include_buffers: nullcontext()) monkeypatch.setattr(example_utils, "get_max_memory", lambda: {0: 1024}) + monkeypatch.setattr(example_utils, "infer_auto_device_map", lambda model, max_memory: {"": 0}) monkeypatch.setattr(torch.cuda, "device_count", lambda: device_count) example_utils.get_model("checkpoint", device="cuda", trust_remote_code=True) @@ -405,24 +464,10 @@ def from_pretrained(*args, **kwargs): ) monkeypatch.setattr(example_utils, "is_nemotron_vl", lambda config: False) monkeypatch.setattr(example_utils, "is_speculative", lambda config: False) + monkeypatch.setattr(example_utils, "init_empty_weights", lambda include_buffers: nullcontext()) monkeypatch.setattr(example_utils, "get_max_memory", lambda: {0: 1024}) + monkeypatch.setattr(example_utils, "infer_auto_device_map", lambda model, max_memory: {"": 0}) example_utils.get_model("checkpoint", device="cpu", trust_remote_code=trust_remote_code) assert used["path"] == ("bundled" if expect_bundled_code else "builtin") - - -def _patch_get_model_deps(monkeypatch, hf_config, model_class): - monkeypatch.setattr(example_utils.AutoConfig, "from_pretrained", lambda *a, **kw: hf_config) - monkeypatch.setattr(example_utils.transformers, "LlamaForCausalLM", model_class) - monkeypatch.setattr(example_utils, "is_nemotron_vl", lambda config: False) - monkeypatch.setattr(example_utils, "is_speculative", lambda config: False) - - -def _llama_config(): - return SimpleNamespace( - architectures=["LlamaForCausalLM"], - dtype=torch.float16, - model_type="llama", - torch_dtype=torch.bfloat16, - ) From c67c8b01f4066ed1eacba0b18245189007127e89 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 10 Aug 2026 22:19:40 +0000 Subject: [PATCH 10/10] Drop the stale phi4mm reference from the 0.46 DiffusionGemma entry 0.46 is unreleased, so the same section cannot both cite modelopt_recipes/huggingface/phi4mm/ as an existing pattern and delete it. nemotron_vl still ships and carries the example on its own. The 0.45 and 0.37 mentions are left alone: those releases shipped with the recipe present, and their entries are a record of what happened then. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c1e207c2bb4..ca4822bd061 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -40,7 +40,7 @@ Changelog - **Deduplicate the modules shared at source** in the quantized export step: ``_export_quantized_weight`` and ``_export_fused_experts`` now alias bit-identical packed ``weight`` / ``weight_scale`` / ``weight_scale_2`` buffers across modules sharing a source weight ``data_ptr()`` so the downstream ``postprocess_state_dict`` dedup catches them (~42% storage reduction on ``nvfp4_experts_only`` for tied 26B MoE checkpoints). - New ``sync_tied_input_amax`` helper max-merges per-side ``input_quantizer.amax`` across tied modules before export so single-backbone consumers that load one ``input_scale`` per parameter don't clip either side. - The exported state_dict is also **reordered (decoder keys win instead of encoder)** so canonical-side keys per HF's ``_tied_weights_keys`` declaration win the data_ptr dedup; gated to the DiffusionGemma model class in ``_reorder_canonical_first``, no-op for every other model. - - New DiffusionGemma model-specific recipe under ``modelopt_recipes/huggingface/diffusion_gemma/ptq/`` (``nvfp4_experts_only.yaml`` + its ``disabled_quantizers.yaml`` unit) adds the ``*self_conditioning*`` exclude on top of the standard default, leaving the shared ``default_disabled_quantizers`` unit clean for non-diffusion models — pattern matches the existing ``phi4mm`` / ``nemotron_vl`` model-specific recipes. + - New DiffusionGemma model-specific recipe under ``modelopt_recipes/huggingface/diffusion_gemma/ptq/`` (``nvfp4_experts_only.yaml`` + its ``disabled_quantizers.yaml`` unit) adds the ``*self_conditioning*`` exclude on top of the standard default, leaving the shared ``default_disabled_quantizers`` unit clean for non-diffusion models — pattern matches the existing ``nemotron_vl`` model-specific recipes. - ``hf_ptq.py`` also unwraps ``ModelOutput`` dataclasses from ``.generate()`` so the preview decode works on diffusion models. Non-tied models see no behavioral change. - Add Torch-TensorRT FP8 deployment example for HuggingFace ViT (``examples/torch_trt/``): ``torch_tensorrt_ptq.py`` covers ``mtq.quantize`` → ``torch_tensorrt.compile(ir="dynamo")``, and ``torch_tensorrt_accuracy.py`` reports the compiled model's ImageNet-1k top-1/top-5 accuracy via the ``onnx_ptq`` ``evaluate`` harness (the unquantized baseline is Torch-TensorRT-compiled too, for an apples-to-apples comparison). Ships a ViT-tuned FP8 PTQ recipe under ``modelopt_recipes/huggingface/vit/ptq/`` (``fp8.yaml``) composed from the shared ``modelopt_recipes/configs/`` units: it quantizes the encoder Linears, patch-embed ``nn.Conv2d``, ``classifier``, and per-block LayerNorm inputs plus the attention Q/K/V BMMs and softmax. Verified on ``google/vit-base-patch16-224`` (ImageNet-1k 50k validation): FP8 stays within 0.13 pp Top-1 of the FP16 baseline. - Add **AutoQuantize recipe** support: ``mtq.auto_quantize`` can be driven declaratively from a YAML recipe (``RecipeType.AUTO_QUANTIZE`` / ``AutoQuantizeConfig``) specifying candidate formats, the ``effective_bits`` target, cost model (incl. ``active_moe`` and ``excluded_module_name_patterns``), scoring method, and disabled layers. Adds an ``effective_bits`` cost-model override on ``QuantizeConfig`` / ``QuantizerAttributeConfig`` (block-scale-accurate NVFP4 = 4.5 via ``configs/numerics/nvfp4``). Shipped recipes live under ``modelopt_recipes/general/auto_quantize/`` and model-specific ones under ``modelopt_recipes/huggingface//auto_quantize/``.