diff --git a/src/diffusers/utils/peft_utils.py b/src/diffusers/utils/peft_utils.py index 748ecfe58e6c..7698160325e2 100644 --- a/src/diffusers/utils/peft_utils.py +++ b/src/diffusers/utils/peft_utils.py @@ -156,6 +156,7 @@ def get_peft_kwargs( rank_pattern = {} alpha_pattern = {} r = lora_alpha = list(rank_dict.values())[0] + has_alphas = network_alpha_dict is not None and len(network_alpha_dict) > 0 if len(set(rank_dict.values())) > 1: # get the rank occurring the most number of times @@ -165,6 +166,13 @@ def get_peft_kwargs( rank_pattern = dict(filter(lambda x: x[1] != r, rank_dict.items())) rank_pattern = {k.split(".lora_B.")[0]: v for k, v in rank_pattern.items()} + if not has_alphas: + # No alpha data in the checkpoint: the diffusers/PEFT convention is + # `W_eff = W + lora_B @ lora_A`, i.e. alpha == rank per module (scale 1.0). + # Mirror the ranks into the alphas so every module keeps scale 1.0. + lora_alpha = r + alpha_pattern = dict(rank_pattern) + if network_alpha_dict is not None and len(network_alpha_dict) > 0: if len(set(network_alpha_dict.values())) > 1: # get the alpha occurring the most number of times diff --git a/tests/lora/test_peft_utils.py b/tests/lora/test_peft_utils.py new file mode 100644 index 000000000000..67b7e2695233 --- /dev/null +++ b/tests/lora/test_peft_utils.py @@ -0,0 +1,79 @@ +# Copyright 2026 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from diffusers.utils.peft_utils import get_peft_kwargs + + +def _rank_dict(module_ranks): + return {f"{module}.lora_B.weight": rank for module, rank in module_ranks.items()} + + +def _peft_state_dict(module_ranks): + state_dict = {} + for module in module_ranks: + state_dict[f"{module}.lora_A.weight"] = None + state_dict[f"{module}.lora_B.weight"] = None + return state_dict + + +def _effective_scale(kwargs, module): + alpha = kwargs["alpha_pattern"].get(module, kwargs["lora_alpha"]) + rank = kwargs["rank_pattern"].get(module, kwargs["r"]) + return alpha / rank + + +def test_mixed_ranks_without_alphas_apply_at_scale_one(): + # An adapter with per-module ranks and no alpha keys means `W_eff = W + lora_B @ lora_A`, + # i.e. alpha == rank, so every module must come out at scale 1.0. + module_ranks = {"blocks.0.adaln": 16, "blocks.0.to_q": 64, "blocks.0.to_v": 64} + kwargs = get_peft_kwargs(_rank_dict(module_ranks), None, _peft_state_dict(module_ranks)) + for module in module_ranks: + assert _effective_scale(kwargs, module) == 1.0 + + +def test_mixed_ranks_without_alphas_are_order_independent(): + # The scale must not depend on which module happens to come first in the state dict. + module_ranks = {"blocks.0.to_q": 64, "blocks.0.to_v": 64, "blocks.0.adaln": 16} + reordered = dict(reversed(module_ranks.items())) + for ranks in (module_ranks, reordered): + kwargs = get_peft_kwargs(_rank_dict(ranks), None, _peft_state_dict(ranks)) + for module in ranks: + assert _effective_scale(kwargs, module) == 1.0 + + +def test_uniform_rank_without_alphas_unchanged(): + module_ranks = {"blocks.0.to_q": 32, "blocks.0.to_v": 32} + kwargs = get_peft_kwargs(_rank_dict(module_ranks), None, _peft_state_dict(module_ranks)) + assert kwargs["r"] == kwargs["lora_alpha"] == 32 + assert kwargs["rank_pattern"] == {} + assert kwargs["alpha_pattern"] == {} + + +def test_mixed_ranks_with_uniform_alpha_keep_declared_alpha(): + # A declared alpha must win over the alpha == rank convention: scale is alpha / rank per module. + module_ranks = {"blocks.0.adaln": 16, "blocks.0.to_q": 64} + network_alphas = {f"{module}.alpha": 32 for module in module_ranks} + kwargs = get_peft_kwargs(_rank_dict(module_ranks), network_alphas, _peft_state_dict(module_ranks)) + assert kwargs["lora_alpha"] == 32 + assert kwargs["alpha_pattern"] == {} + assert _effective_scale(kwargs, "blocks.0.adaln") == 2.0 + assert _effective_scale(kwargs, "blocks.0.to_q") == 0.5 + + +def test_mixed_ranks_with_per_module_alphas_unchanged(): + module_ranks = {"blocks.0.adaln": 16, "blocks.0.to_q": 64} + network_alphas = {"blocks.0.adaln.alpha": 16, "blocks.0.to_q.alpha": 64} + kwargs = get_peft_kwargs(_rank_dict(module_ranks), network_alphas, _peft_state_dict(module_ranks)) + for module in module_ranks: + assert _effective_scale(kwargs, module) == 1.0