r"""

from peft.tuners.tuners_utils import BaseTunerLayer

has_base_layer_pattern = False

for module in model.modules():

if isinstance(module, BaseTunerLayer):

has_base_layer_pattern = hasattr(module, "base_layer")

break

if has_base_layer_pattern:

from peft.utils import _get_submodules

key_list = [key for key, _ in model.named_modules() if "lora" not in key]

for key in key_list:

try:

parent, target, target_name = _get_submodules(model, key)

except AttributeError:

continue

if hasattr(target, "base_layer"):

setattr(parent, target_name, target.get_base_layer())

else:

# This is for backwards compatibility with PEFT <= 0.6.2.

# TODO can be removed once that PEFT version is no longer supported.

from peft.tuners.lora import LoraLayer

for name, module in model.named_children():

if len(list(module.children())) > 0:

## compound module, go inside it

recurse_remove_peft_layers(module)

module_replaced = False

if isinstance(module, LoraLayer) and isinstance(module, torch.nn.Linear):

new_module = torch.nn.Linear(

module.in_features,

module.out_features,

bias=module.bias is not None,

).to(module.weight.device)

new_module.weight = module.weight

if module.bias is not None:

new_module.bias = module.bias

module_replaced = True

elif isinstance(module, LoraLayer) and isinstance(module, torch.nn.Conv2d):

new_module = torch.nn.Conv2d(

module.in_channels,

module.out_channels,

module.kernel_size,

module.stride,

module.padding,

module.dilation,

module.groups,

).to(module.weight.device)

new_module.weight = module.weight

if module.bias is not None:

new_module.bias = module.bias

module_replaced = True

if module_replaced:

setattr(model, name, new_module)

del module

if torch.cuda.is_available():

torch.cuda.empty_cache()

"""

from peft.tuners.tuners_utils import BaseTunerLayer

if weight == 1.0:

return

for module in model.modules():

if isinstance(module, BaseTunerLayer):

"""

from peft.tuners.tuners_utils import BaseTunerLayer

if weight is None or weight == 1.0:

return

for module in model.modules():

if isinstance(module, BaseTunerLayer):

if weight != 0:

module.unscale_layer(weight)

else:

for adapter_name in module.active_adapters:

# if weight == 0 unscale should re-set the scale to the original value.

rank_pattern = {}

alpha_pattern = {}

r = lora_alpha = list(rank_dict.values())[0]

if len(set(rank_dict.values())) > 1:

# get the rank occuring the most number of times

r = collections.Counter(rank_dict.values()).most_common()[0][0]

# for modules with rank different from the most occuring rank, add it to the `rank_pattern`

rank_pattern = dict(filter(lambda x: x[1] != r, rank_dict.items()))

rank_pattern = {k.split(".lora_B.")[0]: v for k, v in rank_pattern.items()}

if network_alpha_dict is not None and len(network_alpha_dict) > 0:

if len(set(network_alpha_dict.values())) > 1:

# get the alpha occuring the most number of times

lora_alpha = collections.Counter(network_alpha_dict.values()).most_common()[0][0]

# for modules with alpha different from the most occuring alpha, add it to the `alpha_pattern`

alpha_pattern = dict(filter(lambda x: x[1] != lora_alpha, network_alpha_dict.items()))

if is_unet:

alpha_pattern = {

".".join(k.split(".lora_A.")[0].split(".")).replace(".alpha", ""): v

for k, v in alpha_pattern.items()

}

else:

alpha_pattern = {".".join(k.split(".down.")[0].split(".")[:-1]): v for k, v in alpha_pattern.items()}

else:

lora_alpha = set(network_alpha_dict.values()).pop()

# layer names without the Diffusers specific

target_modules = list({name.split(".lora")[0] for name in peft_state_dict.keys()})

use_dora = any("lora_magnitude_vector" in k for k in peft_state_dict)

# for now we know that the "bias" keys are only associated with `lora_B`.

lora_bias = any("lora_B" in k and k.endswith(".bias") for k in peft_state_dict)

lora_config_kwargs = {

"r": r,

"lora_alpha": lora_alpha,

"rank_pattern": rank_pattern,

"alpha_pattern": alpha_pattern,

"target_modules": target_modules,

"use_dora": use_dora,

"lora_bias": lora_bias,

}

from peft.tuners.tuners_utils import BaseTunerLayer

for module in model.modules():

if isinstance(module, BaseTunerLayer):

return f"default_{len(module.r)}"

from peft.tuners.tuners_utils import BaseTunerLayer

for module in model.modules():

if isinstance(module, BaseTunerLayer):

# The recent version of PEFT needs to call `enable_adapters` instead

if hasattr(module, "enable_adapters"):

module.enable_adapters(enabled=enabled)

else:

from peft.tuners.tuners_utils import BaseTunerLayer

for module in model.modules():

if isinstance(module, BaseTunerLayer):

if hasattr(module, "delete_adapter"):

module.delete_adapter(adapter_name)

else:

raise ValueError(

"The version of PEFT you are using is not compatible, please use a version that is greater than 0.6.1"

)

# For transformers integration - we need to pop the adapter from the config

if getattr(model, "_hf_peft_config_loaded", False) and hasattr(model, "peft_config"):

model.peft_config.pop(adapter_name, None)

# In case all adapters are deleted, we need to delete the config

# and make sure to set the flag to False

if len(model.peft_config) == 0:

del model.peft_config

from peft.tuners.tuners_utils import BaseTunerLayer

def get_module_weight(weight_for_adapter, module_name):

if not isinstance(weight_for_adapter, dict):

# If weight_for_adapter is a single number, always return it.

return weight_for_adapter

for layer_name, weight_ in weight_for_adapter.items():

if layer_name in module_name:

return weight_

parts = module_name.split(".")

# e.g. key = "down_blocks.1.attentions.0"

key = f"{parts[0]}.{parts[1]}.attentions.{parts[3]}"

block_weight = weight_for_adapter.get(key, 1.0)

return block_weight

for module_name, module in model.named_modules():

if isinstance(module, BaseTunerLayer):

# For backward compatibility with previous PEFT versions, set multiple active adapters

if hasattr(module, "set_adapter"):

module.set_adapter(adapter_names)

else:

module.active_adapter = adapter_names

# Set the scaling weight for each adapter for this module

for adapter_name, weight in zip(adapter_names, weights):

r"""

if not is_peft_available():

raise ValueError("PEFT is not installed. Please install it with `pip install peft`")

is_peft_version_compatible = version.parse(importlib.metadata.version("peft")) > version.parse(min_version)

if not is_peft_version_compatible:

raise ValueError(

f"The version of PEFT you are using is not compatible, please use a version that is greater"

f" than {min_version}"