tessl/pypi-peft
State-of-the-art Parameter-Efficient Fine-Tuning (PEFT) methods for efficiently adapting large pretrained models
—
Pending
advanced-methods.mddocs/
Advanced Methods
Specialized parameter-efficient fine-tuning methods that use orthogonal transformations, structured adaptations, and experimental approaches. These methods often provide unique advantages for specific architectures or training scenarios.
Capabilities
Orthogonal Fine-Tuning (OFT)
Maintains hyperspherical energy by applying orthogonal transformations to preserve geometric properties of pretrained features.
class OFTConfig(PeftConfig):
"""Configuration for OFT (Orthogonal Fine-tuning)."""
def __init__(
self,
r: int = 8,
module_dropout: float = 0.0,
target_modules: Optional[Union[List[str], str]] = None,
init_weights: bool = True,
layers_to_transform: Optional[Union[List[int], int]] = None,
layers_pattern: Optional[str] = None,
modules_to_save: Optional[List[str]] = None,
coft: bool = False,
eps: float = 6e-5,
block_share: bool = False,
**kwargs
):
"""
Args:
r: OFT rank (constraint dimension)
module_dropout: Module dropout probability
target_modules: Names of modules to apply OFT to
init_weights: Whether to initialize weights
layers_to_transform: Layers to apply OFT to
layers_pattern: Pattern for layer names
modules_to_save: Modules to set as trainable and save
coft: Whether to use constrained OFT
eps: Epsilon for numerical stability
block_share: Whether to share blocks across layers
"""
class OFTModel:
"""OFT model implementation."""
def __init__(self, model, config: OFTConfig, adapter_name: str): ...Butterfly OFT (BOFT)
Extends OFT with butterfly factorization for improved parameter efficiency and structured transformations.
class BOFTConfig(PeftConfig):
"""Configuration for BOFT (Butterfly OFT)."""
def __init__(
self,
boft_block_size: int = 4,
boft_block_num: int = 0,
boft_n_butterfly_factor: int = 1,
target_modules: Optional[Union[List[str], str]] = None,
boft_dropout: float = 0.0,
fan_in_fan_out: bool = False,
init_weights: bool = True,
layers_to_transform: Optional[Union[List[int], int]] = None,
layers_pattern: Optional[str] = None,
modules_to_save: Optional[List[str]] = None,
**kwargs
):
"""
Args:
boft_block_size: Size of butterfly blocks
boft_block_num: Number of butterfly blocks
boft_n_butterfly_factor: Number of butterfly factors
target_modules: Names of modules to apply BOFT to
boft_dropout: BOFT dropout probability
fan_in_fan_out: Whether layer stores weights as (fan_in, fan_out)
init_weights: Whether to initialize weights
layers_to_transform: Layers to apply BOFT to
layers_pattern: Pattern for layer names
modules_to_save: Modules to set as trainable and save
"""
class BOFTModel:
"""BOFT model implementation."""
def __init__(self, model, config: BOFTConfig, adapter_name: str): ...IA3 (Infused Adapter by Inhibiting and Amplifying Inner Activations)
Learns vectors that rescale inner activations, providing a lightweight adaptation mechanism.
class IA3Config(PeftConfig):
"""Configuration for IA3."""
def __init__(
self,
target_modules: Optional[Union[List[str], str]] = None,
feedforward_modules: Optional[Union[List[str], str]] = None,
fan_in_fan_out: bool = False,
modules_to_save: Optional[List[str]] = None,
init_ia3_weights: bool = True,
**kwargs
): ...
class IA3Model:
"""IA3 model implementation."""
def __init__(self, model, config: IA3Config, adapter_name: str): ...Vera (Vector-based Random Matrix Adaptation)
Adaptation method using vector-based random matrix decomposition for parameter-efficient fine-tuning.
class VeraConfig(PeftConfig):
"""Configuration for Vera."""
def __init__(
self,
r: int = 256,
target_modules: Optional[Union[List[str], str]] = None,
projection_prng_key: int = 0,
vera_dropout: float = 0.0,
d_initial: float = 0.1,
**kwargs
): ...
class VeraModel:
"""Vera model implementation."""
def __init__(self, model, config: VeraConfig, adapter_name: str): ...Poly (Polynomial Adaptation)
Polynomial-based adaptation method for efficient fine-tuning.
class PolyConfig(PeftConfig):
"""Configuration for Poly."""
def __init__(
self,
r: int = 8,
target_modules: Optional[Union[List[str], str]] = None,
modules_to_save: Optional[List[str]] = None,
poly_type: str = "poly",
n_tasks: int = 1,
n_skills: int = 4,
n_splits: int = 1,
**kwargs
): ...
class PolyModel:
"""Poly model implementation."""
def __init__(self, model, config: PolyConfig, adapter_name: str): ...LayerNorm Tuning
Fine-tuning method that targets LayerNorm parameters for efficient adaptation.
class LNTuningConfig(PeftConfig):
"""Configuration for LayerNorm Tuning."""
def __init__(
self,
target_modules: Optional[Union[List[str], str]] = None,
modules_to_save: Optional[List[str]] = None,
**kwargs
): ...
class LNTuningModel:
"""LNTuning model implementation."""
def __init__(self, model, config: LNTuningConfig, adapter_name: str): ...FourierFT
Fourier Transform-based parameter efficient fine-tuning method.
class FourierFTConfig(PeftConfig):
"""Configuration for FourierFT."""
def __init__(
self,
n_frequency: int = 1000,
scaling: float = 300.0,
target_modules: Optional[Union[List[str], str]] = None,
modules_to_save: Optional[List[str]] = None,
**kwargs
): ...
class FourierFTModel:
"""FourierFT model implementation."""
def __init__(self, model, config: FourierFTConfig, adapter_name: str): ...XLoRA (Mixture of LoRA Experts)
Mixture of LoRA experts with learned gating mechanisms for multi-task adaptation.
class XLoraConfig(LoraConfig):
"""Configuration for XLoRA."""
def __init__(
self,
hidden_size: int,
xlora_depth: int = 1,
xlora_size: int = 2048,
enable_softmax: bool = True,
softmax_temperature: float = 1.0,
layernorm_type: str = "rms",
use_bias: bool = False,
xlora_dropout_p: float = 0.2,
**kwargs
): ...
class XLoraModel:
"""XLoRA model implementation."""
def __init__(self, model, config: XLoraConfig, adapter_name: str): ...BONE (Block-wise One-shot Neural Architecture Evolution)
Block-wise neural architecture evolution method for efficient model adaptation.
class BoneConfig(PeftConfig):
"""Configuration for BONE."""
def __init__(
self,
r: int = 8,
target_modules: Optional[Union[List[str], str]] = None,
modules_to_save: Optional[List[str]] = None,
**kwargs
): ...
class BoneModel:
"""BONE model implementation."""
def __init__(self, model, config: BoneConfig, adapter_name: str): ...VB-LoRA (Variational Bayesian LoRA)
Variational Bayesian extension of LoRA for uncertainty quantification.
class VBLoRAConfig(LoraConfig):
"""Configuration for VB-LoRA."""
def __init__(
self,
vb_r: int = 8,
vb_pi: float = 0.5,
vb_beta: float = 1.0,
**kwargs
): ...
class VBLoRAModel:
"""VB-LoRA model implementation."""
def __init__(self, model, config: VBLoRAConfig, adapter_name: str): ...Additional Advanced Methods
The following methods are also available in PEFT for specialized use cases:
# HRA (High-Rank Adaptation)
class HRAConfig(PeftConfig): ...
class HRAModel: ...
# MISS (Mixture of Integrated Sparse Structures)
class MissConfig(PeftConfig): ...
class MissModel: ...
# RandLoRA (Random LoRA)
class RandLoraConfig(PeftConfig): ...
class RandLoraModel: ...
# SHIRA (Shared Representations Adaptation)
class ShiraConfig(PeftConfig): ...
class ShiraModel: ...
# C3A (Cascaded Cross-domain Continual Adaptation)
class C3AConfig(PeftConfig): ...
class C3AModel: ...
# CPT (Continuous Prompt Tuning)
class CPTConfig(PeftConfig): ...
# AdaptionPrompt (Adaption Prompt)
class AdaptionPromptConfig(PeftConfig): ...
class AdaptionPromptModel: ...
# Trainable Tokens
class TrainableTokensConfig(PeftConfig): ...
class TrainableTokensModel: ...Usage Examples
OFT for Preserving Feature Geometry
from transformers import AutoModelForCausalLM
from peft import get_peft_model, OFTConfig
model = AutoModelForCausalLM.from_pretrained("gpt2")
oft_config = OFTConfig(
r=8,
target_modules=["c_attn", "c_proj"],
module_dropout=0.05,
task_type="CAUSAL_LM"
)
peft_model = get_peft_model(model, oft_config)IA3 for Lightweight Adaptation
from peft import IA3Config
ia3_config = IA3Config(
target_modules=["k_proj", "v_proj", "down_proj"],
feedforward_modules=["down_proj"],
task_type="CAUSAL_LM"
)
peft_model = get_peft_model(model, ia3_config)Vera with Random Projections
from peft import VeraConfig
vera_config = VeraConfig(
r=256,
target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
projection_prng_key=42,
vera_dropout=0.1,
d_initial=0.1,
task_type="CAUSAL_LM"
)
peft_model = get_peft_model(model, vera_config)Layer Normalization Tuning
from peft import LNTuningConfig
ln_config = LNTuningConfig(
target_modules=["input_layernorm", "post_attention_layernorm"],
task_type="CAUSAL_LM"
)
peft_model = get_peft_model(model, ln_config)X-LoRA Mixture of Experts
from peft import XLoraConfig
xlora_config = XLoraConfig(
hidden_size=768,
xlora_size=64,
xlora_depth=2,
layernorm_type="rms",
xlora_dropout_p=0.1,
task_type="CAUSAL_LM"
)
# Load multiple LoRA adapters first, then apply X-LoRA
peft_model = get_peft_model(model, xlora_config)Install with Tessl CLI
npx tessl i tessl/pypi-peft