tessl/pypi-peft

State-of-the-art Parameter-Efficient Fine-Tuning (PEFT) methods for efficiently adapting large pretrained models

—

Pending

Overview

Eval results

Files

Prompt Learning Methods

Name: tessl/pypi-peft
Author: tessl

Prompt-based parameter-efficient fine-tuning methods that learn soft prompts or prefix tokens to guide model behavior without modifying the base model weights. These methods are particularly effective for few-shot learning and task adaptation.

Capabilities

Prompt Tuning

Learns soft prompt tokens that are prepended to input sequences, allowing task-specific adaptation through prompt optimization.

class PromptTuningConfig(PromptLearningConfig):
    """Configuration for Prompt Tuning."""
    def __init__(
        self,
        num_virtual_tokens: int,
        prompt_tuning_init: PromptTuningInit = PromptTuningInit.RANDOM,
        prompt_tuning_init_text: Optional[str] = None,
        prompt_tuning_init_state_dict_path: Optional[str] = None,
        num_transformer_submodules: Optional[int] = None,
        num_attention_heads: Optional[int] = None,
        num_layers: Optional[int] = None,
        token_dim: Optional[int] = None,
        num_text_tokens: Optional[int] = None,
        **kwargs
    ):
        """
        Args:
            num_virtual_tokens: Number of virtual tokens
            prompt_tuning_init: Initialization method for prompt tuning
            prompt_tuning_init_text: Text to initialize prompt (if using TEXT init)
            prompt_tuning_init_state_dict_path: Path to state dict for initialization
            num_transformer_submodules: Number of transformer submodules
            num_attention_heads: Number of attention heads
            num_layers: Number of transformer layers
            token_dim: Token embedding dimension
            num_text_tokens: Number of text tokens for initialization
        """

class PromptTuningInit(str, Enum):
    """Initialization methods for prompt tuning."""
    RANDOM = "RANDOM"
    TEXT = "TEXT"

class PromptEmbedding(torch.nn.Module):
    """Embedding layer for learnable prompt tokens."""
    def __init__(self, config: PromptTuningConfig, word_embeddings): ...
    
    def forward(self, indices): ...

P-Tuning v2

Learns continuous prompt representations using a prompt encoder, providing more flexibility than standard prompt tuning.

class PromptEncoderConfig(PromptLearningConfig):
    """Configuration for P-Tuning v2."""
    def __init__(
        self,
        num_virtual_tokens: int,
        encoder_reparameterization_type: PromptEncoderReparameterizationType = PromptEncoderReparameterizationType.MLP,
        encoder_hidden_size: Optional[int] = None,
        encoder_num_layers: int = 2,
        encoder_dropout: float = 0.0,
        **kwargs
    ):
        """
        Args:
            num_virtual_tokens: Number of virtual tokens
            encoder_reparameterization_type: Type of encoder architecture
            encoder_hidden_size: Hidden size of prompt encoder
            encoder_num_layers: Number of layers in prompt encoder  
            encoder_dropout: Dropout probability in prompt encoder
        """

class PromptEncoderReparameterizationType(str, Enum):
    """Reparameterization types for prompt encoder."""
    MLP = "MLP"  
    LSTM = "LSTM"

class PromptEncoder(torch.nn.Module):
    """Prompt encoder for P-Tuning v2."""
    def __init__(self, config: PromptEncoderConfig): ...
    
    def forward(self, indices): ...

Prefix Tuning

Learns prefix tokens that are prepended to key and value vectors in attention layers, optimizing continuous task-specific vectors.

class PrefixTuningConfig(PromptLearningConfig):
    """Configuration for Prefix Tuning."""
    def __init__(
        self,
        num_virtual_tokens: int,
        encoder_hidden_size: int,
        prefix_projection: bool = False,
        **kwargs
    ):
        """
        Args:
            num_virtual_tokens: Number of virtual tokens
            encoder_hidden_size: Hidden size of the encoder
            prefix_projection: Whether to project prefix parameters
        """

class PrefixEncoder(torch.nn.Module):
    """Prefix encoder for generating prefix embeddings."""
    def __init__(self, config: PrefixTuningConfig): ...
    
    def forward(self, prefix: torch.Tensor): ...

Multitask Prompt Tuning

Extends prompt tuning to handle multiple tasks simultaneously with shared and task-specific prompt components.

class MultitaskPromptTuningConfig(PromptLearningConfig):
    """Configuration for Multitask Prompt Tuning."""
    def __init__(
        self,
        num_virtual_tokens: int,
        prompt_tuning_init: MultitaskPromptTuningInit = MultitaskPromptTuningInit.RANDOM,
        prompt_tuning_init_text: Optional[str] = None,
        num_tasks: int = 1,
        **kwargs
    ):
        """
        Args:
            num_virtual_tokens: Number of virtual tokens per task
            prompt_tuning_init: Initialization method
            prompt_tuning_init_text: Text for initialization
            num_tasks: Number of tasks to support
        """

class MultitaskPromptTuningInit(str, Enum):
    """Initialization methods for multitask prompt tuning."""
    RANDOM = "RANDOM"
    TEXT = "TEXT"

class MultitaskPromptEmbedding(torch.nn.Module):
    """Embedding layer for multitask prompt tuning."""
    def __init__(self, config: MultitaskPromptTuningConfig, word_embeddings): ...
    
    def forward(self, indices, task_ids): ...

Adaption Prompt

Specialized prompt learning method designed for vision-language models and multimodal tasks.

class AdaptionPromptConfig(PeftConfig):
    """Configuration for Adaption Prompt."""
    def __init__(
        self,
        target_modules: Optional[Union[List[str], str]] = None,
        adapter_len: int = 10,
        adapter_layers: int = 30,
        **kwargs
    ):
        """
        Args:
            target_modules: Names of modules to apply adaption prompt to
            adapter_len: Length of adapter prompts
            adapter_layers: Number of layers to apply adapters to
        """

class AdaptionPromptModel:
    """Adaption prompt model implementation."""
    def __init__(self, model, config: AdaptionPromptConfig, adapter_name: str): ...

Base Prompt Learning Configuration

Base configuration class inherited by all prompt learning methods.

class PromptLearningConfig(PeftConfig):
    """Base configuration for prompt learning methods."""
    def __init__(
        self,
        num_virtual_tokens: int,
        token_dim: Optional[int] = None,
        num_transformer_submodules: Optional[int] = None,
        num_attention_heads: Optional[int] = None,
        num_layers: Optional[int] = None,
        encoder_reparameterization_type: Optional[str] = None,
        encoder_hidden_size: Optional[int] = None,
        **kwargs
    ):
        """
        Args:
            num_virtual_tokens: Number of virtual tokens
            token_dim: Dimension of token embeddings
            num_transformer_submodules: Number of transformer submodules
            num_attention_heads: Number of attention heads
            num_layers: Number of layers
            encoder_reparameterization_type: Type of encoder reparameterization
            encoder_hidden_size: Hidden size of encoder
        """
    
    @property
    def is_prompt_learning(self) -> bool:
        """Returns True for prompt learning configs."""
        return True

Usage Examples

Basic Prompt Tuning

from transformers import AutoModelForCausalLM
from peft import get_peft_model, PromptTuningConfig, TaskType

model = AutoModelForCausalLM.from_pretrained("gpt2")

prompt_config = PromptTuningConfig(
    task_type=TaskType.CAUSAL_LM,
    num_virtual_tokens=10,
    prompt_tuning_init="RANDOM"
)

peft_model = get_peft_model(model, prompt_config)

Prompt Tuning with Text Initialization

from peft import PromptTuningConfig, PromptTuningInit

prompt_config = PromptTuningConfig(
    task_type=TaskType.CAUSAL_LM,
    num_virtual_tokens=10,
    prompt_tuning_init=PromptTuningInit.TEXT,
    prompt_tuning_init_text="The following is a helpful response:"
)

peft_model = get_peft_model(model, prompt_config)

P-Tuning v2 with MLP Encoder

from peft import PromptEncoderConfig, PromptEncoderReparameterizationType

p_tuning_config = PromptEncoderConfig(
    task_type=TaskType.CAUSAL_LM,
    num_virtual_tokens=20,
    encoder_reparameterization_type=PromptEncoderReparameterizationType.MLP,
    encoder_hidden_size=128,
    encoder_num_layers=2,
    encoder_dropout=0.1
)

peft_model = get_peft_model(model, p_tuning_config)

Prefix Tuning

from peft import PrefixTuningConfig

prefix_config = PrefixTuningConfig(
    task_type=TaskType.CAUSAL_LM,
    num_virtual_tokens=30,
    encoder_hidden_size=512,
    prefix_projection=True
)

peft_model = get_peft_model(model, prefix_config)

Multitask Prompt Tuning

from peft import MultitaskPromptTuningConfig

multitask_config = MultitaskPromptTuningConfig(
    task_type=TaskType.CAUSAL_LM,
    num_virtual_tokens=10,
    num_tasks=3,
    prompt_tuning_init="RANDOM"
)

peft_model = get_peft_model(model, multitask_config)

# During inference, specify task_id
outputs = peft_model(input_ids=input_ids, task_ids=task_ids)

Adaption Prompt for Vision Models

from transformers import CLIPModel
from peft import AdaptionPromptConfig

clip_model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")

adaption_config = AdaptionPromptConfig(
    target_modules=["visual.transformer.resblocks"],
    adapter_len=10,
    adapter_layers=12
)

peft_model = get_peft_model(clip_model, adaption_config)

Install with Tessl CLI