Teleprompt optimizers that automatically improve DSPy programs by finding better prompts, demonstrations, and parameters. These optimizers "compile" programs to optimize performance on specific tasks and datasets.
Abstract base class for all optimizers.
class Teleprompter:
"""
Base class for DSPy optimizers/compilers.
Teleprompts transform student programs into optimized versions
by finding better prompts, demonstrations, or parameters.
"""
def compile(self, student, *, trainset, **kwargs):
"""
Optimize and compile a program.
Args:
student: Module to optimize
trainset: Training examples for optimization
**kwargs: Additional optimization parameters
Returns:
Compiled/optimized module
"""
passBootstrap demonstrations from labeled examples and program traces.
class BootstrapFewShot:
"""
Bootstrap few-shot optimizer.
Composes demonstrations from labeled examples and bootstrapped
execution traces. Validates demonstrations against a metric.
"""
def __init__(
self,
metric: callable = None,
metric_threshold: float = None,
teacher_settings: dict = None,
max_bootstrapped_demos: int = 4,
max_labeled_demos: int = 16,
max_rounds: int = 1,
max_errors: int = None
):
"""
Initialize bootstrap optimizer.
Args:
metric (callable | None): Metric function(example, pred, trace) -> bool/float
metric_threshold (float | None): Threshold for accepting demonstrations
teacher_settings (dict | None): Settings for teacher model (e.g., {"lm": teacher_lm})
max_bootstrapped_demos (int): Max bootstrapped demos per predictor (default: 4)
max_labeled_demos (int): Max labeled demos per predictor (default: 16)
max_rounds (int): Number of bootstrap rounds (default: 1)
max_errors (int | None): Max errors before stopping
"""
pass
def compile(self, student, teacher=None, trainset=None):
"""
Compile program with bootstrapped demonstrations.
Args:
student: Module to optimize
teacher: Teacher module (default: copy of student)
trainset: Training examples
Returns:
Compiled module with demonstrations
"""
passUsage:
import dspy
# Configure
dspy.configure(lm=dspy.LM('openai/gpt-4o-mini'))
# Define program
class RAG(dspy.Module):
def __init__(self):
super().__init__()
self.retrieve = dspy.Retrieve(k=3)
self.generate = dspy.ChainOfThought("context, question -> answer")
def forward(self, question):
context = self.retrieve(query=question).passages
return self.generate(context=context, question=question)
# Define metric
def validate_answer(example, pred, trace=None):
return example.answer.lower() == pred.answer.lower()
# Create training set
trainset = [
dspy.Example(question="What is ML?", answer="Machine Learning").with_inputs("question"),
# ... more examples
]
# Optimize
optimizer = dspy.BootstrapFewShot(
metric=validate_answer,
max_bootstrapped_demos=4,
max_labeled_demos=8
)
compiled_rag = optimizer.compile(RAG(), trainset=trainset)
# Use optimized program
result = compiled_rag(question="What is deep learning?")Bootstrap with random search over demonstration sets.
class BootstrapFewShotWithRandomSearch:
"""
Bootstrap with random search optimizer.
Extends BootstrapFewShot by trying multiple random demonstration
sets and selecting the best performing combination.
"""
def __init__(
self,
metric: callable = None,
metric_threshold: float = None,
teacher_settings: dict = None,
max_bootstrapped_demos: int = 4,
max_labeled_demos: int = 16,
max_rounds: int = 1,
num_candidate_programs: int = 10,
max_errors: int = None
):
"""
Initialize bootstrap with random search.
Args:
metric (callable | None): Validation metric
metric_threshold (float | None): Acceptance threshold
teacher_settings (dict | None): Teacher model settings
max_bootstrapped_demos (int): Max bootstrapped demos (default: 4)
max_labeled_demos (int): Max labeled demos (default: 16)
max_rounds (int): Bootstrap rounds (default: 1)
num_candidate_programs (int): Number of random candidates (default: 10)
max_errors (int | None): Max errors before stopping
"""
pass
def compile(self, student, teacher=None, trainset=None, valset=None):
"""
Compile with random search.
Args:
student: Module to optimize
teacher: Teacher module
trainset: Training examples
valset: Validation set for selecting best candidate
Returns:
Best compiled module from candidates
"""
passUsage:
import dspy
# Configure
dspy.configure(lm=dspy.LM('openai/gpt-4o-mini'))
# Define metric
def accuracy(example, pred, trace=None):
return example.answer == pred.answer
# Optimize with random search
optimizer = dspy.BootstrapFewShotWithRandomSearch(
metric=accuracy,
num_candidate_programs=20 # Try 20 different demo combinations
)
compiled = optimizer.compile(
program,
trainset=train_data,
valset=val_data
)
# Alias: BootstrapRS
optimizer = dspy.BootstrapRS(metric=accuracy, num_candidate_programs=15)Bootstrap using Optuna for hyperparameter optimization.
class BootstrapFewShotWithOptuna:
"""
Bootstrap with Optuna optimizer.
Uses Optuna library for sophisticated hyperparameter optimization
including number of demonstrations, temperature, etc.
"""
def __init__(
self,
metric: callable = None,
metric_threshold: float = None,
teacher_settings: dict = None,
max_bootstrapped_demos: int = 4,
max_labeled_demos: int = 16,
n_trials: int = 100,
**kwargs
):
"""
Initialize bootstrap with Optuna.
Args:
metric (callable | None): Validation metric
metric_threshold (float | None): Acceptance threshold
teacher_settings (dict | None): Teacher model settings
max_bootstrapped_demos (int): Max bootstrapped demos (default: 4)
max_labeled_demos (int): Max labeled demos (default: 16)
n_trials (int): Number of Optuna trials (default: 100)
**kwargs: Additional Optuna parameters
"""
pass
def compile(self, student, teacher=None, trainset=None, valset=None):
"""
Compile with Optuna optimization.
Args:
student: Module to optimize
teacher: Teacher module
trainset: Training examples
valset: Validation set
Returns:
Optimized module with best hyperparameters
"""
passMulti-prompt Instruction Proposal Optimizer - advanced joint optimization of instructions and demonstrations.
class MIPROv2:
"""
MIPROv2 optimizer.
Advanced optimizer that jointly optimizes both instructions and
demonstrations through iterative proposal and evaluation.
"""
def __init__(
self,
metric: callable,
prompt_model=None,
task_model=None,
teacher_settings: dict = None,
max_bootstrapped_demos: int = 4,
max_labeled_demos: int = 4,
auto: str = "light",
num_candidates: int = None,
num_threads: int = None,
max_errors: int = None,
seed: int = 9,
init_temperature: float = 1.0,
verbose: bool = False,
track_stats: bool = True,
log_dir: str = None,
metric_threshold: float = None
):
"""
Initialize MIPROv2 optimizer.
Args:
metric (callable): Evaluation metric function
prompt_model (LM | None): Model for generating prompts/instructions
task_model (LM | None): Model for executing task
teacher_settings (dict | None): Teacher model settings
max_bootstrapped_demos (int): Max bootstrapped demos (default: 4)
max_labeled_demos (int): Max labeled demos (default: 4)
auto (str | None): Auto-configuration mode ("light", "medium", "heavy")
num_candidates (int | None): Number of instruction candidates to try
num_threads (int | None): Threads for parallel evaluation
max_errors (int | None): Max errors before stopping
seed (int): Random seed (default: 9)
init_temperature (float): Initial sampling temperature (default: 1.0)
verbose (bool): Verbose logging (default: False)
track_stats (bool): Track optimization statistics (default: True)
log_dir (str | None): Directory for logging optimization stats
metric_threshold (float | None): Target metric threshold
"""
pass
def compile(self, student, trainset, valset=None, **kwargs):
"""
Compile program with MIPROv2 optimization.
Args:
student: Module to optimize
trainset: Training examples
valset: Validation set (optional, uses trainset if not provided)
**kwargs: Additional optimization parameters
Returns:
Optimized module with best instructions and demonstrations
"""
passUsage:
import dspy
# Configure models
prompt_model = dspy.LM('openai/gpt-4o')
task_model = dspy.LM('openai/gpt-4o-mini')
dspy.configure(lm=task_model)
# Define metric
from dspy.evaluate import F1
def validate(example, pred, trace=None):
return F1(pred.answer, [example.answer]) > 0.8
# Optimize with MIPROv2
optimizer = dspy.MIPROv2(
metric=validate,
prompt_model=prompt_model,
task_model=task_model,
auto="medium", # Auto-configure for medium optimization effort
num_candidates=30,
verbose=True
)
compiled = optimizer.compile(
program,
trainset=train_data,
valset=val_data
)Coordinate Prompt Optimization - optimizes instructions across multiple predictors.
class COPRO:
"""
COPRO optimizer.
Coordinate Prompt Optimization that optimizes instruction prompts
across all predictors in a program.
"""
def __init__(
self,
metric: callable,
breadth: int = 10,
depth: int = 3,
init_temperature: float = 1.0,
verbose: bool = False,
track_stats: bool = False
):
"""
Initialize COPRO optimizer.
Args:
metric (callable): Evaluation metric
breadth (int): Number of instruction candidates per iteration (default: 10)
depth (int): Number of optimization iterations (default: 3)
init_temperature (float): Initial temperature (default: 1.0)
verbose (bool): Verbose logging (default: False)
track_stats (bool): Track statistics (default: False)
"""
pass
def compile(self, student, trainset, eval_kwargs=None):
"""
Compile with instruction optimization.
Args:
student: Module to optimize
trainset: Training examples
eval_kwargs: Additional evaluation parameters
Returns:
Module with optimized instructions
"""
passAdvanced optimizer using avatar-based learning.
class AvatarOptimizer:
"""
Avatar optimizer.
Advanced optimization using avatar-based learning strategies
for instruction and demonstration optimization.
"""
def __init__(
self,
metric: callable,
max_iters: int = 10,
lower_bound: int = 1,
upper_bound: int = 10,
max_positive_inputs: int = None,
max_negative_inputs: int = None
):
"""
Initialize Avatar optimizer.
Args:
metric (callable): Evaluation metric
max_iters (int): Maximum optimization iterations (default: 10)
lower_bound (int): Lower bound for search (default: 1)
upper_bound (int): Upper bound for search (default: 10)
max_positive_inputs (int | None): Max positive examples
max_negative_inputs (int | None): Max negative examples
"""
pass
def compile(self, student, trainset, **kwargs):
"""
Compile with avatar optimization.
Args:
student: Module to optimize
trainset: Training examples
**kwargs: Additional parameters
Returns:
Optimized module
"""
passSimplified Iterative Method for Better Alignment.
class SIMBA:
"""
SIMBA optimizer.
Simplified Iterative Method for Better Alignment that
optimizes programs through iterative refinement.
"""
def __init__(
self,
metric: callable,
num_iterations: int = 10,
temperature: float = 1.0,
verbose: bool = False
):
"""
Initialize SIMBA optimizer.
Args:
metric (callable): Evaluation metric
num_iterations (int): Number of iterations (default: 10)
temperature (float): Sampling temperature (default: 1.0)
verbose (bool): Verbose logging (default: False)
"""
pass
def compile(self, student, trainset, **kwargs):
"""
Compile with SIMBA optimization.
Args:
student: Module to optimize
trainset: Training examples
**kwargs: Additional parameters
Returns:
Optimized module
"""
passGradient-Efficient Prompt Adaptation.
class GEPA:
"""
GEPA optimizer.
Gradient-Efficient Prompt Adaptation for optimizing prompts
using gradient-based techniques.
"""
def __init__(
self,
metric: callable,
num_steps: int = 100,
learning_rate: float = 0.01,
batch_size: int = 8
):
"""
Initialize GEPA optimizer.
Args:
metric (callable): Evaluation metric
num_steps (int): Optimization steps (default: 100)
learning_rate (float): Learning rate (default: 0.01)
batch_size (int): Batch size (default: 8)
"""
pass
def compile(self, student, trainset, valset=None):
"""
Compile with gradient-based optimization.
Args:
student: Module to optimize
trainset: Training examples
valset: Validation set
Returns:
Optimized module
"""
passEnsemble-based optimization approach.
class BetterTogether:
"""
BetterTogether optimizer.
Ensemble-based optimization that combines multiple optimization
strategies for improved performance.
"""
def __init__(
self,
metric: callable,
strategies: list = None,
max_rounds: int = 5
):
"""
Initialize BetterTogether optimizer.
Args:
metric (callable): Evaluation metric
strategies (list | None): List of optimization strategies to combine
max_rounds (int): Maximum optimization rounds (default: 5)
"""
pass
def compile(self, student, trainset, valset=None):
"""
Compile with ensemble optimization.
Args:
student: Module to optimize
trainset: Training examples
valset: Validation set
Returns:
Optimized ensemble module
"""
passCombine multiple programs into an ensemble.
class Ensemble:
"""
Ensemble optimizer.
Combines multiple programs (possibly optimized with different
strategies) into an ensemble that aggregates predictions.
"""
def __init__(
self,
programs: list = None,
aggregation: str = "majority",
size: int = None,
deterministic: bool = True
):
"""
Initialize ensemble.
Args:
programs (list | None): List of programs to ensemble
aggregation (str): Aggregation method ("majority", "weighted", etc.)
size (int | None): Ensemble size
deterministic (bool): Use deterministic aggregation (default: True)
"""
pass
def compile(self, student, trainset, **kwargs):
"""
Create ensemble from multiple training runs.
Args:
student: Base module to create ensemble from
trainset: Training examples
**kwargs: Additional parameters
Returns:
Ensemble module
"""
pass
def forward(self, **kwargs):
"""
Execute ensemble prediction.
Args:
**kwargs: Input arguments
Returns:
Aggregated prediction from all programs
"""
passUsage:
import dspy
# Train multiple models
programs = []
for i in range(5):
optimizer = dspy.BootstrapFewShot(metric=my_metric)
compiled = optimizer.compile(program, trainset=train_data[i])
programs.append(compiled)
# Create ensemble
ensemble = dspy.Ensemble(
programs=programs,
aggregation="majority"
)
# Use ensemble
result = ensemble(question="What is ML?")Use K-nearest neighbors to select relevant demonstrations at inference time.
class KNNFewShot:
"""
KNN few-shot optimizer.
Uses K-nearest neighbors to dynamically select the most relevant
demonstrations for each input at inference time.
"""
def __init__(
self,
k: int,
trainset: list,
vectorizer=None
):
"""
Initialize KNN few-shot optimizer.
Args:
k (int): Number of nearest neighbors
trainset (list): Training examples
vectorizer: Embedder for computing similarity
"""
pass
def compile(self, student, **kwargs):
"""
Compile with KNN-based demonstration selection.
Args:
student: Module to optimize
**kwargs: Additional parameters
Returns:
Module with dynamic KNN demonstration selection
"""
passUsage:
import dspy
from sentence_transformers import SentenceTransformer
# Set up embedder
model = SentenceTransformer("all-MiniLM-L6-v2")
embedder = dspy.Embedder(model.encode)
# Optimize with KNN
optimizer = dspy.KNNFewShot(
k=5,
trainset=trainset,
vectorizer=embedder
)
compiled = optimizer.compile(program)
# At inference, automatically selects 5 most relevant demos
result = compiled(question="What is machine learning?")Simple few-shot learning with labeled examples.
class LabeledFewShot:
"""
Labeled few-shot optimizer.
Simple optimizer that adds labeled examples as demonstrations
without bootstrapping or search.
"""
def __init__(self, k: int = 16):
"""
Initialize labeled few-shot optimizer.
Args:
k (int): Number of examples per predictor (default: 16)
"""
pass
def compile(self, student, trainset, sample: bool = True):
"""
Compile with labeled demonstrations.
Args:
student: Module to optimize
trainset: Labeled training examples
sample (bool): Whether to sample k examples (default: True)
Returns:
Module with labeled demonstrations
"""
passUsage:
import dspy
# Simple few-shot with labeled examples
optimizer = dspy.LabeledFewShot(k=8)
compiled = optimizer.compile(
program,
trainset=labeled_examples
)
# Each predictor gets up to 8 labeled examples as demos
result = compiled(question="What is AI?")Bootstrap and prepare data for model finetuning.
class BootstrapFinetune:
"""
Bootstrap finetune optimizer.
Bootstraps high-quality training data and prepares it for
finetuning a language model.
"""
def __init__(
self,
metric: callable = None,
num_threads: int = None,
target_model: str = None
):
"""
Initialize bootstrap finetune optimizer.
Args:
metric (callable | None): Quality metric for filtering data
num_threads (int | None): Threads for bootstrapping
target_model (str | None): Target model for finetuning format
"""
pass
def compile(self, student, teacher=None, trainset=None):
"""
Bootstrap and prepare finetuning data.
Args:
student: Module to optimize
teacher: Teacher model for bootstrapping
trainset: Training examples
Returns:
Tuple of (compiled_module, finetuning_data)
"""
passInfer rules from examples for instruction generation.
class InferRules:
"""
Infer rules optimizer.
Infers general rules and patterns from examples to generate
better task instructions.
"""
def __init__(
self,
max_rules: int = 10,
rule_length: int = 50
):
"""
Initialize rule inference optimizer.
Args:
max_rules (int): Maximum number of rules (default: 10)
rule_length (int): Max rule length in tokens (default: 50)
"""
pass
def compile(self, student, trainset, **kwargs):
"""
Compile with inferred rules as instructions.
Args:
student: Module to optimize
trainset: Training examples
**kwargs: Additional parameters
Returns:
Module with rule-based instructions
"""
passUtility function for bootstrapping trace data.
def bootstrap_trace_data(
program,
trainset: list,
metric: callable = None,
max_errors: int = None
):
"""
Bootstrap execution traces from program.
Args:
program: Module to bootstrap from
trainset (list): Training examples
metric (callable | None): Quality metric
max_errors (int | None): Max errors before stopping
Returns:
List of (example, prediction, score) tuples
"""
passUsage:
import dspy
# Bootstrap trace data for analysis
traces = dspy.bootstrap_trace_data(
program=my_program,
trainset=train_examples,
metric=my_metric,
max_errors=10
)
# Analyze successful traces
for example, pred, score in traces:
if score > 0.8:
print(f"Input: {example.inputs()}")
print(f"Output: {pred}")
print(f"Score: {score}")Simple optimization workflow:
import dspy
# 1. Define program
program = MyModule()
# 2. Define metric
def metric(example, pred, trace=None):
return example.answer == pred.answer
# 3. Prepare data
trainset = [...]
# 4. Optimize
optimizer = dspy.BootstrapFewShot(metric=metric)
compiled = optimizer.compile(program, trainset=trainset)
# 5. Use optimized program
result = compiled(question="What is ML?")Combine multiple optimization strategies:
import dspy
# Stage 1: Bootstrap demonstrations
optimizer1 = dspy.BootstrapFewShot(metric=metric)
compiled1 = optimizer1.compile(program, trainset=trainset)
# Stage 2: Optimize instructions
optimizer2 = dspy.MIPROv2(metric=metric, auto="light")
compiled2 = optimizer2.compile(compiled1, trainset=trainset)
# Stage 3: Create ensemble
programs = [compiled2]
for i in range(4):
opt = dspy.BootstrapRS(metric=metric, num_candidate_programs=10)
programs.append(opt.compile(program, trainset=trainset))
final = dspy.Ensemble(programs=programs)Use a stronger model as teacher:
import dspy
# Configure weak task model
task_lm = dspy.LM('openai/gpt-4o-mini')
dspy.configure(lm=task_lm)
# Use strong teacher model
teacher_lm = dspy.LM('openai/gpt-4o')
teacher_settings = {"lm": teacher_lm}
# Optimize
optimizer = dspy.BootstrapFewShot(
metric=metric,
teacher_settings=teacher_settings
)
compiled = optimizer.compile(program, trainset=trainset)
# Now uses gpt-4o-mini but learned from gpt-4o's reasoning