tessl/pypi-ray
Ray is a unified framework for scaling AI and Python applications.
—
Pending
hyperparameter-tuning.mddocs/
Hyperparameter Tuning
Ray Tune provides comprehensive hyperparameter optimization with multiple search algorithms, schedulers, and experiment management. It supports all major ML frameworks and integrates seamlessly with distributed training.
Capabilities
Core Tuning Framework
Main tuning functionality and experiment management.
class Tuner:
"""Main class for hyperparameter tuning experiments."""
def __init__(self, trainable, *, param_space=None, tune_config=None,
run_config=None):
"""
Initialize tuner.
Args:
trainable: Function or class to tune
param_space (dict, optional): Parameter search space
tune_config (TuneConfig, optional): Tuning configuration
run_config (RunConfig, optional): Run configuration
"""
def fit(self):
"""
Execute hyperparameter tuning.
Returns:
ResultGrid: Tuning results
"""
def get_results(self):
"""
Get tuning results.
Returns:
ResultGrid: Tuning results
"""
class TuneConfig:
"""Configuration for hyperparameter tuning."""
def __init__(self, *, metric=None, mode=None, search_alg=None,
scheduler=None, num_samples=10, max_concurrent_trials=None,
time_budget_s=None, **kwargs):
"""
Initialize tune configuration.
Args:
metric (str, optional): Metric to optimize
mode (str, optional): "min" or "max" for optimization
search_alg (SearchAlgorithm, optional): Search algorithm
scheduler (TrialScheduler, optional): Trial scheduler
num_samples (int): Number of trials to run
max_concurrent_trials (int, optional): Max concurrent trials
time_budget_s (float, optional): Time budget in seconds
"""
def run(trainable, *, config=None, metric=None, mode=None,
name=None, stop=None, time_budget_s=None, num_samples=10,
search_alg=None, scheduler=None, **kwargs):
"""
Run hyperparameter tuning experiment (legacy API).
Args:
trainable: Function or class to tune
config (dict, optional): Parameter configuration/search space
metric (str, optional): Metric to optimize
mode (str, optional): "min" or "max"
name (str, optional): Experiment name
stop (dict, optional): Stopping criteria
time_budget_s (float, optional): Time budget
num_samples (int): Number of trials
search_alg (SearchAlgorithm, optional): Search algorithm
scheduler (TrialScheduler, optional): Trial scheduler
Returns:
ResultGrid: Tuning results
"""
class ResultGrid:
"""Container for tuning results."""
def get_best_result(self, metric=None, mode=None):
"""Get best trial result."""
def get_dataframe(self):
"""Get results as pandas DataFrame."""
@property
def errors(self):
"""Get failed trials."""
def __len__(self):
"""Number of trials."""
def __iter__(self):
"""Iterate over results."""Search Algorithms
Various hyperparameter search algorithms.
class BasicVariantGenerator:
"""Grid search and random search."""
def __init__(self, *, max_concurrent=None, random_state=None):
"""
Initialize basic search.
Args:
max_concurrent (int, optional): Max concurrent trials
random_state (int, optional): Random seed
"""
class GridSearch(BasicVariantGenerator):
"""Grid search algorithm."""
class RandomSearch(BasicVariantGenerator):
"""Random search algorithm."""
class ConcurrencyLimiter:
"""Wrapper to limit concurrent trials."""
def __init__(self, searcher, max_concurrent):
"""
Initialize concurrency limiter.
Args:
searcher: Search algorithm to wrap
max_concurrent (int): Max concurrent trials
"""
class BayesOptSearch:
"""Bayesian optimization using Gaussian processes."""
def __init__(self, space=None, *, metric=None, mode="max",
utility_kwargs=None, random_state=None, **kwargs):
"""
Initialize Bayesian optimization.
Args:
space (dict, optional): Search space
metric (str, optional): Metric to optimize
mode (str): "min" or "max"
utility_kwargs (dict, optional): Acquisition function parameters
random_state (int, optional): Random seed
"""
class HyperOptSearch:
"""HyperOpt-based search algorithms."""
def __init__(self, space=None, *, algo=None, metric=None, mode="max",
points_to_evaluate=None, random_state_seed=None, **kwargs):
"""
Initialize HyperOpt search.
Args:
space (dict, optional): HyperOpt search space
algo: HyperOpt algorithm (tpe.suggest, random.suggest, etc.)
metric (str, optional): Metric to optimize
mode (str): "min" or "max"
points_to_evaluate (list, optional): Initial points
random_state_seed (int, optional): Random seed
"""
class OptunaSearch:
"""Optuna-based search algorithm."""
def __init__(self, space=None, *, metric=None, mode="max",
sampler=None, seed=None, **kwargs):
"""
Initialize Optuna search.
Args:
space (dict, optional): Search space
metric (str, optional): Metric to optimize
mode (str): "min" or "max"
sampler: Optuna sampler
seed (int, optional): Random seed
"""
class AxSearch:
"""Ax-based search algorithm."""
def __init__(self, space=None, *, metric=None, mode="max",
parameter_constraints=None, outcome_constraints=None, **kwargs):
"""
Initialize Ax search.
Args:
space (list, optional): Ax search space
metric (str, optional): Metric to optimize
mode (str): "min" or "max"
parameter_constraints (list, optional): Parameter constraints
outcome_constraints (list, optional): Outcome constraints
"""
class DragonflySearch:
"""Dragonfly-based search algorithm."""
def __init__(self, space=None, *, metric=None, mode="max",
domain=None, optimizer=None, **kwargs):
"""
Initialize Dragonfly search.
Args:
space (list, optional): Search space
metric (str, optional): Metric to optimize
mode (str): "min" or "max"
domain: Dragonfly domain
optimizer (str, optional): Optimizer type
"""Trial Schedulers
Schedulers for early stopping and resource allocation.
class FIFOScheduler:
"""First-in-first-out scheduler (no early stopping)."""
def __init__(self):
"""Initialize FIFO scheduler."""
class AsyncHyperBandScheduler:
"""Asynchronous Hyperband scheduler."""
def __init__(self, *, time_attr="training_iteration", metric=None,
mode="max", max_t=81, reduction_factor=3,
brackets=1, grace_period=1, **kwargs):
"""
Initialize AsyncHyperBand scheduler.
Args:
time_attr (str): Time attribute for scheduling
metric (str, optional): Metric to optimize
mode (str): "min" or "max"
max_t (int): Maximum time units
reduction_factor (int): Reduction factor
brackets (int): Number of brackets
grace_period (int): Minimum time before stopping
"""
class ASHAScheduler:
"""Asynchronous Successive Halving Algorithm (ASHA) scheduler."""
def __init__(self, *, time_attr="training_iteration", metric=None,
mode="max", max_t=100, grace_period=1, reduction_factor=4,
brackets=1, **kwargs):
"""
Initialize ASHA scheduler.
Args:
time_attr (str): Time attribute for scheduling
metric (str, optional): Metric to optimize
mode (str): "min" or "max"
max_t (int): Maximum time units
grace_period (int): Grace period before first halving
reduction_factor (int): Reduction factor for successive halving
brackets (int): Number of brackets
"""
class HyperBandScheduler:
"""Synchronous Hyperband scheduler."""
def __init__(self, *, time_attr="training_iteration", metric=None,
mode="max", max_t=81, reduction_factor=3, **kwargs):
"""
Initialize HyperBand scheduler.
Args:
time_attr (str): Time attribute for scheduling
metric (str, optional): Metric to optimize
mode (str): "min" or "max"
max_t (int): Maximum time units
reduction_factor (int): Reduction factor
"""
class MedianStoppingRule:
"""Stop trials below median performance."""
def __init__(self, *, time_attr="training_iteration", metric=None,
mode="max", grace_period=60, min_samples_required=3, **kwargs):
"""
Initialize median stopping rule.
Args:
time_attr (str): Time attribute for scheduling
metric (str, optional): Metric to optimize
mode (str): "min" or "max"
grace_period (int): Grace period before stopping
min_samples_required (int): Minimum samples needed
"""
class PopulationBasedTraining:
"""Population-based training scheduler."""
def __init__(self, *, time_attr="training_iteration", metric=None,
mode="max", perturbation_interval=60,
hyperparam_mutations=None, **kwargs):
"""
Initialize PBT scheduler.
Args:
time_attr (str): Time attribute for scheduling
metric (str, optional): Metric to optimize
mode (str): "min" or "max"
perturbation_interval (int): Interval between perturbations
hyperparam_mutations (dict, optional): Hyperparameter mutations
"""
class PopulationBasedTrainingReplay:
"""Replay population-based training."""
def __init__(self, policy_file):
"""
Initialize PBT replay.
Args:
policy_file (str): Path to PBT policy file
"""Search Space Definition
Define parameter search spaces.
def choice(categories):
"""
Choose from categorical options.
Args:
categories (list): List of options
Returns:
Choice distribution
"""
def randint(lower, upper):
"""
Random integer in range.
Args:
lower (int): Lower bound (inclusive)
upper (int): Upper bound (exclusive)
Returns:
Randint distribution
"""
def uniform(lower, upper):
"""
Uniform distribution in range.
Args:
lower (float): Lower bound
upper (float): Upper bound
Returns:
Uniform distribution
"""
def loguniform(lower, upper, base=10):
"""
Log-uniform distribution.
Args:
lower (float): Lower bound
upper (float): Upper bound
base (float): Logarithm base
Returns:
Loguniform distribution
"""
def randn(mean=0, sd=1):
"""
Normal distribution.
Args:
mean (float): Mean
sd (float): Standard deviation
Returns:
Normal distribution
"""
def lograndn(mean=0, sd=1, base=10):
"""
Log-normal distribution.
Args:
mean (float): Mean of log
sd (float): Standard deviation of log
base (float): Logarithm base
Returns:
Log-normal distribution
"""
def grid_search(values):
"""
Grid search over values.
Args:
values (list): Values to search over
Returns:
Grid search specification
"""
def sample_from(func):
"""
Sample from custom function.
Args:
func: Function that returns sample
Returns:
Sample specification
"""Experiment Analysis
Analyze and visualize tuning results.
class ExperimentAnalysis:
"""Analysis of tuning experiment results."""
def get_best_trial(self, metric=None, mode=None, scope="last"):
"""Get best trial."""
def get_best_config(self, metric=None, mode=None, scope="last"):
"""Get best configuration."""
def get_best_logdir(self, metric=None, mode=None, scope="last"):
"""Get best trial log directory."""
def get_trial_dataframes(self):
"""Get trial results as DataFrames."""
def dataframe(self, metric=None, mode=None):
"""Get results as DataFrame."""
def stats(self):
"""Get experiment statistics."""
def Analysis(experiment_checkpoint_path):
"""
Create ExperimentAnalysis from checkpoint.
Args:
experiment_checkpoint_path (str): Path to experiment checkpoint
Returns:
ExperimentAnalysis: Analysis object
"""Integration with Training
Integration with Ray Train for distributed hyperparameter tuning.
def with_parameters(trainable, **kwargs):
"""
Wrap trainable with fixed parameters.
Args:
trainable: Trainable function or class
**kwargs: Fixed parameters
Returns:
Wrapped trainable
"""
def with_resources(trainable, resources):
"""
Wrap trainable with resource requirements.
Args:
trainable: Trainable function or class
resources (dict): Resource requirements
Returns:
Wrapped trainable
"""Usage Examples
Basic Hyperparameter Tuning
import ray
from ray import tune
from ray.tune import TuneConfig, Tuner
ray.init()
def train_function(config):
# Training logic
for epoch in range(10):
loss = config["lr"] * (0.9 ** epoch)
accuracy = 1 - loss
# Report intermediate results
tune.report({"loss": loss, "accuracy": accuracy, "epoch": epoch})
# Define search space
param_space = {
"lr": tune.loguniform(1e-4, 1e-1),
"batch_size": tune.choice([16, 32, 64, 128]),
"hidden_size": tune.randint(32, 512)
}
# Configure tuner
tuner = Tuner(
train_function,
param_space=param_space,
tune_config=TuneConfig(
metric="accuracy",
mode="max",
num_samples=20
)
)
# Run experiment
results = tuner.fit()
# Get best result
best_result = results.get_best_result()
print(f"Best config: {best_result.config}")
print(f"Best accuracy: {best_result.metrics['accuracy']}")Advanced Tuning with Schedulers
import ray
from ray import tune
from ray.tune.schedulers import AsyncHyperBandScheduler
from ray.tune.search.hyperopt import HyperOptSearch
from hyperopt import hp
ray.init()
# Define search space using HyperOpt
search_space = {
"lr": hp.loguniform("lr", np.log(1e-4), np.log(1e-1)),
"batch_size": hp.choice("batch_size", [16, 32, 64, 128]),
"dropout": hp.uniform("dropout", 0.0, 0.5)
}
# Configure search algorithm
search_alg = HyperOptSearch(
space=search_space,
metric="accuracy",
mode="max"
)
# Configure scheduler
scheduler = AsyncHyperBandScheduler(
metric="accuracy",
mode="max",
grace_period=5,
reduction_factor=2
)
# Run tuning
analysis = tune.run(
train_function,
search_alg=search_alg,
scheduler=scheduler,
num_samples=50,
resources_per_trial={"cpu": 2, "gpu": 0.5}
)
# Analyze results
best_trial = analysis.get_best_trial("accuracy", "max")
print(f"Best trial config: {best_trial.config}")Population-Based Training
import ray
from ray import tune
from ray.tune.schedulers import PopulationBasedTraining
ray.init()
# Configure PBT
pbt = PopulationBasedTraining(
time_attr="training_iteration",
perturbation_interval=20,
hyperparam_mutations={
"lr": tune.loguniform(1e-4, 1e-1),
"batch_size": [16, 32, 64, 128]
}
)
# Run with PBT
analysis = tune.run(
train_function,
scheduler=pbt,
metric="accuracy",
mode="max",
num_samples=8,
config={
"lr": tune.choice([0.001, 0.01, 0.1]),
"batch_size": tune.choice([16, 32, 64])
}
)Integration with Ray Train
import ray
from ray import tune, train
from ray.train.torch import TorchTrainer
from ray.tune import TuneConfig, Tuner
ray.init()
def train_loop_per_worker(config):
# PyTorch training logic with config
model = create_model(config["hidden_size"])
optimizer = torch.optim.Adam(model.parameters(), lr=config["lr"])
for epoch in range(config["num_epochs"]):
# Training step
loss = train_step(model, optimizer)
accuracy = evaluate(model)
# Report to both Train and Tune
train.report({"loss": loss, "accuracy": accuracy})
# Define trainer
trainer = TorchTrainer(
train_loop_per_worker=train_loop_per_worker,
scaling_config=ScalingConfig(num_workers=4, use_gpu=True)
)
# Tune the trainer
tuner = Tuner(
trainer,
param_space={
"train_loop_config": {
"lr": tune.loguniform(1e-4, 1e-1),
"hidden_size": tune.randint(64, 512),
"num_epochs": 20
}
},
tune_config=TuneConfig(
metric="accuracy",
mode="max",
num_samples=10
)
)
results = tuner.fit()Install with Tessl CLI
npx tessl i tessl/pypi-ray