0
# Core Framework
1
2
Essential classes for configuring optimization environments, managing trial history, and handling optimization events through callbacks.
3
4
## Capabilities
5
6
### Environment Configuration
7
8
The Scenario class defines all optimization settings, constraints, and environment configuration.
9
10
```python { .api }
11
class Scenario:
12
def __init__(
13
self,
14
configspace: ConfigurationSpace,
15
name: str | None = None,
16
output_directory: Path = Path("smac3_output"),
17
deterministic: bool = False,
18
objectives: str | list[str] | None = "cost",
19
crash_cost: float | list[float] = np.inf,
20
termination_cost_threshold: float | list[float] = np.inf,
21
walltime_limit: float = np.inf,
22
cputime_limit: float = np.inf,
23
trial_walltime_limit: float | None = None,
24
trial_memory_limit: int | None = None,
25
n_trials: int = 100,
26
use_default_config: bool = False,
27
instances: list[str] | None = None,
28
instance_features: dict[str, list[float]] | None = None,
29
min_budget: float | int | None = None,
30
max_budget: float | int | None = None,
31
seed: int = 0,
32
n_workers: int = 1
33
):
34
"""
35
Environment configuration for optimization.
36
37
Parameters:
38
- configspace: Search space definition (required)
39
- name: Run identifier for output organization
40
- output_directory: Directory for logs and results
41
- deterministic: Whether to use single seed per evaluation
42
- objectives: Objective names (single or multi-objective)
43
- crash_cost: Cost assigned to failed trials
44
- termination_cost_threshold: Early stopping threshold
45
- walltime_limit: Maximum wall-clock time in seconds
46
- cputime_limit: Maximum CPU time in seconds
47
- trial_walltime_limit: Per-trial time limit in seconds
48
- trial_memory_limit: Per-trial memory limit in MB
49
- n_trials: Maximum number of trials
50
- use_default_config: Include default config in initial design
51
- instances: Problem instances for algorithm configuration
52
- instance_features: Feature vectors for each instance
53
- min_budget: Multi-fidelity minimum budget
54
- max_budget: Multi-fidelity maximum budget
55
- seed: Random seed for reproducibility
56
- n_workers: Number of parallel workers
57
"""
58
59
def count_objectives(self) -> int:
60
"""Get number of objectives."""
61
62
def count_instance_features(self) -> int:
63
"""Get number of instance features."""
64
65
def save(self) -> None:
66
"""Save scenario to file."""
67
68
@staticmethod
69
def load(path: Path) -> Scenario:
70
"""Load scenario from file."""
71
72
@staticmethod
73
def make_serializable(scenario: Scenario) -> dict[str, Any]:
74
"""Convert scenario to JSON-serializable format."""
75
76
@property
77
def meta(self) -> dict[str, Any]:
78
"""Metadata dictionary."""
79
```
80
81
Example usage:
82
83
```python
84
from smac import Scenario
85
from ConfigSpace import ConfigurationSpace, Float
86
from pathlib import Path
87
88
# Basic scenario
89
config_space = ConfigurationSpace()
90
config_space.add_hyperparameter(Float("learning_rate", bounds=(1e-5, 1e-1), log=True))
91
92
scenario = Scenario(
93
configspace=config_space,
94
name="ml_optimization",
95
n_trials=100,
96
walltime_limit=3600, # 1 hour
97
seed=42
98
)
99
100
# Multi-fidelity scenario
101
multi_fidelity_scenario = Scenario(
102
configspace=config_space,
103
name="multi_fidelity_opt",
104
n_trials=50,
105
min_budget=0.1,
106
max_budget=1.0,
107
seed=42
108
)
109
110
# Algorithm configuration scenario
111
instances = ["instance_1", "instance_2", "instance_3"]
112
instance_features = {
113
"instance_1": [1.0, 2.0, 3.0],
114
"instance_2": [2.0, 3.0, 4.0],
115
"instance_3": [3.0, 4.0, 5.0]
116
}
117
118
algo_config_scenario = Scenario(
119
configspace=config_space,
120
instances=instances,
121
instance_features=instance_features,
122
n_trials=200,
123
trial_walltime_limit=60, # 1 minute per trial
124
seed=42
125
)
126
```
127
128
### Trial History Management
129
130
The RunHistory class stores and manages optimization trial results with support for multi-objective and multi-fidelity scenarios.
131
132
```python { .api }
133
class RunHistory:
134
def __init__(
135
self,
136
multi_objective_algorithm: AbstractMultiObjectiveAlgorithm | None = None,
137
overwrite_existing_trials: bool = False
138
):
139
"""
140
Container for storing and managing optimization trial results.
141
142
Parameters:
143
- multi_objective_algorithm: Strategy for handling multiple objectives
144
- overwrite_existing_trials: Whether to overwrite existing trials
145
"""
146
147
def add(
148
self,
149
config: Configuration,
150
cost: int | float | list[int | float],
151
time: float = 0.0,
152
cpu_time: float = 0.0,
153
status: StatusType = StatusType.SUCCESS,
154
instance: str | None = None,
155
seed: int | None = None,
156
budget: float | None = None,
157
starttime: float = 0.0,
158
endtime: float = 0.0,
159
additional_info: dict[str, Any] = None,
160
force_update: bool = False
161
) -> None:
162
"""Add trial result with individual parameters."""
163
164
def add_trial(self, info: TrialInfo, value: TrialValue) -> None:
165
"""Add trial using structured data objects."""
166
167
def add_running_trial(self, trial: TrialInfo) -> None:
168
"""Mark trial as currently running."""
169
170
def get_cost(
171
self,
172
config: Configuration,
173
*,
174
instance: str | None = None,
175
seed: int | None = None,
176
budget: float | None = None
177
) -> float:
178
"""Get empirical cost for specific configuration context."""
179
180
def get_min_cost(
181
self,
182
config: Configuration,
183
*,
184
instance: str | None = None,
185
seed: int | None = None,
186
budget: float | None = None
187
) -> float:
188
"""Get minimum observed cost for configuration context."""
189
190
def average_cost(
191
self,
192
config: Configuration,
193
*,
194
instances: list[str] | None = None,
195
seeds: list[int] | None = None,
196
budgets: list[float] | None = None,
197
normalize: bool = True
198
) -> float:
199
"""Compute average cost across specified contexts."""
200
201
def sum_cost(
202
self,
203
config: Configuration,
204
*,
205
instances: list[str] | None = None,
206
seeds: list[int] | None = None,
207
budgets: list[float] | None = None,
208
normalize: bool = True
209
) -> float:
210
"""Compute sum of costs across specified contexts."""
211
212
def min_cost(
213
self,
214
config: Configuration,
215
*,
216
instances: list[str] | None = None,
217
seeds: list[int] | None = None,
218
budgets: list[float] | None = None,
219
normalize: bool = True
220
) -> float:
221
"""Compute minimum cost across specified contexts."""
222
223
def get_configs(self, sort_by: str | None = None) -> list[Configuration]:
224
"""Get all evaluated configurations, optionally sorted."""
225
226
def get_trials(
227
self,
228
config: Configuration,
229
highest_observed_budget_only: bool = True
230
) -> list[TrialKey]:
231
"""Get trials for specific configuration."""
232
233
def get_running_trials(self, config: Configuration) -> list[TrialKey]:
234
"""Get currently running trials for configuration."""
235
236
def update_cost(self, config: Configuration) -> None:
237
"""Recompute aggregated costs for configuration."""
238
239
def save(self, filename: str | Path) -> None:
240
"""Save run history to JSON file."""
241
242
@staticmethod
243
def load(filename: str | Path, configspace: ConfigurationSpace) -> RunHistory:
244
"""Load run history from JSON file."""
245
246
def update(self, runhistory: RunHistory) -> None:
247
"""Merge another run history into this one."""
248
249
def get_config(self, config_id: int) -> Configuration:
250
"""Get configuration by ID."""
251
252
def get_config_id(self, config: Configuration) -> int:
253
"""Get ID for configuration."""
254
255
def has_config(self, config: Configuration) -> bool:
256
"""Check if configuration exists in history."""
257
258
def empty(self) -> bool:
259
"""Check if run history is empty."""
260
261
def reset(self) -> None:
262
"""Reset run history to empty state."""
263
264
@property
265
def submitted(self) -> int:
266
"""Number of submitted trials."""
267
268
@property
269
def finished(self) -> int:
270
"""Number of finished trials."""
271
272
@property
273
def running(self) -> int:
274
"""Number of currently running trials."""
275
276
@property
277
def ids_config(self) -> dict[int, Configuration]:
278
"""Mapping from configuration ID to Configuration object."""
279
280
@property
281
def config_ids(self) -> dict[Configuration, int]:
282
"""Mapping from Configuration object to ID."""
283
284
@property
285
def objective_bounds(self) -> list[tuple[float, float]]:
286
"""Min/max bounds for each objective."""
287
```
288
289
### Trial Execution Status
290
291
Enumeration of possible trial execution outcomes.
292
293
```python { .api }
294
class StatusType(IntEnum):
295
"""Trial execution status constants."""
296
RUNNING = 0 # In case a job was submitted, but it has not finished
297
SUCCESS = 1 # Trial completed successfully
298
CRASHED = 2 # Trial crashed or failed
299
TIMEOUT = 3 # Trial exceeded time limit
300
MEMORYOUT = 4 # Trial exceeded memory limit
301
```
302
303
### Event Handling
304
305
Base callback class for handling optimization events and implementing custom logging, visualization, or early stopping logic.
306
307
```python { .api }
308
class Callback:
309
"""Abstract base class for optimization event hooks."""
310
311
def on_start(self, smbo: SMBO) -> None:
312
"""Called before optimization starts."""
313
314
def on_end(self, smbo: SMBO) -> None:
315
"""Called after optimization finishes."""
316
317
def on_iteration_start(self, smbo: SMBO) -> None:
318
"""Called before each optimization iteration."""
319
320
def on_iteration_end(self, smbo: SMBO) -> None:
321
"""Called after each optimization iteration."""
322
323
def on_next_configurations_start(self, config_selector: ConfigSelector) -> None:
324
"""Called before model training and configuration selection."""
325
326
def on_next_configurations_end(
327
self,
328
config_selector: ConfigSelector,
329
config: list[Configuration]
330
) -> None:
331
"""Called after configuration selection."""
332
333
def on_ask_start(self, smbo: SMBO) -> None:
334
"""Called before intensifier asks for next trial."""
335
336
def on_ask_end(self, smbo: SMBO, info: TrialInfo) -> None:
337
"""Called after intensifier provides trial information."""
338
339
def on_tell_start(self, smbo: SMBO, info: TrialInfo, value: TrialValue) -> None:
340
"""Called before processing trial result."""
341
342
def on_tell_end(self, smbo: SMBO, info: TrialInfo, value: TrialValue) -> None:
343
"""Called after processing trial result."""
344
```
345
346
### Metadata Callback
347
348
Concrete callback for saving optimization run metadata.
349
350
```python { .api }
351
class MetadataCallback(Callback):
352
def __init__(self, **kwargs: str | int | float | dict | list) -> None:
353
"""
354
Callback for saving optimization run metadata.
355
356
Parameters:
357
- **kwargs: Arbitrary JSON-serializable metadata key-value pairs to save
358
"""
359
360
def on_start(self, smbo: SMBO) -> None:
361
"""Called before the optimization starts to save metadata."""
362
```
363
364
Example callback usage:
365
366
```python
367
from smac import HyperparameterOptimizationFacade, Scenario, Callback, MetadataCallback
368
369
class EarlyStoppingCallback(Callback):
370
def __init__(self, patience=10, min_improvement=0.01):
371
self.patience = patience
372
self.min_improvement = min_improvement
373
self.best_cost = float('inf')
374
self.no_improvement_count = 0
375
376
def on_tell_end(self, smbo, info, value):
377
current_cost = value.cost
378
if current_cost < self.best_cost - self.min_improvement:
379
self.best_cost = current_cost
380
self.no_improvement_count = 0
381
else:
382
self.no_improvement_count += 1
383
384
if self.no_improvement_count >= self.patience:
385
print(f"Early stopping after {self.patience} iterations without improvement")
386
# Custom early stopping logic here
387
388
# Use callbacks
389
metadata_callback = MetadataCallback(
390
experiment_name="hyperparameter_tuning",
391
dataset="mnist",
392
model_type="neural_network"
393
)
394
395
early_stop_callback = EarlyStoppingCallback(patience=15)
396
397
facade = HyperparameterOptimizationFacade(
398
scenario,
399
objective,
400
callbacks=[metadata_callback, early_stop_callback]
401
)
402
```