0
# Configuration
1

2
Configuration classes for Docker containerization, resource allocation, scheduling, caching, and other pipeline/step settings.
3

4
## Capabilities
5

6
### Docker Settings
7

8
```python { .api }
9
class DockerSettings:
10
    """
11
    Configuration for Docker containerization.
12

13
    Controls how ZenML builds and runs Docker containers for pipeline steps.
14

15
    Attributes:
16
    - parent_image: Base Docker image
17
    - dockerfile: Path to custom Dockerfile
18
    - build_context_root: Build context directory
19
    - build_options: Additional Docker build options dict
20
    - install_stack_requirements: Install stack component requirements
21
    - apt_packages: List of apt packages to install
22
    - requirements: List of pip requirements or path to requirements.txt
23
    - required_integrations: List of ZenML integrations to install
24
    - required_hub_plugins: List of ZenML Hub plugins
25
    - replicate_local_python_environment: Replicate local environment
26
    - environment: Environment variables dict
27
    - user: User to run container as
28
    - python_package_installer: Package installer (from PythonPackageInstaller enum)
29
    - python_package_installer_args: Additional installer arguments
30
    - skip_build: Skip building new image
31
    - target_repository: Target repository for built image
32
    """
33

34
    def __init__(
35
        self,
36
        parent_image: str = None,
37
        dockerfile: str = None,
38
        build_context_root: str = None,
39
        build_options: dict = None,
40
        install_stack_requirements: bool = True,
41
        apt_packages: list = None,
42
        requirements: list = None,
43
        required_integrations: list = None,
44
        required_hub_plugins: list = None,
45
        replicate_local_python_environment: str = None,
46
        environment: dict = None,
47
        user: str = None,
48
        python_package_installer: str = None,
49
        python_package_installer_args: dict = None,
50
        skip_build: bool = False,
51
        target_repository: str = None
52
    ):
53
        """
54
        Initialize Docker settings.
55

56
        Example:
57
        ```python
58
        from zenml.config import DockerSettings
59

60
        docker_settings = DockerSettings(
61
            parent_image="python:3.9-slim",
62
            requirements=["pandas==2.0.0", "scikit-learn==1.3.0"],
63
            apt_packages=["git", "curl"],
64
            environment={"MY_VAR": "value"},
65
            python_package_installer="pip"
66
        )
67
        ```
68
        """
69
```
70

71
Import from:
72

73
```python
74
from zenml.config import DockerSettings
75
```
76

77
### Resource Settings
78

79
```python { .api }
80
class ResourceSettings:
81
    """
82
    Hardware resource settings for steps and deployed pipelines.
83

84
    Attributes:
85
    - cpu_count: Amount of CPU cores (can be fractional, e.g., 0.5)
86
    - gpu_count: Number of GPUs
87
    - memory: Memory allocation string (e.g., "4GB", "512MB")
88
    - min_replicas: Minimum number of replicas (for deployed pipelines)
89
    - max_replicas: Maximum number of replicas (for deployed pipelines)
90
    - autoscaling_metric: Metric for autoscaling ("cpu", "memory", "concurrency", "rps")
91
    - autoscaling_target: Target value for autoscaling metric
92
    - max_concurrency: Maximum concurrent requests per instance
93
    """
94

95
    def __init__(
96
        self,
97
        cpu_count: float = None,
98
        gpu_count: int = None,
99
        memory: str = None,
100
        min_replicas: int = None,
101
        max_replicas: int = None,
102
        autoscaling_metric: str = None,
103
        autoscaling_target: float = None,
104
        max_concurrency: int = None
105
    ):
106
        """
107
        Initialize resource settings.
108

109
        Parameters:
110
        - cpu_count: Number of CPU cores (can be fractional, e.g., 0.5, 2.0)
111
        - gpu_count: Number of GPUs to allocate
112
        - memory: Memory to allocate (with unit, e.g., "4GB", "512MB", "2048MB")
113
        - min_replicas: Minimum replicas (for deployers/deployed pipelines)
114
        - max_replicas: Maximum replicas (for deployers/deployed pipelines)
115
        - autoscaling_metric: Metric for autoscaling - "cpu", "memory", "concurrency", "rps"
116
        - autoscaling_target: Target value for the metric (e.g., 75.0 for CPU percentage)
117
        - max_concurrency: Max concurrent requests per instance
118

119
        Example:
120
        ```python
121
        from zenml.config import ResourceSettings
122

123
        # Basic resources
124
        resources = ResourceSettings(
125
            cpu_count=8.0,
126
            gpu_count=2,
127
            memory="16GB"
128
        )
129

130
        # Deployed pipeline with autoscaling
131
        deployed_resources = ResourceSettings(
132
            cpu_count=2.0,
133
            memory="4GB",
134
            min_replicas=1,
135
            max_replicas=10,
136
            autoscaling_metric="cpu",
137
            autoscaling_target=75.0,
138
            max_concurrency=50
139
        )
140
        ```
141
        """
142
```
143

144
Import from:
145

146
```python
147
from zenml.config import ResourceSettings
148
from zenml.steps import ResourceSettings
149
```
150

151
### Schedule
152

153
```python { .api }
154
class Schedule:
155
    """
156
    Schedule configuration for pipeline runs.
157

158
    Supports both cron-based and interval-based scheduling.
159

160
    Attributes:
161
    - name: Schedule name
162
    - cron_expression: Cron expression (e.g., "0 0 * * *")
163
    - start_time: Schedule start datetime
164
    - end_time: Schedule end datetime
165
    - interval_second: Interval as timedelta between runs
166
    - catchup: Whether to catch up on missed runs
167
    - run_once_start_time: When to run the pipeline once
168
    """
169

170
    def __init__(
171
        self,
172
        name: str = None,
173
        cron_expression: str = None,
174
        start_time: datetime = None,
175
        end_time: datetime = None,
176
        interval_second: timedelta = None,
177
        catchup: bool = False,
178
        run_once_start_time: datetime = None
179
    ):
180
        """
181
        Initialize schedule configuration.
182

183
        Use either cron_expression or interval_second, not both.
184

185
        Parameters:
186
        - name: Schedule name
187
        - cron_expression: Cron expression
188
        - start_time: When to start the schedule
189
        - end_time: When to end the schedule
190
        - interval_second: Run interval as timedelta object
191
        - catchup: Whether to catch up on missed runs
192
        - run_once_start_time: When to run the pipeline once
193

194
        Example:
195
        ```python
196
        from zenml.config import Schedule
197
        from datetime import datetime, timedelta
198

199
        # Cron schedule - daily at midnight
200
        daily_schedule = Schedule(
201
            name="daily_training",
202
            cron_expression="0 0 * * *",
203
            start_time=datetime.now()
204
        )
205

206
        # Interval schedule - every 2 hours
207
        interval_schedule = Schedule(
208
            name="periodic_check",
209
            interval_second=2 * 60 * 60,
210
            start_time=datetime.now(),
211
            end_time=datetime.now() + timedelta(days=30)
212
        )
213
        ```
214
        """
215
```
216

217
Import from:
218

219
```python
220
from zenml.config import Schedule
221
from zenml.pipelines import Schedule
222
```
223

224
### Step Retry Config
225

226
```python { .api }
227
class StepRetryConfig:
228
    """
229
    Configuration for step retry behavior.
230

231
    Controls how steps are retried on failure.
232

233
    Attributes:
234
    - max_retries: Maximum number of retries
235
    - delay: Delay between retries in seconds
236
    - backoff: Backoff multiplier for delay
237
    """
238

239
    def __init__(
240
        self,
241
        max_retries: int = 0,
242
        delay: int = 1,
243
        backoff: int = 1
244
    ):
245
        """
246
        Initialize retry configuration.
247

248
        Parameters:
249
        - max_retries: Maximum number of retry attempts
250
        - delay: Initial delay between retries in seconds
251
        - backoff: Multiplier for delay on each retry (exponential backoff)
252

253
        Example:
254
        ```python
255
        from zenml.config import StepRetryConfig
256

257
        # Retry up to 3 times with exponential backoff
258
        retry_config = StepRetryConfig(
259
            max_retries=3,
260
            delay=5,      # Start with 5 seconds
261
            backoff=2     # Double delay each time: 5s, 10s, 20s
262
        )
263
        ```
264
        """
265
```
266

267
Import from:
268

269
```python
270
from zenml.config import StepRetryConfig
271
```
272

273
### Cache Policy
274

275
```python { .api }
276
class CachePolicy:
277
    """
278
    Configuration for step caching behavior.
279

280
    Controls which components are included in the cache key to determine when step outputs can be reused.
281

282
    Attributes:
283
    - include_step_code: Include step code in cache key (default: True)
284
    - include_step_parameters: Include step parameters in cache key (default: True)
285
    - include_artifact_values: Include artifact values in cache key (default: True)
286
    - include_artifact_ids: Include artifact IDs in cache key (default: True)
287
    - ignored_inputs: List of input names to ignore in cache key (default: None)
288
    """
289

290
    def __init__(
291
        self,
292
        include_step_code: bool = True,
293
        include_step_parameters: bool = True,
294
        include_artifact_values: bool = True,
295
        include_artifact_ids: bool = True,
296
        ignored_inputs: list = None
297
    ):
298
        """
299
        Initialize cache policy.
300

301
        Parameters:
302
        - include_step_code: Whether to include step code in cache key
303
        - include_step_parameters: Whether to include step parameters in cache key
304
        - include_artifact_values: Whether to include artifact values in cache key
305
        - include_artifact_ids: Whether to include artifact IDs in cache key
306
        - ignored_inputs: List of input names to ignore when computing cache key
307

308
        Example:
309
        ```python
310
        from zenml.config import CachePolicy
311

312
        # Default policy - includes everything
313
        default_policy = CachePolicy()
314

315
        # Ignore specific inputs
316
        selective_cache = CachePolicy(
317
            ignored_inputs=["timestamp", "random_seed"]
318
        )
319

320
        # Only cache based on step code, ignore parameters
321
        code_only = CachePolicy(
322
            include_step_code=True,
323
            include_step_parameters=False,
324
            include_artifact_values=False,
325
            include_artifact_ids=False
326
        )
327
        ```
328
        """
329

330
    @classmethod
331
    def default():
332
        """
333
        Get the default cache policy.
334

335
        Returns:
336
        CachePolicy: Default policy with all flags enabled
337
        """
338

339
    @classmethod
340
    def from_string(value: str):
341
        """
342
        Create a cache policy from a string.
343

344
        Parameters:
345
        - value: String value (currently supports "default")
346

347
        Returns:
348
        CachePolicy: Cache policy instance
349

350
        Raises:
351
        ValueError: If string is not a valid cache policy
352
        """
353
```
354

355
Import from:
356

357
```python
358
from zenml.config import CachePolicy
359
```
360

361
### Store Configuration
362

363
```python { .api }
364
class StoreConfiguration:
365
    """
366
    Configuration for the ZenML store backend.
367

368
    Controls connection to ZenML server or local store.
369

370
    Attributes:
371
    - type: Store type (SQL or REST)
372
    - url: Store URL
373
    - secrets_store: Secrets store configuration
374
    - backup_secrets_store: Backup secrets store configuration
375
    """
376
```
377

378
Import from:
379

380
```python
381
from zenml.config import StoreConfiguration
382
```
383

384
### Python Package Installer Enum
385

386
```python { .api }
387
class PythonPackageInstaller(str, Enum):
388
    """
389
    Python package installer options.
390

391
    Values:
392
    - PIP: Use pip
393
    - UV: Use uv (faster pip alternative)
394
    """
395
    PIP = "pip"
396
    UV = "uv"
397
```
398

399
Import from:
400

401
```python
402
from zenml.config import PythonPackageInstaller
403
```
404

405
### Byte Unit Enum
406

407
```python { .api }
408
class ByteUnit(str, Enum):
409
    """
410
    Units for memory/storage specifications.
411

412
    Values:
413
    - KB: Kilobytes
414
    - MB: Megabytes
415
    - GB: Gigabytes
416
    - TB: Terabytes
417
    - KIB: Kibibytes
418
    - MIB: Mebibytes
419
    - GIB: Gibibytes
420
    - TIB: Tebibytes
421
    """
422
    KB = "KB"
423
    MB = "MB"
424
    GB = "GB"
425
    TB = "TB"
426
    KIB = "KiB"
427
    MIB = "MiB"
428
    GIB = "GiB"
429
    TIB = "TiB"
430
```
431

432
Import from:
433

434
```python
435
from zenml.config import ByteUnit
436
```
437

438
## Usage Examples
439

440
### Docker Settings for Pipeline
441

442
```python
443
from zenml import pipeline, step
444
from zenml.config import DockerSettings
445

446
docker_settings = DockerSettings(
447
    parent_image="python:3.9-slim",
448
    requirements=["tensorflow==2.13.0", "numpy==1.24.0"],
449
    apt_packages=["libgomp1"],
450
    environment={
451
        "TF_ENABLE_ONEDNN_OPTS": "0",
452
        "CUDA_VISIBLE_DEVICES": "0"
453
    }
454
)
455

456
@step
457
def train_model(data: list) -> dict:
458
    import tensorflow as tf
459
    # Training logic
460
    return {"model": "trained"}
461

462
@pipeline(
463
    settings={
464
        "docker": docker_settings
465
    }
466
)
467
def training_pipeline():
468
    data = [1, 2, 3]
469
    model = train_model(data)
470
    return model
471
```
472

473
### Resource Allocation
474

475
```python
476
from zenml import step
477
from zenml.config import ResourceSettings
478

479
@step(
480
    settings={
481
        "resources": ResourceSettings(
482
            cpu_count=16,
483
            gpu_count=4,
484
            memory="64GB"
485
        )
486
    }
487
)
488
def large_scale_training(data: list) -> dict:
489
    """Step requiring significant resources."""
490
    # Heavy training logic
491
    return {"model": "large_model"}
492
```
493

494
### Scheduled Pipeline
495

496
```python
497
from zenml import pipeline
498
from zenml.config import Schedule
499
from datetime import datetime, timedelta
500

501
# Daily schedule at 2 AM
502
schedule = Schedule(
503
    name="nightly_training",
504
    cron_expression="0 2 * * *",
505
    start_time=datetime.now(),
506
    end_time=datetime.now() + timedelta(days=365)
507
)
508

509
@pipeline(schedule=schedule)
510
def scheduled_pipeline():
511
    # Pipeline definition
512
    pass
513
```
514

515
### Retry Configuration
516

517
```python
518
from zenml import step
519
from zenml.config import StepRetryConfig
520

521
retry_config = StepRetryConfig(
522
    max_retries=5,
523
    delay=10,
524
    backoff=2
525
)
526

527
@step(
528
    settings={
529
        "retry": retry_config
530
    }
531
)
532
def flaky_external_api_call(endpoint: str) -> dict:
533
    """Step that might fail due to network issues."""
534
    # API call that might fail
535
    return {"status": "success"}
536
```
537

538
### Cache Policy
539

540
```python
541
from zenml import step
542
from zenml.config import CachePolicy
543

544
# Ignore specific inputs in cache key
545
selective_cache = CachePolicy(
546
    ignored_inputs=["timestamp", "random_seed"]
547
)
548

549
@step(cache_policy=selective_cache)
550
def process_data(data: dict, timestamp: str, random_seed: int) -> dict:
551
    """Step that ignores timestamp and random_seed for caching."""
552
    # Process data - cache will only consider 'data' input
553
    return {"processed": data}
554

555
# Cache only based on step code, ignore parameters and artifacts
556
code_only_cache = CachePolicy(
557
    include_step_code=True,
558
    include_step_parameters=False,
559
    include_artifact_values=False,
560
    include_artifact_ids=False
561
)
562

563
@step(cache_policy=code_only_cache)
564
def generate_random_data() -> list:
565
    """Step cached only by code version."""
566
    import random
567
    return [random.random() for _ in range(10)]
568

569
# Use default cache policy
570
@step(cache_policy=CachePolicy.default())
571
def standard_processing(input_data: str) -> dict:
572
    """Step with default caching behavior."""
573
    return {"result": input_data}
574
```
575

576
### Combined Settings
577

578
```python
579
from zenml import pipeline, step
580
from zenml.config import DockerSettings, ResourceSettings, StepRetryConfig
581

582
docker_settings = DockerSettings(
583
    parent_image="nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04",
584
    requirements=["torch==2.0.0", "transformers==4.30.0"],
585
    environment={"HF_HOME": "/cache"}
586
)
587

588
resource_settings = ResourceSettings(
589
    cpu_count=8,
590
    gpu_count=2,
591
    memory="32GB"
592
)
593

594
retry_config = StepRetryConfig(
595
    max_retries=3,
596
    delay=30,
597
    backoff=2
598
)
599

600
@step(
601
    settings={
602
        "docker": docker_settings,
603
        "resources": resource_settings,
604
        "retry": retry_config
605
    }
606
)
607
def train_transformer(data: str) -> dict:
608
    """LLM training with full configuration."""
609
    # Training logic
610
    return {"model": "trained_transformer"}
611

612
@pipeline(
613
    settings={
614
        "docker": docker_settings
615
    }
616
)
617
def llm_pipeline():
618
    model = train_transformer("training_data")
619
    return model
620
```
621

622
### Custom Docker Image
623

624
```python
625
from zenml import pipeline
626
from zenml.config import DockerSettings
627

628
# Use custom Dockerfile
629
docker_settings = DockerSettings(
630
    dockerfile="./docker/Dockerfile",
631
    build_context_root="./",
632
    build_options={
633
        "buildargs": {
634
            "BASE_IMAGE": "python:3.9"
635
        }
636
    }
637
)
638

639
@pipeline(
640
    settings={
641
        "docker": docker_settings
642
    }
643
)
644
def custom_docker_pipeline():
645
    # Pipeline using custom Docker image
646
    pass
647
```
648