0
# Parameters
1
2
Luigi's parameter system provides type-safe configuration for tasks with automatic parsing, validation, and command-line integration. Parameters make tasks reusable and configurable.
3
4
## Capabilities
5
6
### Base Parameter Class
7
8
The fundamental parameter class that all Luigi parameters inherit from. Provides basic parameter functionality and configuration options.
9
10
```python { .api }
11
class Parameter:
12
"""Base parameter class for string values."""
13
14
def __init__(self, default=_no_value, is_global: bool = False, significant: bool = True,
15
description: str = None, config_path: dict = None, positional: bool = True,
16
always_in_help: bool = False, batch_method=None,
17
visibility: ParameterVisibility = ParameterVisibility.PUBLIC):
18
"""
19
Initialize parameter.
20
21
Args:
22
default: Default value if not provided (_no_value means required)
23
is_global: Whether parameter is global across all tasks
24
significant: Whether parameter is part of task's unique identifier
25
description: Parameter description for help text
26
config_path: Configuration path for parameter lookup
27
positional: Whether parameter can be specified positionally
28
always_in_help: Whether to always show parameter in help
29
batch_method: Method for handling batch parameter values
30
visibility: Parameter visibility level
31
"""
32
33
def parse(self, x: str):
34
"""
35
Parse string value to parameter type.
36
37
Args:
38
x: String value to parse
39
40
Returns:
41
Parsed value of appropriate type
42
"""
43
44
def serialize(self, x) -> str:
45
"""
46
Serialize parameter value to string.
47
48
Args:
49
x: Value to serialize
50
51
Returns:
52
str: Serialized string representation
53
"""
54
55
def normalize(self, x):
56
"""Normalize parameter value."""
57
58
@property
59
def has_task_value(self, task_family: str, param_name: str) -> bool:
60
"""Check if parameter has value for specific task."""
61
```
62
63
### Primitive Parameter Types
64
65
Basic parameter types for common data types with automatic parsing and validation.
66
67
```python { .api }
68
class IntParameter(Parameter):
69
"""Parameter for integer values."""
70
71
def parse(self, x: str) -> int:
72
"""Parse string to integer."""
73
74
class FloatParameter(Parameter):
75
"""Parameter for floating-point values."""
76
77
def parse(self, x: str) -> float:
78
"""Parse string to float."""
79
80
class BoolParameter(Parameter):
81
"""Parameter for boolean values."""
82
83
def parse(self, x: str) -> bool:
84
"""Parse string to boolean (accepts 'true', 'false', etc.)."""
85
86
class NumericalParameter(Parameter):
87
"""Base class for numerical parameters with min/max validation."""
88
89
def __init__(self, min_value=None, max_value=None, **kwargs):
90
"""
91
Initialize numerical parameter.
92
93
Args:
94
min_value: Minimum allowed value
95
max_value: Maximum allowed value
96
**kwargs: Additional parameter options
97
"""
98
99
class ChoiceParameter(Parameter):
100
"""Parameter that accepts one value from a predefined set."""
101
102
def __init__(self, choices, var_type=str, **kwargs):
103
"""
104
Initialize choice parameter.
105
106
Args:
107
choices: List/tuple of allowed values
108
var_type: Type for choice values
109
**kwargs: Additional parameter options
110
"""
111
```
112
113
### Date and Time Parameters
114
115
Specialized parameters for handling date and time values with automatic parsing and formatting.
116
117
```python { .api }
118
class DateParameter(Parameter):
119
"""Parameter for date values (datetime.date)."""
120
121
def parse(self, x: str) -> date:
122
"""Parse string to date (YYYY-MM-DD format)."""
123
124
class MonthParameter(DateParameter):
125
"""Parameter for month values (YYYY-MM format)."""
126
127
def parse(self, x: str) -> date:
128
"""Parse string to first day of month."""
129
130
class YearParameter(DateParameter):
131
"""Parameter for year values (YYYY format)."""
132
133
def parse(self, x: str) -> date:
134
"""Parse string to first day of year."""
135
136
class DateHourParameter(Parameter):
137
"""Parameter for date-hour values (YYYY-MM-DD-HH format)."""
138
139
def parse(self, x: str) -> datetime:
140
"""Parse string to datetime with hour precision."""
141
142
class DateMinuteParameter(Parameter):
143
"""Parameter for date-minute values (YYYY-MM-DD-HH-MM format)."""
144
145
def parse(self, x: str) -> datetime:
146
"""Parse string to datetime with minute precision."""
147
148
class DateSecondParameter(Parameter):
149
"""Parameter for date-second values (YYYY-MM-DD-HH-MM-SS format)."""
150
151
def parse(self, x: str) -> datetime:
152
"""Parse string to datetime with second precision."""
153
154
class DateIntervalParameter(Parameter):
155
"""Parameter for date interval values."""
156
157
def parse(self, x: str):
158
"""Parse string to DateInterval object."""
159
160
class TimeDeltaParameter(Parameter):
161
"""Parameter for time delta values."""
162
163
def parse(self, x: str) -> timedelta:
164
"""Parse string to timedelta object."""
165
```
166
167
### Collection Parameters
168
169
Parameters for handling collections of values including lists, tuples, and dictionaries.
170
171
```python { .api }
172
class ListParameter(Parameter):
173
"""Parameter for list values."""
174
175
def parse(self, x: str) -> list:
176
"""Parse JSON string to list."""
177
178
def normalize(self, x) -> tuple:
179
"""Normalize list to tuple for hashing."""
180
181
class TupleParameter(Parameter):
182
"""Parameter for tuple values."""
183
184
def parse(self, x: str) -> tuple:
185
"""Parse JSON string to tuple."""
186
187
class DictParameter(Parameter):
188
"""Parameter for dictionary values."""
189
190
def parse(self, x: str) -> dict:
191
"""Parse JSON string to dictionary."""
192
193
def normalize(self, x):
194
"""Normalize dictionary for hashing."""
195
196
class EnumParameter(Parameter):
197
"""Parameter for enumeration values."""
198
199
def __init__(self, enum_class, **kwargs):
200
"""
201
Initialize enum parameter.
202
203
Args:
204
enum_class: Enum class for allowed values
205
**kwargs: Additional parameter options
206
"""
207
208
def parse(self, x: str):
209
"""Parse string to enum value."""
210
211
class EnumListParameter(ListParameter):
212
"""Parameter for lists of enumeration values."""
213
214
def __init__(self, enum_class, **kwargs):
215
"""
216
Initialize enum list parameter.
217
218
Args:
219
enum_class: Enum class for list elements
220
**kwargs: Additional parameter options
221
"""
222
```
223
224
### Advanced Parameters
225
226
Specialized parameters for complex use cases including task references and optional parameters.
227
228
```python { .api }
229
class TaskParameter(Parameter):
230
"""Parameter that references another task."""
231
232
def parse(self, x: str):
233
"""Parse string to task instance."""
234
235
class OptionalParameter(Parameter):
236
"""Wrapper that makes any parameter optional."""
237
238
def __init__(self, parameter: Parameter, **kwargs):
239
"""
240
Initialize optional parameter wrapper.
241
242
Args:
243
parameter: Base parameter to make optional
244
**kwargs: Additional parameter options
245
"""
246
247
def parse(self, x: str):
248
"""Parse using wrapped parameter if value provided."""
249
```
250
251
### Parameter Configuration
252
253
Configuration classes and enums for parameter behavior and visibility.
254
255
```python { .api }
256
class ParameterVisibility:
257
"""Enumeration for parameter visibility levels."""
258
PUBLIC = 0 # Visible in help and task signatures
259
HIDDEN = 1 # Hidden from help but included in task signatures
260
PRIVATE = 2 # Excluded from task signatures
261
262
@classmethod
263
def has_value(cls, value: int) -> bool:
264
"""Check if value is valid visibility level."""
265
266
def serialize(self) -> int:
267
"""Serialize visibility level."""
268
```
269
270
### Parameter Exceptions
271
272
Exception classes for parameter parsing and validation errors.
273
274
```python { .api }
275
class ParameterException(Exception):
276
"""Base exception for parameter errors."""
277
278
class MissingParameterException(ParameterException):
279
"""Exception raised when required parameter is missing."""
280
281
class UnknownParameterException(ParameterException):
282
"""Exception raised when unknown parameter is provided."""
283
284
class DuplicateParameterException(ParameterException):
285
"""Exception raised when parameter is specified multiple times."""
286
```
287
288
## Usage Examples
289
290
### Basic Parameter Usage
291
292
```python
293
import luigi
294
from luigi import Task, Parameter, IntParameter, DateParameter
295
from datetime import date
296
297
class ProcessDataTask(Task):
298
"""Task with various parameter types."""
299
300
# String parameter with default
301
dataset_name = Parameter(default="default_dataset")
302
303
# Integer parameter
304
batch_size = IntParameter(default=1000)
305
306
# Date parameter
307
process_date = DateParameter(default=date.today())
308
309
def output(self):
310
return luigi.LocalTarget(f"output/{self.dataset_name}_{self.process_date}.csv")
311
312
def run(self):
313
print(f"Processing {self.dataset_name} with batch size {self.batch_size} for {self.process_date}")
314
315
# Task logic here
316
with self.output().open('w') as f:
317
f.write(f"Processed {self.batch_size} records")
318
319
# Run with parameters from command line:
320
# python script.py ProcessDataTask --dataset-name "production" --batch-size 5000 --process-date 2023-01-15
321
```
322
323
### Collection Parameters
324
325
```python
326
import luigi
327
from luigi import Task, ListParameter, DictParameter
328
import json
329
330
class ConfigurableTask(Task):
331
"""Task with collection parameters."""
332
333
# List of input files
334
input_files = ListParameter()
335
336
# Configuration dictionary
337
config = DictParameter(default={})
338
339
def run(self):
340
print(f"Processing files: {self.input_files}")
341
print(f"Configuration: {self.config}")
342
343
# Process each input file
344
for file_path in self.input_files:
345
print(f"Processing {file_path}")
346
347
# Run with collection parameters:
348
# python script.py ConfigurableTask --input-files '["file1.csv", "file2.csv"]' --config '{"timeout": 30, "retry": true}'
349
```
350
351
### Optional and Choice Parameters
352
353
```python
354
import luigi
355
from luigi import Task, ChoiceParameter, OptionalParameter, IntParameter
356
357
class FlexibleTask(Task):
358
"""Task with optional and choice parameters."""
359
360
# Choice parameter with predefined options
361
mode = ChoiceParameter(choices=['fast', 'accurate', 'balanced'], default='balanced')
362
363
# Optional parameter that may or may not be provided
364
max_memory = OptionalParameter(IntParameter())
365
366
def run(self):
367
print(f"Running in {self.mode} mode")
368
369
if self.max_memory is not None:
370
print(f"Memory limit: {self.max_memory} MB")
371
else:
372
print("No memory limit specified")
373
374
# Run with optional parameters:
375
# python script.py FlexibleTask --mode fast --max-memory 2048
376
# python script.py FlexibleTask --mode accurate # max_memory will be None
377
```
378
379
### Date and Time Parameters
380
381
```python
382
import luigi
383
from luigi import Task, DateParameter, DateHourParameter, TimeDeltaParameter
384
from datetime import date, datetime, timedelta
385
386
class TimeBasedTask(Task):
387
"""Task with various date/time parameters."""
388
389
# Daily processing date
390
date = DateParameter()
391
392
# Hourly processing timestamp
393
hour = DateHourParameter()
394
395
# Processing window duration
396
window = TimeDeltaParameter(default=timedelta(hours=1))
397
398
def run(self):
399
print(f"Processing date: {self.date}")
400
print(f"Processing hour: {self.hour}")
401
print(f"Window duration: {self.window}")
402
403
# Run with date/time parameters:
404
# python script.py TimeBasedTask --date 2023-01-15 --hour 2023-01-15-14 --window "2:00:00"
405
```
406
407
### Task Parameter for Dependencies
408
409
```python
410
import luigi
411
from luigi import Task, TaskParameter
412
413
class DataSource(Task):
414
"""Source data task."""
415
source_id = luigi.Parameter()
416
417
def output(self):
418
return luigi.LocalTarget(f"data/source_{self.source_id}.csv")
419
420
class DataProcessor(Task):
421
"""Task that processes data from a configurable source."""
422
423
# Parameter that references another task
424
source_task = TaskParameter()
425
426
def requires(self):
427
return self.source_task
428
429
def output(self):
430
return luigi.LocalTarget(f"data/processed_{self.source_task.source_id}.csv")
431
432
def run(self):
433
with self.input().open('r') as f:
434
data = f.read()
435
436
# Process data
437
processed = data.upper()
438
439
with self.output().open('w') as f:
440
f.write(processed)
441
442
# Run with task parameter:
443
# python script.py DataProcessor --source-task DataSource --source-task-source-id "dataset1"
444
```
445
446
## Constants
447
448
```python { .api }
449
# Special default value sentinel indicating parameter is required
450
_no_value = object()
451
452
class ParameterVisibility:
453
"""Parameter visibility levels."""
454
PUBLIC = 0
455
HIDDEN = 1
456
PRIVATE = 2
457
```