0
# Context Management
1
2
Process context configuration for controlling process start methods and execution environments. Billiard provides flexible context management for different multiprocessing scenarios.
3
4
## Capabilities
5
6
### Context Functions
7
8
Global functions for managing the default process context.
9
10
```python { .api }
11
def get_context(method=None):
12
"""
13
Get a process context.
14
15
Parameters:
16
- method: start method ('fork', 'spawn', 'forkserver', or None for default)
17
18
Returns:
19
BaseContext object configured for the specified method
20
"""
21
22
def set_start_method(method, force=False):
23
"""
24
Set the default start method for processes.
25
26
Parameters:
27
- method: start method ('fork', 'spawn', 'forkserver')
28
- force: if True, allow changing an already set method
29
30
Raises:
31
- RuntimeError: if method already set and force=False
32
- ValueError: if method is not supported on current platform
33
"""
34
35
def get_start_method(allow_none=False) -> str:
36
"""
37
Get the current default start method.
38
39
Parameters:
40
- allow_none: if True, return None if no method set; otherwise set default
41
42
Returns:
43
String name of current start method
44
"""
45
46
def get_all_start_methods() -> list[str]:
47
"""
48
Get list of all available start methods on current platform.
49
50
Returns:
51
List of available start method names
52
"""
53
```
54
55
Usage example:
56
57
```python
58
import billiard as mp
59
from billiard import Process
60
import os
61
62
def worker_info():
63
"""Print information about worker process"""
64
print(f"Worker PID: {os.getpid()}")
65
print(f"Parent PID: {os.getppid()}")
66
print(f"Start method: {mp.get_start_method()}")
67
68
def context_basics_example():
69
"""Demonstrate basic context management"""
70
print(f"Available start methods: {mp.get_all_start_methods()}")
71
print(f"Current start method: {mp.get_start_method()}")
72
73
# Set start method (must be done before creating processes)
74
try:
75
mp.set_start_method('spawn', force=True)
76
print(f"Set start method to: {mp.get_start_method()}")
77
except ValueError as e:
78
print(f"Could not set start method: {e}")
79
80
# Create process with current context
81
process = Process(target=worker_info)
82
process.start()
83
process.join()
84
85
if __name__ == '__main__':
86
context_basics_example()
87
```
88
89
### Context Objects
90
91
Context objects provide isolated multiprocessing environments with specific configurations.
92
93
```python { .api }
94
class BaseContext:
95
"""
96
Base class for multiprocessing contexts.
97
"""
98
# Process management
99
def Process(self, group=None, target=None, name=None, args=(), kwargs={}, daemon=None):
100
"""Create a Process using this context."""
101
102
def current_process(self):
103
"""Get current process object."""
104
105
def active_children(self):
106
"""Get list of active child processes."""
107
108
def cpu_count(self):
109
"""Get number of CPUs."""
110
111
# Synchronization primitives
112
def Lock(self):
113
"""Create a Lock using this context."""
114
115
def RLock(self):
116
"""Create an RLock using this context."""
117
118
def Semaphore(self, value=1):
119
"""Create a Semaphore using this context."""
120
121
def BoundedSemaphore(self, value=1):
122
"""Create a BoundedSemaphore using this context."""
123
124
def Condition(self, lock=None):
125
"""Create a Condition using this context."""
126
127
def Event(self):
128
"""Create an Event using this context."""
129
130
def Barrier(self, parties, action=None, timeout=None):
131
"""Create a Barrier using this context."""
132
133
# Communication
134
def Pipe(self, duplex=True):
135
"""Create a Pipe using this context."""
136
137
def Queue(self, maxsize=0):
138
"""Create a Queue using this context."""
139
140
def JoinableQueue(self, maxsize=0):
141
"""Create a JoinableQueue using this context."""
142
143
def SimpleQueue(self):
144
"""Create a SimpleQueue using this context."""
145
146
# Shared memory
147
def Value(self, typecode_or_type, *args, lock=True):
148
"""Create a shared Value using this context."""
149
150
def Array(self, typecode_or_type, size_or_initializer, lock=True):
151
"""Create a shared Array using this context."""
152
153
# Pool
154
def Pool(self, processes=None, initializer=None, initargs=(),
155
maxtasksperchild=None):
156
"""Create a Pool using this context."""
157
158
# Manager
159
def Manager(self):
160
"""Create a Manager using this context."""
161
```
162
163
Usage example:
164
165
```python
166
import billiard as mp
167
import os
168
import time
169
170
def context_worker(context_name, queue):
171
"""Worker that reports its context"""
172
pid = os.getpid()
173
queue.put(f"Worker from {context_name} context: PID {pid}")
174
175
def multiple_contexts_example():
176
"""Demonstrate using multiple contexts"""
177
# Get different contexts
178
fork_ctx = mp.get_context('fork') if 'fork' in mp.get_all_start_methods() else None
179
spawn_ctx = mp.get_context('spawn')
180
181
processes = []
182
results_queue = mp.Queue()
183
184
# Create processes with different contexts
185
if fork_ctx:
186
p1 = fork_ctx.Process(target=context_worker,
187
args=('fork', results_queue))
188
processes.append(p1)
189
190
p2 = spawn_ctx.Process(target=context_worker,
191
args=('spawn', results_queue))
192
processes.append(p2)
193
194
# Start all processes
195
for p in processes:
196
p.start()
197
198
# Collect results
199
for _ in processes:
200
result = results_queue.get()
201
print(result)
202
203
# Wait for completion
204
for p in processes:
205
p.join()
206
207
def context_isolation_example():
208
"""Demonstrate context isolation"""
209
# Create separate contexts
210
ctx1 = mp.get_context('spawn')
211
ctx2 = mp.get_context('spawn')
212
213
# Each context has its own objects
214
queue1 = ctx1.Queue()
215
queue2 = ctx2.Queue()
216
217
def isolated_worker(queue, context_id):
218
queue.put(f"Message from context {context_id}")
219
220
# Create processes in different contexts
221
p1 = ctx1.Process(target=isolated_worker, args=(queue1, 1))
222
p2 = ctx2.Process(target=isolated_worker, args=(queue2, 2))
223
224
p1.start()
225
p2.start()
226
227
print("Context 1:", queue1.get())
228
print("Context 2:", queue2.get())
229
230
p1.join()
231
p2.join()
232
233
if __name__ == '__main__':
234
multiple_contexts_example()
235
context_isolation_example()
236
```
237
238
### Start Methods
239
240
Different process creation methods with distinct characteristics.
241
242
#### Fork Method
243
244
```python
245
# Only available on Unix-like systems
246
ctx = mp.get_context('fork')
247
```
248
249
**Characteristics:**
250
- **Fast startup**: Child inherits parent's memory space
251
- **Shared resources**: All imported modules and objects are available
252
- **Threading issues**: Not safe with threads (can cause deadlocks)
253
- **Memory efficient**: Copy-on-write memory sharing
254
- **Unix only**: Not available on Windows
255
256
#### Spawn Method
257
258
```python
259
# Available on all platforms
260
ctx = mp.get_context('spawn')
261
```
262
263
**Characteristics:**
264
- **Clean startup**: Fresh Python interpreter for each process
265
- **Thread safe**: No threading issues
266
- **Slower startup**: Must import modules and initialize objects
267
- **Cross-platform**: Works on Windows, macOS, and Linux
268
- **Isolated**: Each process has separate memory space
269
270
#### Forkserver Method
271
272
```python
273
# Available on Unix with proper setup
274
ctx = mp.get_context('forkserver')
275
```
276
277
**Characteristics:**
278
- **Server-based**: Uses dedicated fork server process
279
- **Thread safe**: Avoids threading issues of fork
280
- **Resource efficient**: Server pre-imports common modules
281
- **Security**: Isolated from parent process state
282
- **Unix only**: Not available on Windows
283
284
Usage example:
285
286
```python
287
import billiard as mp
288
import os
289
import threading
290
import time
291
292
def threaded_parent_worker():
293
"""Worker function that creates threads"""
294
def thread_worker(thread_id):
295
time.sleep(0.1)
296
print(f"Thread {thread_id} completed")
297
298
# Create threads in parent
299
threads = []
300
for i in range(3):
301
t = threading.Thread(target=thread_worker, args=(i,))
302
threads.append(t)
303
t.start()
304
305
for t in threads:
306
t.join()
307
308
print("All threads completed in parent")
309
310
def child_process_work(method_name):
311
"""Work done in child process"""
312
print(f"Child process ({method_name}): PID {os.getpid()}, "
313
f"PPID {os.getppid()}")
314
print(f"Active threads in child: {threading.active_count()}")
315
316
def start_method_comparison():
317
"""Compare different start methods"""
318
# Start some threads in parent process
319
parent_thread = threading.Thread(target=threaded_parent_worker)
320
parent_thread.start()
321
322
time.sleep(0.05) # Let threads start
323
324
available_methods = mp.get_all_start_methods()
325
print(f"Available methods: {available_methods}")
326
print(f"Active threads in parent: {threading.active_count()}")
327
328
# Test each available method
329
for method in available_methods:
330
print(f"\n--- Testing {method} method ---")
331
try:
332
ctx = mp.get_context(method)
333
process = ctx.Process(target=child_process_work, args=(method,))
334
process.start()
335
process.join()
336
except Exception as e:
337
print(f"Error with {method}: {e}")
338
339
# Wait for parent threads
340
parent_thread.join()
341
342
if __name__ == '__main__':
343
start_method_comparison()
344
```
345
346
### Context Best Practices
347
348
#### Application Structure
349
350
```python
351
import billiard as mp
352
353
def main():
354
"""Main application entry point"""
355
# Set start method early in program
356
mp.set_start_method('spawn') # or 'fork', 'forkserver'
357
358
# Create context for specific needs
359
ctx = mp.get_context()
360
361
# Use context consistently
362
queue = ctx.Queue()
363
processes = []
364
365
for i in range(4):
366
p = ctx.Process(target=worker, args=(queue, i))
367
processes.append(p)
368
p.start()
369
370
for p in processes:
371
p.join()
372
373
def worker(queue, worker_id):
374
"""Worker function"""
375
queue.put(f"Result from worker {worker_id}")
376
377
if __name__ == '__main__':
378
main()
379
```
380
381
#### Context Selection Guidelines
382
383
```python
384
import billiard as mp
385
import sys
386
import platform
387
388
def select_optimal_context():
389
"""Select optimal context based on platform and requirements"""
390
391
if sys.platform == 'win32':
392
# Windows only supports spawn
393
return mp.get_context('spawn')
394
395
elif 'darwin' in sys.platform:
396
# macOS: spawn is often safer due to framework issues
397
return mp.get_context('spawn')
398
399
else:
400
# Linux/Unix: more options available
401
if threading.active_count() > 1:
402
# Threads present: avoid fork
403
if 'forkserver' in mp.get_all_start_methods():
404
return mp.get_context('forkserver')
405
else:
406
return mp.get_context('spawn')
407
else:
408
# No threading issues: fork is fastest
409
return mp.get_context('fork')
410
411
def context_factory_pattern():
412
"""Factory pattern for context creation"""
413
414
class ProcessContextFactory:
415
@staticmethod
416
def create_high_performance_context():
417
"""Context optimized for performance"""
418
if 'fork' in mp.get_all_start_methods():
419
return mp.get_context('fork')
420
return mp.get_context('spawn')
421
422
@staticmethod
423
def create_safe_context():
424
"""Context optimized for safety"""
425
if 'forkserver' in mp.get_all_start_methods():
426
return mp.get_context('forkserver')
427
return mp.get_context('spawn')
428
429
@staticmethod
430
def create_portable_context():
431
"""Context for maximum portability"""
432
return mp.get_context('spawn')
433
434
# Usage
435
factory = ProcessContextFactory()
436
437
if platform.system() == 'Windows':
438
ctx = factory.create_portable_context()
439
elif performance_critical:
440
ctx = factory.create_high_performance_context()
441
else:
442
ctx = factory.create_safe_context()
443
444
return ctx
445
446
# Example usage
447
performance_critical = True
448
optimal_ctx = select_optimal_context()
449
print(f"Selected context with start method: {optimal_ctx._name}")
450
```
451
452
## Platform Considerations
453
454
### Windows
455
- Only `spawn` method available
456
- All imports must be guarded with `if __name__ == '__main__':`
457
- Slower process creation due to full interpreter startup
458
459
### macOS
460
- All methods available but `spawn` often preferred
461
- Framework compatibility issues with `fork`
462
- Consider `forkserver` for mixed threading scenarios
463
464
### Linux
465
- All methods available
466
- `fork` fastest for CPU-intensive tasks
467
- `spawn` safest for mixed workloads
468
- `forkserver` good compromise for long-running applications
469
470
## Context Configuration Examples
471
472
### High-Performance Computing
473
474
```python
475
import billiard as mp
476
477
# Configure for HPC workload
478
if 'fork' in mp.get_all_start_methods():
479
mp.set_start_method('fork') # Fastest startup
480
else:
481
mp.set_start_method('spawn')
482
483
ctx = mp.get_context()
484
485
# Use large pools for parallel computation
486
with ctx.Pool(processes=mp.cpu_count()) as pool:
487
results = pool.map(compute_intensive_function, data)
488
```
489
490
### Web Service Backend
491
492
```python
493
import billiard as mp
494
495
# Configure for web service (thread-safe)
496
mp.set_start_method('spawn') # Safe with web frameworks
497
ctx = mp.get_context()
498
499
# Create worker pool for request processing
500
request_pool = ctx.Pool(processes=4, maxtasksperchild=100)
501
```
502
503
### Distributed System
504
505
```python
506
import billiard as mp
507
508
# Configure for distributed processing
509
if 'forkserver' in mp.get_all_start_methods():
510
mp.set_start_method('forkserver') # Isolated and efficient
511
else:
512
mp.set_start_method('spawn')
513
514
ctx = mp.get_context()
515
manager = ctx.Manager() # For distributed state
516
```