0
# Bulk Operations Service
1

2
High-level service for bulk operations with multi-threading, progress reporting, automatic retries, and comprehensive error handling for upload, download, delete, and copy operations.
3

4
## Capabilities
5

6
### SwiftService Class
7

8
Main service class providing high-level bulk operations with automatic threading, retries, and progress tracking.
9

10
```python { .api }
11
class SwiftService:
12
    def __init__(self, options=None):
13
        """
14
        Initialize Swift service with configuration options.
15

16
        Parameters:
17
        - options: dict, service configuration options
18

19
        Service options include:
20
        - auth_url: str, authentication URL
21
        - username: str, username for authentication
22
        - password: str, password for authentication
23
        - auth_version: str, authentication version
24
        - os_*: OpenStack identity options
25
        - retries: int, number of operation retries
26
        - object_dd_threads: int, object download/delete thread count
27
        - object_uu_threads: int, object upload/update thread count
28
        - container_threads: int, container operation thread count
29
        - segment_threads: int, segment operation thread count
30
        """
31

32
    def upload(self, container, objects, options=None):
33
        """
34
        Upload objects to Swift.
35

36
        Parameters:
37
        - container: str, container name to upload to
38
        - objects: iterable, SwiftUploadObject instances or file paths
39
        - options: dict, upload-specific options
40

41
        Yields:
42
        dict: upload result for each object
43

44
        Upload options include:
45
        - leave_segments: bool, don't delete old segments on update
46
        - changed: bool, only upload changed files
47
        - skip_identical: bool, skip files with matching ETags
48
        - fail_fast: bool, stop on first error
49
        - dir_marker: bool, create directory marker objects
50
        """
51

52
    def download(self, container=None, objects=None, options=None):
53
        """
54
        Download objects from Swift.
55

56
        Parameters:
57
        - container: str, container name to download from (None for all containers)
58
        - objects: iterable, object names to download (None for all objects)
59
        - options: dict, download-specific options
60

61
        Yields:
62
        dict: download result for each object
63

64
        Download options include:
65
        - out_directory: str, local directory to download to
66
        - prefix: str, only download objects with this prefix
67
        - remove_prefix: bool, remove prefix from local file names
68
        - no_download: bool, print download URLs without downloading
69
        - skip_identical: bool, skip files with matching local copies
70
        """
71

72
    def delete(self, container=None, objects=None, options=None):
73
        """
74
        Delete objects and containers from Swift.
75

76
        Parameters:
77
        - container: str, container name to delete from (None for account)
78
        - objects: iterable, object names to delete (None for all objects)
79
        - options: dict, delete-specific options
80

81
        Yields:
82
        dict: delete result for each object/container
83

84
        Delete options include:
85
        - prefix: str, only delete objects with this prefix
86
        - yes_all: bool, delete all containers and objects
87
        - leave_segments: bool, don't delete segments of manifest objects
88
        """
89

90
    def copy(self, container=None, objects=None, options=None):
91
        """
92
        Copy objects in Swift.
93

94
        Parameters:
95
        - container: str, source container name
96
        - objects: iterable, SwiftCopyObject instances
97
        - options: dict, copy-specific options
98

99
        Yields:
100
        dict: copy result for each object
101

102
        Copy options include:
103
        - destination: str, destination container
104
        - fresh_metadata: bool, don't copy existing metadata
105
        """
106

107
    def post(self, container=None, objects=None, options=None):
108
        """
109
        Update metadata for containers and objects.
110

111
        Parameters:
112
        - container: str, container name to update (None for account)
113
        - objects: iterable, SwiftPostObject instances
114
        - options: dict, post-specific options
115

116
        Yields:
117
        dict: post result for each container/object
118
        """
119

120
    def list(self, container=None, options=None):
121
        """
122
        List containers and objects.
123

124
        Parameters:
125
        - container: str, container name to list (None for containers)
126
        - options: dict, list-specific options
127

128
        Yields:
129
        dict: listing result for each container/object
130

131
        List options include:
132
        - long: bool, include detailed information
133
        - prefix: str, only list items with this prefix
134
        - delimiter: str, delimiter for hierarchical listings
135
        """
136

137
    def stat(self, container=None, objects=None, options=None):
138
        """
139
        Display account, container, and object statistics.
140

141
        Parameters:
142
        - container: str, container name to stat (None for account)
143
        - objects: iterable, object names to stat (None for container)
144
        - options: dict, stat-specific options
145

146
        Yields:
147
        dict: stat result for each item
148
        """
149

150
    def get_capabilities(self, url=None):
151
        """
152
        Retrieve Swift cluster capabilities.
153

154
        Parameters:
155
        - url: str, Swift cluster URL (uses auth URL if None)
156

157
        Returns:
158
        dict: cluster capabilities
159
        """
160
```
161

162
### Operation Object Classes
163

164
Configuration objects for specifying bulk operation parameters.
165

166
```python { .api }
167
class SwiftUploadObject:
168
    def __init__(
169
        self,
170
        source,
171
        object_name=None,
172
        options=None
173
    ):
174
        """
175
        Specify object for upload operation.
176

177
        Parameters:
178
        - source: str, local file path or file-like object
179
        - object_name: str, object name in Swift (derived from source if None)
180
        - options: dict, per-object upload options
181
        """
182

183
class SwiftPostObject:
184
    def __init__(
185
        self,
186
        object_name,
187
        options=None
188
    ):
189
        """
190
        Specify object for metadata update operation.
191

192
        Parameters:
193
        - object_name: str, object name to update
194
        - options: dict, per-object options including headers
195
        """
196

197
class SwiftDeleteObject:
198
    def __init__(
199
        self,
200
        object_name,
201
        options=None
202
    ):
203
        """
204
        Specify object for delete operation.
205

206
        Parameters:
207
        - object_name: str, object name to delete
208
        - options: dict, per-object delete options
209
        """
210

211
class SwiftCopyObject:
212
    def __init__(
213
        self,
214
        object_name,
215
        source_container,
216
        destination_container,
217
        destination_object=None,
218
        options=None
219
    ):
220
        """
221
        Specify object for copy operation.
222

223
        Parameters:
224
        - object_name: str, source object name
225
        - source_container: str, source container name
226
        - destination_container: str, destination container name
227
        - destination_object: str, destination object name (same as source if None)
228
        - options: dict, per-object copy options
229
        """
230
```
231

232
### Error Classes
233

234
```python { .api }
235
class SwiftError(Exception):
236
    def __init__(
237
        self,
238
        value,
239
        container=None,
240
        obj=None,
241
        segment=None,
242
        exc=None,
243
        transaction_id=None
244
    ):
245
        """
246
        Service-level error with operation context.
247

248
        Parameters:
249
        - value: str, error description
250
        - container: str, container name involved in error
251
        - obj: str, object name involved in error
252
        - segment: str, segment name for large object operations
253
        - exc: Exception, underlying exception
254
        - transaction_id: str, Swift transaction ID
255
        """
256
```
257

258
### Service Configuration Functions
259

260
```python { .api }
261
def get_conn(options):
262
    """
263
    Create Connection object from service options.
264

265
    Parameters:
266
    - options: dict, connection options
267

268
    Returns:
269
    Connection: configured connection object
270
    """
271

272
def process_options(options):
273
    """
274
    Process and normalize authentication and service options.
275

276
    Parameters:
277
    - options: dict, raw options dictionary
278

279
    Returns:
280
    dict: processed options with normalized authentication settings
281
    """
282
```
283

284
## Usage Examples
285

286
### Basic Upload Operations
287

288
```python
289
from swiftclient.service import SwiftService, SwiftUploadObject
290

291
# Configure service
292
options = {
293
    'auth_url': 'https://identity.example.com:5000/v3',
294
    'username': 'myuser',
295
    'password': 'mypassword',
296
    'auth_version': '3',
297
    'os_project_name': 'myproject',
298
    'os_user_domain_name': 'mydomain',
299
    'os_project_domain_name': 'mydomain',
300
}
301

302
with SwiftService(options=options) as swift:
303
    # Upload individual files
304
    upload_objects = [
305
        SwiftUploadObject('local/file1.txt'),
306
        SwiftUploadObject('local/file2.txt', object_name='remote/file2.txt'),
307
        SwiftUploadObject('local/image.jpg', options={'content-type': 'image/jpeg'}),
308
    ]
309

310
    for result in swift.upload('documents', upload_objects):
311
        if result['success']:
312
            print(f"Uploaded: {result['object']}")
313
        else:
314
            print(f"Failed to upload {result['object']}: {result['error']}")
315

316
    # Upload directory recursively
317
    for result in swift.upload('backup', ['./my_directory']):
318
        if result['success']:
319
            print(f"Uploaded: {result['object']}")
320
```
321

322
### Bulk Download Operations
323

324
```python
325
# Download entire container
326
download_options = {
327
    'out_directory': './downloads',
328
    'skip_identical': True,
329
}
330

331
with SwiftService(options=options) as swift:
332
    for result in swift.download('documents', options=download_options):
333
        if result['success']:
334
            print(f"Downloaded: {result['object']} -> {result['path']}")
335
        else:
336
            print(f"Failed to download {result['object']}: {result['error']}")
337

338
    # Download specific objects
339
    object_names = ['file1.txt', 'images/photo1.jpg', 'data/dataset.csv']
340
    for result in swift.download('documents', object_names, options=download_options):
341
        if result['success']:
342
            print(f"Downloaded: {result['object']}")
343
```
344

345
### Bulk Delete Operations
346

347
```python
348
# Delete specific objects
349
delete_objects = ['old_file1.txt', 'temp/old_file2.txt', 'cache/temp.dat']
350

351
with SwiftService(options=options) as swift:
352
    for result in swift.delete('documents', delete_objects):
353
        if result['success']:
354
            print(f"Deleted: {result['object']}")
355
        else:
356
            print(f"Failed to delete {result['object']}: {result['error']}")
357

358
    # Delete all objects with prefix
359
    delete_options = {'prefix': 'temp/'}
360
    for result in swift.delete('documents', options=delete_options):
361
        if result['success']:
362
            print(f"Deleted: {result['object']}")
363
```
364

365
### Copy Operations
366

367
```python
368
from swiftclient.service import SwiftCopyObject
369

370
copy_objects = [
371
    SwiftCopyObject('file1.txt', 'documents', 'backup'),
372
    SwiftCopyObject('data.csv', 'documents', 'archive', 'data-2024.csv'),
373
]
374

375
with SwiftService(options=options) as swift:
376
    for result in swift.copy('documents', copy_objects):
377
        if result['success']:
378
            print(f"Copied: {result['object']} -> {result['destination']}")
379
        else:
380
            print(f"Failed to copy {result['object']}: {result['error']}")
381
```
382

383
### Metadata Updates
384

385
```python
386
from swiftclient.service import SwiftPostObject
387

388
# Update object metadata
389
post_objects = [
390
    SwiftPostObject('file1.txt', options={
391
        'header': ['X-Object-Meta-Author:John Doe', 'X-Object-Meta-Version:2.0']
392
    }),
393
    SwiftPostObject('file2.txt', options={
394
        'header': ['X-Object-Meta-Department:Engineering']
395
    }),
396
]
397

398
with SwiftService(options=options) as swift:
399
    for result in swift.post('documents', post_objects):
400
        if result['success']:
401
            print(f"Updated metadata: {result['object']}")
402
        else:
403
            print(f"Failed to update {result['object']}: {result['error']}")
404

405
    # Update container metadata
406
    container_options = {
407
        'header': ['X-Container-Meta-Owner:TeamA', 'X-Container-Read:.r:*']
408
    }
409
    for result in swift.post('documents', options=container_options):
410
        if result['success']:
411
            print(f"Updated container: {result['container']}")
412
```
413

414
### Listing and Statistics
415

416
```python
417
# List containers with details
418
with SwiftService(options=options) as swift:
419
    for result in swift.list(options={'long': True}):
420
        if result['success']:
421
            if 'container' in result:
422
                container = result['container']
423
                print(f"Container: {container['name']}")
424
                print(f"  Count: {container['count']}, Bytes: {container['bytes']}")
425

426
    # List objects in container
427
    for result in swift.list('documents', options={'long': True, 'prefix': 'images/'}):
428
        if result['success'] and 'object' in result:
429
            obj = result['object']
430
            print(f"Object: {obj['name']}, Size: {obj['bytes']}, ETag: {obj['hash']}")
431

432
    # Get statistics
433
    for result in swift.stat('documents'):
434
        if result['success']:
435
            stats = result['items']
436
            for key, value in stats:
437
                print(f"{key}: {value}")
438
```
439

440
### Advanced Upload Configuration
441

442
```python
443
# Upload with advanced options
444
upload_options = {
445
    'segment_size': 1048576 * 100,  # 100MB segments for large files
446
    'use_slo': True,  # Use Static Large Objects
447
    'segment_container': 'documents_segments',
448
    'leave_segments': False,  # Clean up old segments
449
    'changed': True,  # Only upload changed files
450
    'skip_identical': True,  # Skip identical files
451
    'fail_fast': False,  # Continue on errors
452
    'object_uu_threads': 10,  # Upload thread count
453
}
454

455
large_files = [
456
    SwiftUploadObject('large_dataset.csv', options={'content-type': 'text/csv'}),
457
    SwiftUploadObject('video.mp4', options={'content-type': 'video/mp4'}),
458
]
459

460
with SwiftService(options={**options, **upload_options}) as swift:
461
    for result in swift.upload('documents', large_files):
462
        if result['success']:
463
            print(f"Uploaded: {result['object']} ({result['bytes_uploaded']} bytes)")
464
            if 'manifest' in result:
465
                print(f"  Created manifest with {len(result['manifest'])} segments")
466
        else:
467
            print(f"Failed: {result['object']} - {result['error']}")
468
```
469

470
### Progress Monitoring
471

472
```python
473
import time
474

475
def monitor_operations(swift_generator, operation_name):
476
    """Monitor and report progress of bulk operations."""
477
    start_time = time.time()
478
    success_count = 0
479
    error_count = 0
480
    total_bytes = 0
481

482
    for result in swift_generator:
483
        if result['success']:
484
            success_count += 1
485
            total_bytes += result.get('bytes_uploaded', result.get('bytes_downloaded', 0))
486
        else:
487
            error_count += 1
488
            print(f"ERROR: {result.get('object', result.get('container', 'unknown'))}: {result['error']}")
489

490
        if (success_count + error_count) % 10 == 0:
491
            elapsed = time.time() - start_time
492
            print(f"{operation_name}: {success_count} success, {error_count} errors, "
493
                  f"{total_bytes} bytes in {elapsed:.1f}s")
494

495
    total_time = time.time() - start_time
496
    print(f"\n{operation_name} Complete:")
497
    print(f"  Successful: {success_count}")
498
    print(f"  Failed: {error_count}")
499
    print(f"  Total bytes: {total_bytes}")
500
    print(f"  Total time: {total_time:.1f}s")
501

502
# Usage
503
with SwiftService(options=options) as swift:
504
    upload_generator = swift.upload('documents', ['./large_directory'])
505
    monitor_operations(upload_generator, "Upload")
506
```