0
# Advanced Operations
1
2
Advanced S3 features including presigned URLs for secure temporary access, SQL SELECT queries for object content filtering, bucket notifications for event-driven workflows, and sophisticated object composition operations.
3
4
## Capabilities
5
6
### Presigned URLs
7
8
Generate temporary URLs for secure access to objects without exposing credentials. Useful for web applications, direct uploads, and temporary sharing.
9
10
```python { .api }
11
def presigned_get_object(
12
self,
13
bucket_name: str,
14
object_name: str,
15
expires: datetime.timedelta = datetime.timedelta(days=7),
16
response_headers: dict[str, str] | None = None,
17
request_date: datetime.datetime | None = None,
18
version_id: str | None = None,
19
extra_query_params: dict[str, str] | None = None
20
) -> str:
21
"""
22
Generate a presigned URL for GET operations.
23
24
Args:
25
bucket_name: Name of the bucket
26
object_name: Name of the object
27
expires: Expiration time for the URL (max 7 days)
28
response_headers: Headers to override in response
29
request_date: Date for the request (defaults to current time)
30
version_id: Specific version to access
31
extra_query_params: Additional query parameters
32
33
Returns:
34
Presigned URL string
35
36
Raises:
37
S3Error: If URL generation fails
38
"""
39
40
def presigned_put_object(
41
self,
42
bucket_name: str,
43
object_name: str,
44
expires: datetime.timedelta = datetime.timedelta(days=7),
45
response_headers: dict[str, str] | None = None,
46
request_date: datetime.datetime | None = None
47
) -> str:
48
"""
49
Generate a presigned URL for PUT operations.
50
51
Args:
52
bucket_name: Name of the bucket
53
object_name: Name of the object
54
expires: Expiration time for the URL (max 7 days)
55
response_headers: Headers to override in response
56
request_date: Date for the request (defaults to current time)
57
58
Returns:
59
Presigned URL string
60
61
Raises:
62
S3Error: If URL generation fails
63
"""
64
65
def presigned_post_policy(self, policy: PostPolicy) -> dict[str, str]:
66
"""
67
Generate presigned POST policy for browser-based uploads.
68
69
Args:
70
policy: PostPolicy object with upload constraints
71
72
Returns:
73
Dictionary containing form fields for HTML form
74
75
Raises:
76
S3Error: If policy generation fails
77
"""
78
```
79
80
### SQL Select Operations
81
82
Query object content using SQL expressions without downloading entire objects. Supports CSV, JSON, and Parquet formats.
83
84
```python { .api }
85
def select_object_content(
86
self,
87
bucket_name: str,
88
object_name: str,
89
request: SelectRequest
90
) -> SelectObjectReader:
91
"""
92
Select object content using SQL expressions.
93
94
Args:
95
bucket_name: Name of the bucket
96
object_name: Name of the object to query
97
request: SelectRequest with SQL expression and format specifications
98
99
Returns:
100
SelectObjectReader for streaming results
101
102
Raises:
103
S3Error: If select operation fails
104
"""
105
```
106
107
### Bucket Notifications
108
109
Listen for real-time bucket events such as object creation, deletion, and modifications. Essential for event-driven architectures.
110
111
```python { .api }
112
def listen_bucket_notification(
113
self,
114
bucket_name: str,
115
prefix: str = "",
116
suffix: str = "",
117
events: list[str] = ["s3:ObjectCreated:*", "s3:ObjectRemoved:*"]
118
) -> EventIterable:
119
"""
120
Listen for bucket notifications in real-time.
121
122
Args:
123
bucket_name: Name of the bucket to monitor
124
prefix: Filter events by object name prefix
125
suffix: Filter events by object name suffix
126
events: List of S3 event types to monitor
127
128
Returns:
129
EventIterable for streaming notification events
130
131
Raises:
132
S3Error: If notification setup fails
133
"""
134
135
def set_bucket_notification(
136
self,
137
bucket_name: str,
138
config: NotificationConfig
139
) -> None:
140
"""
141
Set bucket notification configuration.
142
143
Args:
144
bucket_name: Name of the bucket
145
config: NotificationConfig with destination settings
146
147
Raises:
148
S3Error: If configuration fails
149
"""
150
151
def get_bucket_notification(self, bucket_name: str) -> NotificationConfig:
152
"""
153
Get current bucket notification configuration.
154
155
Args:
156
bucket_name: Name of the bucket
157
158
Returns:
159
NotificationConfig with current settings
160
161
Raises:
162
S3Error: If retrieval fails
163
"""
164
165
def delete_bucket_notification(self, bucket_name: str) -> None:
166
"""
167
Remove bucket notification configuration.
168
169
Args:
170
bucket_name: Name of the bucket
171
172
Raises:
173
S3Error: If removal fails
174
"""
175
```
176
177
### Client Configuration
178
179
Configure client behavior including endpoints, tracing, and advanced features.
180
181
```python { .api }
182
def set_app_info(self, app_name: str, app_version: str) -> None:
183
"""
184
Set application information for user agent string.
185
186
Args:
187
app_name: Name of the application
188
app_version: Version of the application
189
"""
190
191
def trace_on(self, stream: TextIO) -> None:
192
"""
193
Enable HTTP request/response tracing.
194
195
Args:
196
stream: Text stream to write trace output to
197
"""
198
199
def trace_off(self) -> None:
200
"""
201
Disable HTTP request/response tracing.
202
"""
203
204
def enable_accelerate_endpoint(self) -> None:
205
"""
206
Enable S3 transfer acceleration endpoint.
207
"""
208
209
def disable_accelerate_endpoint(self) -> None:
210
"""
211
Disable S3 transfer acceleration endpoint.
212
"""
213
214
def enable_dualstack_endpoint(self) -> None:
215
"""
216
Enable dual-stack (IPv4/IPv6) endpoints.
217
"""
218
219
def disable_dualstack_endpoint(self) -> None:
220
"""
221
Disable dual-stack endpoints.
222
"""
223
224
def enable_virtual_style_endpoint(self) -> None:
225
"""
226
Enable virtual-style endpoint addressing.
227
"""
228
229
def disable_virtual_style_endpoint(self) -> None:
230
"""
231
Disable virtual-style endpoint addressing.
232
"""
233
```
234
235
### Batch Operations
236
237
Efficient batch operations for handling multiple objects simultaneously.
238
239
```python { .api }
240
def upload_snowball_objects(
241
self,
242
bucket_name: str,
243
object_list: Iterable[SnowballObject],
244
metadata: dict[str, str] | None = None,
245
sse: Sse | None = None,
246
tags: Tags | None = None,
247
retention: Retention | None = None,
248
legal_hold: bool = False,
249
staging_filename: str | None = None,
250
compression: bool = True
251
) -> ObjectWriteResult:
252
"""
253
Upload multiple objects in a single compressed archive.
254
255
Args:
256
bucket_name: Name of the bucket
257
object_list: Iterable of SnowballObject specifications
258
metadata: Common metadata for all objects
259
sse: Server-side encryption
260
tags: Common tags for all objects
261
retention: Retention configuration
262
legal_hold: Enable legal hold
263
staging_filename: Temporary file for staging
264
compression: Enable compression of archive
265
266
Returns:
267
ObjectWriteResult for the uploaded archive
268
269
Raises:
270
S3Error: If batch upload fails
271
"""
272
```
273
274
## Types
275
276
### Presigned URL Types
277
278
```python { .api }
279
class PostPolicy:
280
"""Policy configuration for presigned POST uploads."""
281
282
def set_bucket_name(self, bucket_name: str) -> None:
283
"""Set the bucket name for uploads."""
284
285
def set_bucket_name_starts_with(self, bucket_name_starts_with: str) -> None:
286
"""Set bucket name prefix constraint."""
287
288
def set_content_type(self, content_type: str) -> None:
289
"""Set allowed content type."""
290
291
def set_content_length_range(self, lower_limit: int, upper_limit: int) -> None:
292
"""Set file size constraints."""
293
294
def set_expires(self, expires: datetime.datetime) -> None:
295
"""Set policy expiration time."""
296
297
def set_key(self, object_name: str) -> None:
298
"""Set exact object name for uploads."""
299
300
def set_key_starts_with(self, key_starts_with: str) -> None:
301
"""Set object name prefix constraint."""
302
```
303
304
### Select Query Types
305
306
```python { .api }
307
class SelectRequest:
308
"""SQL select request configuration."""
309
def __init__(
310
self,
311
expression: str,
312
input_serialization: InputSerialization,
313
output_serialization: OutputSerialization,
314
request_progress: bool = False
315
) -> None: ...
316
expression: str
317
input_serialization: InputSerialization
318
output_serialization: OutputSerialization
319
request_progress: bool
320
321
class SelectObjectReader:
322
"""Reader for streaming select query results."""
323
324
def stream(self) -> Iterator[bytes]:
325
"""Stream selected records as bytes."""
326
327
def stats(self) -> dict[str, Any]:
328
"""Get selection statistics."""
329
330
class CSVInputSerialization:
331
"""CSV input format specification."""
332
def __init__(
333
self,
334
file_header_info: str = "USE",
335
record_delimiter: str = "\n",
336
field_delimiter: str = ",",
337
quote_character: str = '"',
338
quote_escape_character: str = '"',
339
comments: str = "#",
340
allow_quoted_record_delimiter: bool = False
341
) -> None: ...
342
343
class JSONInputSerialization:
344
"""JSON input format specification."""
345
def __init__(self, json_type: str = "DOCUMENT") -> None: ...
346
347
class ParquetInputSerialization:
348
"""Parquet input format specification."""
349
def __init__(self) -> None: ...
350
351
class CSVOutputSerialization:
352
"""CSV output format specification."""
353
def __init__(
354
self,
355
record_delimiter: str = "\n",
356
field_delimiter: str = ",",
357
quote_character: str = '"',
358
quote_escape_character: str = '"',
359
quote_fields: str = "ASNEEDED"
360
) -> None: ...
361
362
class JSONOutputSerialization:
363
"""JSON output format specification."""
364
def __init__(self, record_delimiter: str = "\n") -> None: ...
365
```
366
367
### Notification Types
368
369
```python { .api }
370
class EventIterable:
371
"""Iterator for streaming bucket notification events."""
372
373
def __iter__(self) -> Iterator[dict[str, Any]]:
374
"""Iterate over notification events."""
375
376
class NotificationConfig:
377
"""Bucket notification configuration."""
378
def __init__(
379
self,
380
cloud_func_config_list: list[CloudFunctionConfig] | None = None,
381
queue_config_list: list[QueueConfig] | None = None,
382
topic_config_list: list[TopicConfig] | None = None
383
) -> None: ...
384
```
385
386
### Batch Upload Types
387
388
```python { .api }
389
class SnowballObject:
390
"""Specification for batch upload objects."""
391
def __init__(
392
self,
393
object_name: str,
394
filename: str | None = None,
395
data: io.IOBase | None = None,
396
length: int = -1,
397
metadata: dict[str, str] | None = None
398
) -> None: ...
399
object_name: str
400
filename: str | None
401
data: io.IOBase | None
402
length: int
403
metadata: dict[str, str] | None
404
```
405
406
## Usage Examples
407
408
### Presigned URLs
409
410
```python
411
import datetime
412
from minio import Minio
413
from minio.datatypes import PostPolicy
414
415
client = Minio("localhost:9000", "minio", "minio123")
416
417
# Generate presigned GET URL (valid for 1 hour)
418
url = client.presigned_get_object(
419
"my-bucket",
420
"my-object.jpg",
421
expires=datetime.timedelta(hours=1)
422
)
423
print(f"Download URL: {url}")
424
425
# Generate presigned PUT URL for uploads
426
upload_url = client.presigned_put_object(
427
"my-bucket",
428
"uploads/new-file.txt",
429
expires=datetime.timedelta(minutes=30)
430
)
431
print(f"Upload URL: {upload_url}")
432
433
# Generate POST policy for browser uploads
434
policy = PostPolicy()
435
policy.set_bucket_name("my-bucket")
436
policy.set_key_starts_with("uploads/")
437
policy.set_content_length_range(1024, 10*1024*1024) # 1KB to 10MB
438
policy.set_expires(datetime.datetime.utcnow() + datetime.timedelta(hours=1))
439
440
form_data = client.presigned_post_policy(policy)
441
print("Form fields for HTML upload:")
442
for key, value in form_data.items():
443
print(f"{key}: {value}")
444
```
445
446
### SQL Select Queries
447
448
```python
449
from minio.select import (
450
SelectRequest,
451
CSVInputSerialization,
452
CSVOutputSerialization
453
)
454
455
# Query CSV data with SQL
456
input_serialization = CSVInputSerialization(
457
file_header_info="USE",
458
record_delimiter="\n",
459
field_delimiter=","
460
)
461
462
output_serialization = CSVOutputSerialization(
463
record_delimiter="\n",
464
field_delimiter=","
465
)
466
467
request = SelectRequest(
468
expression="SELECT name, age FROM S3Object WHERE age > 25",
469
input_serialization=input_serialization,
470
output_serialization=output_serialization,
471
request_progress=True
472
)
473
474
try:
475
reader = client.select_object_content("data-bucket", "people.csv", request)
476
477
# Stream results
478
for data in reader.stream():
479
print(data.decode('utf-8'), end='')
480
481
# Get statistics
482
stats = reader.stats()
483
print(f"Bytes scanned: {stats.get('bytes_scanned', 0)}")
484
print(f"Bytes processed: {stats.get('bytes_processed', 0)}")
485
486
except Exception as e:
487
print(f"Select query failed: {e}")
488
```
489
490
### Bucket Notifications
491
492
```python
493
# Listen for bucket events
494
try:
495
events = client.listen_bucket_notification(
496
"my-bucket",
497
prefix="uploads/",
498
events=["s3:ObjectCreated:*", "s3:ObjectRemoved:*"]
499
)
500
501
for event in events:
502
for record in event.get("Records", []):
503
event_name = record.get("eventName")
504
bucket = record["s3"]["bucket"]["name"]
505
obj_key = record["s3"]["object"]["key"]
506
print(f"Event: {event_name} - {bucket}/{obj_key}")
507
508
except KeyboardInterrupt:
509
print("Stopped listening for events")
510
except Exception as e:
511
print(f"Notification error: {e}")
512
```
513
514
### Batch Uploads
515
516
```python
517
from minio.commonconfig import SnowballObject
518
519
# Batch upload multiple small objects efficiently
520
objects_to_upload = [
521
SnowballObject("file1.txt", "/path/to/file1.txt"),
522
SnowballObject("file2.txt", "/path/to/file2.txt"),
523
SnowballObject("data/file3.txt", "/path/to/file3.txt"),
524
]
525
526
try:
527
result = client.upload_snowball_objects(
528
"my-bucket",
529
objects_to_upload,
530
compression=True
531
)
532
print(f"Batch upload completed: {result.etag}")
533
534
except Exception as e:
535
print(f"Batch upload failed: {e}")
536
```