0
# I/O Connections
1
2
Advanced I/O system supporting custom sources and targets for streaming operations, enabling integration with various data sources and destinations beyond the filesystem. This system provides flexibility for network streams, databases, cloud storage, and custom protocols.
3
4
## Capabilities
5
6
### Source Objects
7
8
Input connections for reading image data from various sources with streaming support.
9
10
```python { .api }
11
class Source:
12
"""Input connection for streaming image data."""
13
14
@classmethod
15
def new_from_file(cls, filename: str) -> 'Source':
16
"""
17
Create source from file.
18
19
Parameters:
20
- filename: str, path to input file
21
22
Returns:
23
Source object for the file
24
"""
25
26
@classmethod
27
def new_from_memory(cls, data: bytes) -> 'Source':
28
"""
29
Create source from memory buffer.
30
31
Parameters:
32
- data: bytes, image data in memory
33
34
Returns:
35
Source object for the memory buffer
36
"""
37
38
@classmethod
39
def new_from_descriptor(cls, descriptor: int) -> 'Source':
40
"""
41
Create source from file descriptor.
42
43
Parameters:
44
- descriptor: int, file descriptor
45
46
Returns:
47
Source object for the descriptor
48
"""
49
50
def filename(self) -> str:
51
"""Get associated filename if available."""
52
53
def nick(self) -> str:
54
"""Get human-readable name for the source."""
55
```
56
57
Example usage:
58
59
```python
60
# File source
61
source = pyvips.Source.new_from_file('input.jpg')
62
image = pyvips.Image.new_from_source(source, '')
63
64
# Memory source
65
with open('image.png', 'rb') as f:
66
data = f.read()
67
source = pyvips.Source.new_from_memory(data)
68
image = pyvips.Image.new_from_source(source, '.png')
69
70
# File descriptor source (Unix systems)
71
import os
72
fd = os.open('image.tiff', os.O_RDONLY)
73
source = pyvips.Source.new_from_descriptor(fd)
74
image = pyvips.Image.new_from_source(source, '.tiff')
75
os.close(fd)
76
77
# Source introspection
78
print(f"Source filename: {source.filename()}")
79
print(f"Source name: {source.nick()}")
80
```
81
82
### Target Objects
83
84
Output connections for writing image data to various destinations with streaming support.
85
86
```python { .api }
87
class Target:
88
"""Output connection for streaming image data."""
89
90
@classmethod
91
def new_to_file(cls, filename: str) -> 'Target':
92
"""
93
Create target for file output.
94
95
Parameters:
96
- filename: str, path to output file
97
98
Returns:
99
Target object for the file
100
"""
101
102
@classmethod
103
def new_to_memory(cls) -> 'Target':
104
"""
105
Create target for memory output.
106
107
Returns:
108
Target object for memory buffer
109
"""
110
111
@classmethod
112
def new_to_descriptor(cls, descriptor: int) -> 'Target':
113
"""
114
Create target for file descriptor.
115
116
Parameters:
117
- descriptor: int, file descriptor
118
119
Returns:
120
Target object for the descriptor
121
"""
122
123
def filename(self) -> str:
124
"""Get associated filename if available."""
125
126
def nick(self) -> str:
127
"""Get human-readable name for the target."""
128
```
129
130
Example usage:
131
132
```python
133
# File target
134
target = pyvips.Target.new_to_file('output.jpg')
135
image.write_to_target(target, '.jpg', Q=90)
136
137
# Memory target
138
target = pyvips.Target.new_to_memory()
139
image.write_to_target(target, '.png', compression=6)
140
# Note: Getting data from memory target requires libvips >= 8.13
141
142
# File descriptor target (Unix systems)
143
import os
144
fd = os.open('output.webp', os.O_WRONLY | os.O_CREAT, 0o644)
145
target = pyvips.Target.new_to_descriptor(fd)
146
image.write_to_target(target, '.webp', Q=80)
147
os.close(fd)
148
149
# Target introspection
150
print(f"Target filename: {target.filename()}")
151
print(f"Target name: {target.nick()}")
152
```
153
154
### Custom Sources
155
156
Create custom input sources for specialized data sources and protocols.
157
158
```python { .api }
159
class SourceCustom(Source):
160
"""Custom input source for specialized data sources."""
161
162
def __init__(self):
163
"""Create new custom source."""
164
165
def on_read(self, handler: callable) -> None:
166
"""
167
Attach read handler.
168
169
Parameters:
170
- handler: callable, function(size) -> bytes
171
Should return up to 'size' bytes, empty bytes for EOF
172
"""
173
174
def on_seek(self, handler: callable) -> None:
175
"""
176
Attach seek handler (optional).
177
178
Parameters:
179
- handler: callable, function(offset, whence) -> int
180
Should seek to position and return new position
181
whence: 0=SEEK_SET, 1=SEEK_CUR, 2=SEEK_END
182
"""
183
```
184
185
Example usage:
186
187
```python
188
# HTTP source example
189
import requests
190
191
class HTTPSource:
192
def __init__(self, url):
193
self.response = requests.get(url, stream=True)
194
self.iter = iter(self.response.iter_content(chunk_size=8192))
195
self.position = 0
196
197
def read_handler(self, size):
198
try:
199
chunk = next(self.iter)
200
self.position += len(chunk)
201
return chunk
202
except StopIteration:
203
return b'' # EOF
204
205
# Use HTTP source
206
http_source = HTTPSource('https://example.com/image.jpg')
207
source = pyvips.SourceCustom()
208
source.on_read(http_source.read_handler)
209
image = pyvips.Image.new_from_source(source, '')
210
211
# Database source example
212
import sqlite3
213
214
class DatabaseSource:
215
def __init__(self, db_path, image_id):
216
self.conn = sqlite3.connect(db_path)
217
cursor = self.conn.cursor()
218
cursor.execute("SELECT image_data FROM images WHERE id = ?", (image_id,))
219
self.data = cursor.fetchone()[0]
220
self.position = 0
221
222
def read_handler(self, size):
223
if self.position >= len(self.data):
224
return b''
225
chunk = self.data[self.position:self.position + size]
226
self.position += len(chunk)
227
return chunk
228
229
def seek_handler(self, offset, whence):
230
if whence == 0: # SEEK_SET
231
self.position = offset
232
elif whence == 1: # SEEK_CUR
233
self.position += offset
234
elif whence == 2: # SEEK_END
235
self.position = len(self.data) + offset
236
return self.position
237
238
# Use database source
239
db_source = DatabaseSource('images.db', 123)
240
source = pyvips.SourceCustom()
241
source.on_read(db_source.read_handler)
242
source.on_seek(db_source.seek_handler)
243
image = pyvips.Image.new_from_source(source, '')
244
```
245
246
### Custom Targets
247
248
Create custom output targets for specialized data destinations and protocols.
249
250
```python { .api }
251
class TargetCustom(Target):
252
"""Custom output target for specialized destinations."""
253
254
def __init__(self):
255
"""Create new custom target."""
256
257
def on_write(self, handler: callable) -> None:
258
"""
259
Attach write handler.
260
261
Parameters:
262
- handler: callable, function(data) -> int
263
Should write bytes and return number of bytes written
264
"""
265
266
def on_read(self, handler: callable) -> None:
267
"""
268
Attach read handler (optional, for read-back capability).
269
270
Parameters:
271
- handler: callable, function(size) -> bytes
272
"""
273
274
def on_seek(self, handler: callable) -> None:
275
"""
276
Attach seek handler (optional).
277
278
Parameters:
279
- handler: callable, function(offset, whence) -> int
280
"""
281
282
def on_end(self, handler: callable) -> None:
283
"""
284
Attach end handler (libvips >= 8.13).
285
286
Parameters:
287
- handler: callable, function() -> None
288
Called when write operation completes
289
"""
290
291
def on_finish(self, handler: callable) -> None:
292
"""
293
Attach finish handler (deprecated, use on_end).
294
295
Parameters:
296
- handler: callable, function() -> None
297
"""
298
```
299
300
Example usage:
301
302
```python
303
# HTTP upload target example
304
import requests
305
306
class HTTPUploadTarget:
307
def __init__(self, upload_url):
308
self.upload_url = upload_url
309
self.buffer = bytearray()
310
311
def write_handler(self, data):
312
self.buffer.extend(data)
313
return len(data)
314
315
def end_handler(self):
316
# Upload complete buffer
317
files = {'image': ('image.jpg', bytes(self.buffer), 'image/jpeg')}
318
response = requests.post(self.upload_url, files=files)
319
print(f"Upload status: {response.status_code}")
320
321
# Use HTTP upload target
322
upload_target = HTTPUploadTarget('https://api.example.com/upload')
323
target = pyvips.TargetCustom()
324
target.on_write(upload_target.write_handler)
325
target.on_end(upload_target.end_handler)
326
image.write_to_target(target, '.jpg', Q=90)
327
328
# Cloud storage target example
329
import boto3
330
331
class S3Target:
332
def __init__(self, bucket, key):
333
self.s3 = boto3.client('s3')
334
self.bucket = bucket
335
self.key = key
336
self.parts = []
337
self.buffer = bytearray()
338
339
def write_handler(self, data):
340
self.buffer.extend(data)
341
return len(data)
342
343
def end_handler(self):
344
# Upload to S3
345
self.s3.put_object(
346
Bucket=self.bucket,
347
Key=self.key,
348
Body=bytes(self.buffer)
349
)
350
print(f"Uploaded to s3://{self.bucket}/{self.key}")
351
352
# Use S3 target
353
s3_target = S3Target('my-bucket', 'processed/image.jpg')
354
target = pyvips.TargetCustom()
355
target.on_write(s3_target.write_handler)
356
target.on_end(s3_target.end_handler)
357
image.write_to_target(target, '.jpg', Q=85)
358
359
# Database target example
360
class DatabaseTarget:
361
def __init__(self, db_path, image_id):
362
self.conn = sqlite3.connect(db_path)
363
self.image_id = image_id
364
self.buffer = bytearray()
365
366
def write_handler(self, data):
367
self.buffer.extend(data)
368
return len(data)
369
370
def end_handler(self):
371
cursor = self.conn.cursor()
372
cursor.execute(
373
"UPDATE images SET image_data = ? WHERE id = ?",
374
(bytes(self.buffer), self.image_id)
375
)
376
self.conn.commit()
377
self.conn.close()
378
379
# Use database target
380
db_target = DatabaseTarget('images.db', 456)
381
target = pyvips.TargetCustom()
382
target.on_write(db_target.write_handler)
383
target.on_end(db_target.end_handler)
384
image.write_to_target(target, '.png', compression=6)
385
```
386
387
## Connection Base Class
388
389
Common functionality for all connection types.
390
391
```python { .api }
392
class Connection:
393
"""Abstract base connection class."""
394
395
def filename(self) -> str:
396
"""
397
Get associated filename.
398
399
Returns:
400
str, filename if available, empty string otherwise
401
"""
402
403
def nick(self) -> str:
404
"""
405
Get human-readable name.
406
407
Returns:
408
str, descriptive name for the connection
409
"""
410
```
411
412
## Streaming Benefits
413
414
The I/O connection system provides several advantages:
415
416
### Memory Efficiency
417
- Process large images without loading entirely into memory
418
- Constant memory usage regardless of image size
419
- Efficient for gigapixel+ images
420
421
### Network Integration
422
- Stream images directly from HTTP/HTTPS sources
423
- Upload processed results without temporary files
424
- Support for various protocols and APIs
425
426
### Database Integration
427
- Load and save images directly to/from databases
428
- Support for BLOB fields and binary data
429
- No intermediate file storage required
430
431
### Custom Protocols
432
- Implement specialized data sources and sinks
433
- Support for encrypted or compressed streams
434
- Integration with existing data pipelines
435
436
## Error Handling
437
438
```python
439
try:
440
source = pyvips.Source.new_from_file('nonexistent.jpg')
441
image = pyvips.Image.new_from_source(source, '')
442
except pyvips.Error as e:
443
print(f"Source creation failed: {e.message}")
444
445
# Custom source error handling
446
class SafeHTTPSource:
447
def __init__(self, url):
448
try:
449
self.response = requests.get(url, stream=True, timeout=30)
450
self.response.raise_for_status()
451
self.iter = iter(self.response.iter_content(chunk_size=8192))
452
except requests.RequestException as e:
453
raise pyvips.Error(f"HTTP request failed: {e}")
454
455
def read_handler(self, size):
456
try:
457
chunk = next(self.iter)
458
return chunk
459
except StopIteration:
460
return b''
461
except Exception as e:
462
raise pyvips.Error(f"Read failed: {e}")
463
464
# Robust custom target
465
class SafeFileTarget:
466
def __init__(self, filename):
467
self.filename = filename
468
self.file = None
469
470
def write_handler(self, data):
471
try:
472
if self.file is None:
473
self.file = open(self.filename, 'wb')
474
written = self.file.write(data)
475
return written
476
except IOError as e:
477
raise pyvips.Error(f"Write failed: {e}")
478
479
def end_handler(self):
480
if self.file:
481
self.file.close()
482
```
483
484
## Performance Tips
485
486
- Use sequential access for large images with `access='sequential'`
487
- Implement efficient buffering in custom handlers
488
- Consider chunk sizes for network operations
489
- Use seek handlers when random access is needed
490
- Handle errors gracefully to avoid resource leaks
491
- Close connections and files in end/finish handlers