0
# Persistent Data Structures
1
2
DiskCache provides persistent, disk-backed data structures that offer familiar interfaces while storing data reliably on disk. The Deque class implements a double-ended queue with the collections.abc.Sequence interface, while the Index class implements a mapping/dictionary with the collections.abc.MutableMapping interface.
3
4
## Capabilities
5
6
### Deque - Persistent Double-Ended Queue
7
8
A persistent sequence supporting efficient append and pop operations from both ends, with full compatibility with Python's collections.abc.Sequence interface.
9
10
```python { .api }
11
class Deque:
12
def __init__(self, iterable=(), directory=None, maxlen=None):
13
"""
14
Initialize persistent deque.
15
16
Args:
17
iterable: Initial values to populate deque
18
directory (str, optional): Directory for storage. If None, creates temp directory.
19
maxlen (int, optional): Maximum length. When full, adding items removes from opposite end.
20
"""
21
22
@classmethod
23
def fromcache(cls, cache, iterable=(), maxlen=None):
24
"""
25
Create Deque from existing Cache instance.
26
27
Args:
28
cache (Cache): Existing Cache instance to use for storage
29
iterable: Initial values to populate deque
30
maxlen (int, optional): Maximum length
31
32
Returns:
33
Deque: New Deque instance using the provided cache
34
"""
35
36
@property
37
def cache(self):
38
"""Underlying Cache instance used for storage."""
39
40
@property
41
def directory(self):
42
"""Directory path for storage."""
43
44
@property
45
def maxlen(self):
46
"""Maximum length (None if unlimited)."""
47
48
@maxlen.setter
49
def maxlen(self, value):
50
"""Set maximum length."""
51
```
52
53
#### Sequence Operations
54
55
Standard sequence operations with persistent storage.
56
57
```python { .api }
58
def __getitem__(self, index):
59
"""
60
Get item by index using deque[index] syntax.
61
62
Args:
63
index (int): Index (supports negative indexing)
64
65
Returns:
66
Item at the specified index
67
68
Raises:
69
IndexError: If index is out of range
70
"""
71
72
def __setitem__(self, index, value):
73
"""
74
Set item by index using deque[index] = value syntax.
75
76
Args:
77
index (int): Index (supports negative indexing)
78
value: Value to set
79
80
Raises:
81
IndexError: If index is out of range
82
"""
83
84
def __delitem__(self, index):
85
"""
86
Delete item by index using del deque[index] syntax.
87
88
Args:
89
index (int): Index (supports negative indexing)
90
91
Raises:
92
IndexError: If index is out of range
93
"""
94
95
def __len__(self):
96
"""Get number of items in deque."""
97
98
def __iter__(self):
99
"""Iterate from front to back."""
100
101
def __reversed__(self):
102
"""Iterate from back to front."""
103
104
def __contains__(self, value):
105
"""Check if value exists in deque using 'value in deque' syntax."""
106
```
107
108
#### Deque-Specific Operations
109
110
Efficient operations for double-ended queue functionality.
111
112
```python { .api }
113
def append(self, value):
114
"""
115
Add value to the back (right side) of deque.
116
117
Args:
118
value: Value to append
119
"""
120
121
def appendleft(self, value):
122
"""
123
Add value to the front (left side) of deque.
124
125
Args:
126
value: Value to append to front
127
"""
128
129
def pop(self):
130
"""
131
Remove and return value from the back (right side) of deque.
132
133
Returns:
134
Value from back of deque
135
136
Raises:
137
IndexError: If deque is empty
138
"""
139
140
def popleft(self):
141
"""
142
Remove and return value from the front (left side) of deque.
143
144
Returns:
145
Value from front of deque
146
147
Raises:
148
IndexError: If deque is empty
149
"""
150
151
def peek(self):
152
"""
153
Return value from back without removing it.
154
155
Returns:
156
Value from back of deque
157
158
Raises:
159
IndexError: If deque is empty
160
"""
161
162
def peekleft(self):
163
"""
164
Return value from front without removing it.
165
166
Returns:
167
Value from front of deque
168
169
Raises:
170
IndexError: If deque is empty
171
"""
172
```
173
174
#### Extended Operations
175
176
Additional operations for working with sequences and managing the deque.
177
178
```python { .api }
179
def extend(self, iterable):
180
"""
181
Extend deque by appending elements from iterable to the back.
182
183
Args:
184
iterable: Sequence of values to append
185
"""
186
187
def extendleft(self, iterable):
188
"""
189
Extend deque by appending elements from iterable to the front.
190
Note: Order is reversed - first item in iterable becomes last added.
191
192
Args:
193
iterable: Sequence of values to append to front
194
"""
195
196
def clear(self):
197
"""Remove all elements from deque."""
198
199
def copy(self):
200
"""
201
Create shallow copy of deque.
202
203
Returns:
204
Deque: New deque with same elements
205
"""
206
207
def count(self, value):
208
"""
209
Count occurrences of value in deque.
210
211
Args:
212
value: Value to count
213
214
Returns:
215
int: Number of occurrences
216
"""
217
218
def remove(self, value):
219
"""
220
Remove first occurrence of value from deque.
221
222
Args:
223
value: Value to remove
224
225
Raises:
226
ValueError: If value is not found
227
"""
228
229
def reverse(self):
230
"""Reverse the deque in place."""
231
232
def rotate(self, steps=1):
233
"""
234
Rotate deque steps positions to the right.
235
236
Args:
237
steps (int): Number of steps to rotate. Positive values rotate right,
238
negative values rotate left. Default 1.
239
"""
240
```
241
242
#### Comparison Operations
243
244
Rich comparison operations following sequence semantics.
245
246
```python { .api }
247
def __eq__(self, other):
248
"""
249
Test equality with another sequence.
250
251
Args:
252
other: Another sequence to compare
253
254
Returns:
255
bool: True if sequences are equal
256
"""
257
258
def __ne__(self, other):
259
"""Test inequality with another sequence."""
260
261
def __lt__(self, other):
262
"""Test if lexicographically less than another sequence."""
263
264
def __le__(self, other):
265
"""Test if lexicographically less than or equal to another sequence."""
266
267
def __gt__(self, other):
268
"""Test if lexicographically greater than another sequence."""
269
270
def __ge__(self, other):
271
"""Test if lexicographically greater than or equal to another sequence."""
272
```
273
274
#### Advanced Deque Operations
275
276
Advanced operations for transactions and serialization.
277
278
```python { .api }
279
def __iadd__(self, iterable):
280
"""
281
In-place concatenation using deque += iterable syntax.
282
283
Args:
284
iterable: Sequence of values to append
285
286
Returns:
287
Deque: Self (for chaining)
288
"""
289
290
def __repr__(self):
291
"""String representation of deque."""
292
293
def transact(self):
294
"""
295
Context manager for atomic transactions.
296
297
Returns:
298
Context manager for transaction on underlying cache
299
"""
300
301
def __getstate__(self):
302
"""Support for pickle serialization."""
303
304
def __setstate__(self, state):
305
"""Support for pickle deserialization."""
306
```
307
308
### Index - Persistent Mapping
309
310
A persistent mapping/dictionary that implements the collections.abc.MutableMapping interface with disk-based storage.
311
312
```python { .api }
313
class Index:
314
def __init__(self, *args, **kwargs):
315
"""
316
Initialize persistent mapping.
317
318
Args:
319
directory (str, optional): Directory for storage as first positional arg
320
Other args/kwargs: Initial mapping data (same as dict constructor)
321
"""
322
323
@classmethod
324
def fromcache(cls, cache, *args, **kwargs):
325
"""
326
Create Index from existing Cache instance.
327
328
Args:
329
cache (Cache): Existing Cache instance to use for storage
330
Other args/kwargs: Initial mapping data
331
332
Returns:
333
Index: New Index instance using the provided cache
334
"""
335
336
@property
337
def cache(self):
338
"""Underlying Cache instance used for storage."""
339
340
@property
341
def directory(self):
342
"""Directory path for storage."""
343
```
344
345
#### Mapping Operations
346
347
Standard mapping operations with persistent storage.
348
349
```python { .api }
350
def __getitem__(self, key):
351
"""
352
Get value by key using index[key] syntax.
353
354
Args:
355
key: Key to retrieve
356
357
Returns:
358
Value associated with key
359
360
Raises:
361
KeyError: If key is not found
362
"""
363
364
def __setitem__(self, key, value):
365
"""
366
Set key-value pair using index[key] = value syntax.
367
368
Args:
369
key: Key to store
370
value: Value to associate with key
371
"""
372
373
def __delitem__(self, key):
374
"""
375
Delete key using del index[key] syntax.
376
377
Args:
378
key: Key to delete
379
380
Raises:
381
KeyError: If key is not found
382
"""
383
384
def __len__(self):
385
"""Get number of key-value pairs in index."""
386
387
def __iter__(self):
388
"""Iterate over keys in sorted order."""
389
390
def __reversed__(self):
391
"""Iterate over keys in reverse sorted order."""
392
393
def __contains__(self, key):
394
"""Check if key exists using 'key in index' syntax."""
395
```
396
397
#### Dictionary Methods
398
399
Standard dictionary methods for persistent mapping.
400
401
```python { .api }
402
def get(self, key, default=None):
403
"""
404
Get value with default fallback.
405
406
Args:
407
key: Key to retrieve
408
default: Default value if key not found
409
410
Returns:
411
Value associated with key or default
412
"""
413
414
def pop(self, key, default=ENOVAL):
415
"""
416
Remove key and return its value.
417
418
Args:
419
key: Key to remove
420
default: Default value if key not found (optional)
421
422
Returns:
423
Value that was associated with key
424
425
Raises:
426
KeyError: If key not found and no default provided
427
"""
428
429
def popitem(self, last=True):
430
"""
431
Remove and return arbitrary key-value pair.
432
433
Args:
434
last (bool): If True, LIFO order; if False, FIFO order. Default True.
435
436
Returns:
437
Tuple of (key, value)
438
439
Raises:
440
KeyError: If index is empty
441
"""
442
443
def setdefault(self, key, default=None):
444
"""
445
Get value for key, setting to default if key doesn't exist.
446
447
Args:
448
key: Key to get or set
449
default: Default value to set if key doesn't exist
450
451
Returns:
452
Existing value for key or default (which is also stored)
453
"""
454
455
def clear(self):
456
"""Remove all key-value pairs from index."""
457
458
def update(self, *args, **kwargs):
459
"""
460
Update index with key-value pairs from other mapping or iterable.
461
462
Args:
463
Same as dict.update() - mapping, iterable of pairs, or keyword args
464
"""
465
```
466
467
#### View Objects
468
469
Dictionary view objects for keys, values, and items.
470
471
```python { .api }
472
def keys(self):
473
"""
474
Return KeysView of index keys.
475
476
Returns:
477
KeysView: View of keys that updates with index changes
478
"""
479
480
def values(self):
481
"""
482
Return ValuesView of index values.
483
484
Returns:
485
ValuesView: View of values that updates with index changes
486
"""
487
488
def items(self):
489
"""
490
Return ItemsView of index key-value pairs.
491
492
Returns:
493
ItemsView: View of (key, value) pairs that updates with index changes
494
"""
495
```
496
497
#### Queue Operations
498
499
Use the index as a queue with key-value pairs.
500
501
```python { .api }
502
def push(self, value, prefix=None, side='back'):
503
"""
504
Push value to queue with generated key.
505
506
Args:
507
value: Value to push
508
prefix (str, optional): Key prefix for queue namespace
509
side (str): Which side to push to ('back' or 'front'). Default 'back'.
510
511
Returns:
512
Generated key for the pushed value
513
"""
514
515
def pull(self, prefix=None, default=(None, None), side='front'):
516
"""
517
Pull key-value pair from queue.
518
519
Args:
520
prefix (str, optional): Key prefix for queue namespace
521
default: Default value if queue is empty. Default (None, None).
522
side (str): Which side to pull from ('front' or 'back'). Default 'front'.
523
524
Returns:
525
Tuple of (key, value) or default if queue empty
526
"""
527
528
def peekitem(self, last=True):
529
"""
530
Peek at key-value pair without removing it.
531
532
Args:
533
last (bool): Peek at last item if True, first if False. Default True.
534
535
Returns:
536
Tuple of (key, value) or (None, None) if index empty
537
"""
538
```
539
540
#### Comparison Operations
541
542
Comparison operations for mappings.
543
544
```python { .api }
545
def __eq__(self, other):
546
"""
547
Test equality with another mapping.
548
549
Args:
550
other: Another mapping to compare
551
552
Returns:
553
bool: True if mappings have same key-value pairs
554
"""
555
556
def __ne__(self, other):
557
"""Test inequality with another mapping."""
558
```
559
560
#### Advanced Index Operations
561
562
Advanced operations for memoization, transactions, and serialization.
563
564
```python { .api }
565
def __repr__(self):
566
"""String representation of index."""
567
568
def memoize(self, name=None, typed=False, ignore=()):
569
"""
570
Memoization decorator using this index.
571
572
Args:
573
name (str, optional): Name for memoized function. Default function name.
574
typed (bool): Distinguish arguments by type. Default False.
575
ignore (tuple): Argument positions/names to ignore in caching
576
577
Returns:
578
Decorator function
579
"""
580
581
def transact(self):
582
"""
583
Context manager for atomic transactions.
584
585
Returns:
586
Context manager for transaction on underlying cache
587
"""
588
589
def __getstate__(self):
590
"""Support for pickle serialization."""
591
592
def __setstate__(self, state):
593
"""Support for pickle deserialization."""
594
```
595
596
## Usage Examples
597
598
### Deque Usage
599
600
```python
601
import diskcache
602
603
# Create persistent deque
604
deque = diskcache.Deque('/tmp/mydeque')
605
606
# Basic deque operations
607
deque.append('item1')
608
deque.appendleft('item0')
609
deque.extend(['item2', 'item3'])
610
611
print(f"Length: {len(deque)}") # 4
612
print(f"Front: {deque.peekleft()}") # 'item0'
613
print(f"Back: {deque.peek()}") # 'item3'
614
615
# Pop items
616
back_item = deque.pop() # 'item3'
617
front_item = deque.popleft() # 'item0'
618
619
# Sequence operations
620
print(deque[0]) # 'item1'
621
deque[0] = 'modified_item1'
622
623
# Bounded deque
624
bounded = diskcache.Deque(maxlen=3)
625
bounded.extend([1, 2, 3, 4, 5]) # Only keeps last 3: [3, 4, 5]
626
```
627
628
### Index Usage
629
630
```python
631
import diskcache
632
633
# Create persistent index
634
index = diskcache.Index('/tmp/myindex')
635
636
# Basic mapping operations
637
index['user:123'] = {'name': 'Alice', 'age': 30}
638
index['user:456'] = {'name': 'Bob', 'age': 25}
639
640
print(index['user:123']) # {'name': 'Alice', 'age': 30}
641
print(len(index)) # 2
642
643
# Dictionary methods
644
user = index.get('user:789', {'name': 'Unknown'})
645
index.setdefault('settings', {'theme': 'light'})
646
647
# Iterate over keys, values, items
648
for key in index:
649
print(f"Key: {key}")
650
651
for value in index.values():
652
print(f"Value: {value}")
653
654
for key, value in index.items():
655
print(f"{key}: {value}")
656
657
# Queue operations with generated keys
658
task_key = index.push({'task': 'send_email', 'user_id': 123})
659
key, task = index.pull()
660
```
661
662
### Advanced Usage
663
664
```python
665
# Create from existing cache
666
cache = diskcache.Cache('/tmp/shared')
667
deque = diskcache.Deque.fromcache(cache, [1, 2, 3])
668
index = diskcache.Index.fromcache(cache, {'a': 1, 'b': 2})
669
670
# Atomic operations
671
with index.transact():
672
index['counter'] = index.get('counter', 0) + 1
673
index['last_update'] = time.time()
674
675
# Memoization with Index
676
@index.memoize(typed=True)
677
def expensive_function(x, y):
678
return x ** y
679
680
result = expensive_function(2, 10) # Computed and stored
681
result = expensive_function(2, 10) # Retrieved from index
682
683
# Deque as a work queue
684
work_queue = diskcache.Deque('/tmp/work_queue')
685
686
# Producer
687
for i in range(100):
688
work_queue.append({'job_id': i, 'data': f'job_{i}'})
689
690
# Consumer
691
while len(work_queue) > 0:
692
job = work_queue.popleft()
693
print(f"Processing {job}")
694
```
695
696
### Performance Considerations
697
698
```python
699
# For large datasets, consider using transactions
700
large_index = diskcache.Index('/tmp/large_data')
701
702
# Bulk insert with transaction
703
with large_index.transact():
704
for i in range(10000):
705
large_index[f'key_{i}'] = {'value': i, 'squared': i**2}
706
707
# Bounded deque for fixed-size buffers
708
log_buffer = diskcache.Deque('/tmp/logs', maxlen=1000)
709
710
# Always keeps only the last 1000 log entries
711
for i in range(5000):
712
log_buffer.append(f'Log entry {i}')
713
714
print(len(log_buffer)) # 1000 (not 5000)
715
```