0
# Extended Functionality
1

2
Advanced utilities including caching systems, file operations, async helpers, data serialization, and system integration tools for complex development scenarios. The xtras module provides essential utilities that extend the core functionality with specialized tools for file handling, caching, execution, and data transformation.
3

4
## Capabilities
5

6
### File System Operations
7

8
Comprehensive file system utilities including directory walking, glob operations, and path manipulation with advanced filtering capabilities.
9

10
```python { .api }
11
def walk(path, symlinks=True, keep_file=ret_true, keep_folder=ret_true, skip_folder=ret_false, func=os.path.join, ret_folders=False):
12
    """
13
    Generator version of os.walk with functional filtering.
14
    
15
    Enhanced directory traversal with customizable file and folder filtering
16
    using predicate functions. Provides more control than standard os.walk.
17
    
18
    Parameters:
19
    - path: Path|str, starting directory path
20
    - symlinks: bool, follow symbolic links (default: True)
21
    - keep_file: callable, function returning True for wanted files (default: ret_true)
22
    - keep_folder: callable, function returning True for folders to enter (default: ret_true)
23
    - skip_folder: callable, function returning True for folders to skip (default: ret_false)
24
    - func: callable, function to apply to each matched file (default: os.path.join)
25
    - ret_folders: bool, yield folders in addition to files (default: False)
26
    
27
    Yields:
28
    Results of func(root, name) for each matching file/folder
29
    """
30

31
def globtastic(path, recursive=True, symlinks=True, file_glob=None, file_re=None, 
32
               folder_re=None, skip_file_glob=None, skip_file_re=None, 
33
               skip_folder_re=None, func=os.path.join, ret_folders=False):
34
    """
35
    Enhanced glob with regex matches, symlink handling, and skip parameters.
36
    
37
    More powerful alternative to glob.glob with support for regex patterns,
38
    exclusion rules, and advanced filtering options.
39
    
40
    Parameters:
41
    - path: Path|str, starting directory path
42
    - recursive: bool, search subdirectories (default: True)
43
    - symlinks: bool, follow symbolic links (default: True)
44
    - file_glob: str, glob pattern for file inclusion
45
    - file_re: str, regex pattern for file inclusion
46
    - folder_re: str, regex pattern for folder inclusion
47
    - skip_file_glob: str, glob pattern for file exclusion
48
    - skip_file_re: str, regex pattern for file exclusion
49
    - skip_folder_re: str, regex pattern for folder exclusion
50
    - func: callable, function to apply to matches
51
    - ret_folders: bool, include folders in results
52
    
53
    Returns:
54
    L: List of paths matching the criteria
55
    """
56

57
def mkdir(path, exist_ok=False, parents=False, overwrite=False, **kwargs):
58
    """
59
    Create directory with enhanced options.
60
    
61
    Creates directory with Path support and optional overwrite capability.
62
    More flexible than standard os.makedirs.
63
    
64
    Parameters:
65
    - path: Path|str, directory to create
66
    - exist_ok: bool, don't raise error if directory exists
67
    - parents: bool, create parent directories as needed
68
    - overwrite: bool, remove existing directory first
69
    - **kwargs: additional arguments passed to Path.mkdir
70
    
71
    Returns:
72
    Path: The created directory path
73
    """
74

75
@contextmanager
76
def maybe_open(f, mode='r', **kwargs):
77
    """
78
    Context manager: open file if path, otherwise return as-is.
79
    
80
    Flexible file handling that works with both file paths and file objects.
81
    Automatically handles opening and closing for paths while passing through
82
    already opened file objects.
83
    
84
    Parameters:
85
    - f: str|PathLike|file-like, file path or file object
86
    - mode: str, file opening mode (default: 'r')
87
    - **kwargs: additional arguments for open()
88
    
89
    Yields:
90
    File object ready for reading/writing
91
    """
92
```
93

94
### Data Serialization and Loading
95

96
Functions for handling various data formats including JSON, pickle, and compressed data.
97

98
```python { .api }
99
def loads(s, **kwargs):
100
    """
101
    Load JSON with enhanced error handling.
102
    
103
    JSON loading with better error messages and support for Path objects.
104
    Handles common JSON parsing errors gracefully.
105
    
106
    Parameters:
107
    - s: str|Path, JSON string or file path
108
    - **kwargs: additional arguments for json.loads
109
    
110
    Returns:
111
    Parsed JSON object
112
    
113
    Raises:
114
    Informative errors for malformed JSON
115
    """
116

117
def loads_multi(s, **kwargs):
118
    """
119
    Load multiple JSON objects from string or file.
120
    
121
    Handles JSONL (JSON Lines) format and multiple JSON objects
122
    separated by whitespace.
123
    
124
    Parameters:
125
    - s: str|Path, JSON string or file path containing multiple objects
126
    - **kwargs: additional arguments for json.loads
127
    
128
    Returns:
129
    List of parsed JSON objects
130
    """
131

132
def dumps(obj, **kwargs):
133
    """
134
    Dump object to JSON with Path handling.
135
    
136
    Enhanced JSON serialization with support for Path objects
137
    and sensible defaults for common use cases.
138
    
139
    Parameters:
140
    - obj: Object to serialize
141
    - **kwargs: additional arguments for json.dumps
142
    
143
    Returns:
144
    str: JSON string representation
145
    """
146

147
def save_pickle(fn, o):
148
    """
149
    Save object to pickle file.
150
    
151
    Parameters:
152
    - fn: Path|str, file path for pickle
153
    - o: Object to pickle
154
    """
155

156
def load_pickle(fn):
157
    """
158
    Load object from pickle file.
159
    
160
    Parameters:
161
    - fn: Path|str, pickle file path
162
    
163
    Returns:
164
    Unpickled object
165
    """
166

167
def bunzip(fn):
168
    """
169
    Decompress bzip2 data from file or bytes.
170
    
171
    Parameters:
172
    - fn: Path|str|bytes, bzip2 compressed data or file path
173
    
174
    Returns:
175
    bytes: Decompressed data
176
    """
177
```
178

179
### System Execution and Environment
180

181
Utilities for running system commands and managing execution environment.
182

183
```python { .api }
184
def run(cmd, *rest, same_in_win=False, ignore_ex=False, as_bytes=False, stderr=False):
185
    """
186
    Run system command with enhanced error handling.
187
    
188
    Execute shell commands with flexible argument handling and
189
    comprehensive error reporting. Supports both string and list
190
    command specifications.
191
    
192
    Parameters:
193
    - cmd: str|list, command to execute
194
    - *rest: additional command arguments
195
    - same_in_win: bool, don't modify command for Windows compatibility
196
    - ignore_ex: bool, return None instead of raising on errors
197
    - as_bytes: bool, return output as bytes instead of string
198
    - stderr: bool, include stderr in output
199
    
200
    Returns:
201
    str|bytes|None: Command output or None if error and ignore_ex=True
202
    
203
    Raises:
204
    RuntimeError: If command fails and ignore_ex=False
205
    """
206

207
def exec_eval(code, glob=None, loc=None):
208
    """
209
    Execute code and return result of last expression.
210
    
211
    Enhanced exec that can return the value of the last expression
212
    in the code block, combining exec and eval functionality.
213
    
214
    Parameters:
215
    - code: str, Python code to execute
216
    - glob: dict, global namespace (default: None)
217
    - loc: dict, local namespace (default: None)
218
    
219
    Returns:
220
    Result of last expression in code, or None if no expression
221
    """
222

223
@contextmanager
224
def modified_env(**kwargs):
225
    """
226
    Context manager to temporarily modify environment variables.
227
    
228
    Parameters:
229
    - **kwargs: environment variables to set
230
    
231
    Usage:
232
    with modified_env(PATH="/new/path", DEBUG="1"):
233
        # Environment modified here
234
        subprocess.run(...)
235
    # Environment restored here
236
    """
237

238
def set_num_threads(n):
239
    """
240
    Set number of threads for various libraries.
241
    
242
    Sets thread counts for NumPy, OpenMP, and other numerical libraries
243
    to optimize performance on multi-core systems.
244
    
245
    Parameters:
246
    - n: int, number of threads to use
247
    """
248
```
249

250
### Caching and Performance
251

252
Advanced caching utilities including flexible caching policies and cached iterators.
253

254
```python { .api }
255
def flexicache(cache_dir=None, **kwargs):
256
    """
257
    Flexible caching decorator with multiple policies.
258
    
259
    Advanced caching with support for different cache policies including
260
    time-based expiration and modification time checking.
261
    
262
    Parameters:
263
    - cache_dir: Path|str, directory for cache files
264
    - **kwargs: additional caching options
265
    
266
    Returns:
267
    Decorator function for caching
268
    """
269

270
def time_policy(seconds):
271
    """
272
    Cache policy based on time expiration.
273
    
274
    Parameters:
275
    - seconds: int|float, cache duration in seconds
276
    
277
    Returns:
278
    Policy function for time-based cache expiration
279
    """
280

281
def mtime_policy():
282
    """
283
    Cache policy based on file modification time.
284
    
285
    Invalidates cache when source files are modified.
286
    
287
    Returns:
288
    Policy function for mtime-based cache invalidation
289
    """
290

291
def timed_cache(seconds=3600):
292
    """
293
    Simple time-based cache decorator.
294
    
295
    Parameters:
296
    - seconds: int, cache duration (default: 3600)
297
    
298
    Returns:
299
    Caching decorator
300
    """
301

302
class CachedIter:
303
    """
304
    Iterator that caches results for repeated iteration.
305
    
306
    Allows multiple iterations over the same data without
307
    recomputing expensive operations.
308
    
309
    Parameters:
310
    - func: callable, function that returns an iterator
311
    - *args: arguments for func
312
    - **kwargs: keyword arguments for func
313
    """
314
    
315
    def __init__(self, func, *args, **kwargs): ...
316
    def __iter__(self): ...
317

318
class CachedAwaitable:
319
    """
320
    Cache results of async operations.
321
    
322
    Provides caching for coroutines and async functions to avoid
323
    redundant async operations.
324
    
325
    Parameters:
326
    - coro: coroutine to cache
327
    """
328
    
329
    def __init__(self, coro): ...
330
    def __await__(self): ...
331

332
def reawaitable(f):
333
    """
334
    Decorator to make async functions re-awaitable.
335
    
336
    Allows the same coroutine result to be awaited multiple times
337
    without re-execution.
338
    
339
    Parameters:
340
    - f: async function to make re-awaitable
341
    
342
    Returns:
343
    Decorated async function
344
    """
345
```
346

347
### Data Transformation and Utilities
348

349
Specialized classes and functions for data manipulation and object transformation.
350

351
```python { .api }
352
class IterLen:
353
    """
354
    Iterator wrapper that supports len().
355
    
356
    Wraps iterators to provide length information when the
357
    underlying iterable has a __len__ method.
358
    
359
    Parameters:
360
    - items: iterable to wrap
361
    """
362
    
363
    def __init__(self, items): ...
364
    def __iter__(self): ...
365
    def __len__(self): ...
366

367
class ReindexCollection:
368
    """
369
    Collection that supports reindexing operations.
370
    
371
    Provides advanced indexing capabilities with support for
372
    reordering and subset selection.
373
    
374
    Parameters:
375
    - items: initial collection items
376
    """
377
    
378
    def __init__(self, items): ...
379
    def reindex(self, idxs): ...
380
    def __getitem__(self, idx): ...
381

382
class SaveReturn:
383
    """
384
    Context manager to save and return iteration results.
385
    
386
    Captures all yielded values from an iterator for later access
387
    while still allowing normal iteration.
388
    
389
    Parameters:
390
    - it: iterator to wrap
391
    """
392
    
393
    def __init__(self, it): ...
394
    def __enter__(self): ...
395
    def __exit__(self, *args): ...
396
    def __iter__(self): ...
397

398
def dict2obj(d, **kwargs):
399
    """
400
    Convert dictionary to object with attribute access.
401
    
402
    Parameters:
403
    - d: dict, dictionary to convert
404
    - **kwargs: additional attributes to set
405
    
406
    Returns:
407
    Object with dictionary keys as attributes
408
    """
409

410
def obj2dict(obj, **kwargs):
411
    """
412
    Convert object to dictionary.
413
    
414
    Parameters:
415
    - obj: object to convert
416
    - **kwargs: additional dictionary entries
417
    
418
    Returns:
419
    dict: Object attributes as dictionary
420
    """
421

422
def is_listy(x):
423
    """
424
    Check if object is list-like but not string/dict.
425
    
426
    Parameters:
427
    - x: object to check
428
    
429
    Returns:
430
    bool: True if list-like (iterable but not string/dict)
431
    """
432

433
def mapped(f):
434
    """
435
    Create a mapped version of an iterable.
436
    
437
    Parameters:
438
    - f: function to map over items
439
    
440
    Returns:
441
    Function that applies f to iterables
442
    """
443
```
444

445
### String and Formatting Utilities
446

447
Text processing and formatting functions for display and analysis.
448

449
```python { .api }
450
def truncstr(s, maxlen=50, suf='…'):
451
    """
452
    Truncate string with suffix if too long.
453
    
454
    Parameters:
455
    - s: str, string to truncate
456
    - maxlen: int, maximum length (default: 50)
457
    - suf: str, suffix for truncated strings (default: '…')
458
    
459
    Returns:
460
    str: Truncated string with suffix if needed
461
    """
462

463
def sparkline(data, mn=None, mx=None):
464
    """
465
    Create Unicode sparkline from numeric data.
466
    
467
    Parameters:
468
    - data: iterable of numbers
469
    - mn: float, minimum value (auto-detected if None)
470
    - mx: float, maximum value (auto-detected if None)
471
    
472
    Returns:
473
    str: Unicode sparkline representation
474
    """
475

476
class PartialFormatter(string.Formatter):
477
    """
478
    String formatter that handles missing keys gracefully.
479
    
480
    Allows partial string formatting where some placeholders
481
    can remain unfilled without raising KeyError.
482
    """
483
    
484
    def get_value(self, key, args, kwargs): ...
485

486
def partial_format(s, **kwargs):
487
    """
488
    Partially format string, leaving unfilled placeholders.
489
    
490
    Parameters:
491
    - s: str, format string with placeholders
492
    - **kwargs: values for available placeholders
493
    
494
    Returns:
495
    str: Partially formatted string
496
    """
497

498
def stringfmt_names(s):
499
    """
500
    Extract placeholder names from format string.
501
    
502
    Parameters:
503
    - s: str, format string
504
    
505
    Returns:
506
    set: Set of placeholder names
507
    """
508
```
509

510
### Development and Debugging Tools
511

512
Utilities for development workflow including timing, tracing, and code analysis.
513

514
```python { .api }
515
class EventTimer:
516
    """
517
    Timer for measuring event durations with context manager support.
518
    
519
    Provides precise timing measurements with automatic formatting
520
    and optional storage of timing results.
521
    
522
    Parameters:
523
    - store: bool, store timing results (default: False)
524
    - run: bool, start timing immediately (default: True)
525
    """
526
    
527
    def __init__(self, store=False, run=True): ...
528
    def __enter__(self): ...
529
    def __exit__(self, *args): ...
530
    def start(self): ...
531
    def stop(self): ...
532

533
def trace(f):
534
    """
535
    Decorator to trace function calls with arguments and results.
536
    
537
    Prints function name, arguments, and return values for debugging.
538
    Useful for understanding program flow and data transformations.
539
    
540
    Parameters:
541
    - f: function to trace
542
    
543
    Returns:
544
    Wrapped function with tracing
545
    """
546

547
def modify_exception(e, msg_func, **kwargs):
548
    """
549
    Modify exception message with additional context.
550
    
551
    Parameters:
552
    - e: Exception to modify
553
    - msg_func: function to transform exception message
554
    - **kwargs: additional context for message
555
    
556
    Returns:
557
    Modified exception
558
    """
559

560
def round_multiple(x, mult=1):
561
    """
562
    Round number to nearest multiple.
563
    
564
    Parameters:
565
    - x: number to round
566
    - mult: multiple to round to (default: 1)
567
    
568
    Returns:
569
    Number rounded to nearest multiple
570
    """
571

572
class ContextManagers:
573
    """
574
    Manage multiple context managers as a single unit.
575
    
576
    Allows entering and exiting multiple context managers together,
577
    with proper cleanup even if some fail.
578
    
579
    Parameters:
580
    - *cms: context managers to manage
581
    """
582
    
583
    def __init__(self, *cms): ...
584
    def __enter__(self): ...
585
    def __exit__(self, *args): ...
586
```
587

588
## Usage Examples
589

590
### Advanced File Operations
591

592
```python
593
from fastcore.xtras import walk, globtastic, mkdir
594

595
# Advanced directory walking with filtering
596
python_files = list(walk(
597
    "src", 
598
    keep_file=lambda root, name: name.endswith('.py'),
599
    skip_folder=lambda root, name: name.startswith('.')
600
))
601

602
# Powerful glob with regex and exclusions
603
source_files = globtastic(
604
    "project",
605
    file_re=r".*\.(py|js|ts)$",
606
    skip_folder_re=r"(node_modules|__pycache__|\.git)",
607
    recursive=True
608
)
609

610
# Create directory with overwrite
611
build_dir = mkdir("build", parents=True, overwrite=True)
612
```
613

614
### Caching and Performance
615

616
```python
617
from fastcore.xtras import flexicache, timed_cache, CachedIter
618

619
# Flexible caching with time policy
620
@flexicache(cache_dir=".cache")
621
def expensive_computation(data):
622
    # Time-intensive processing
623
    return process_data(data)
624

625
# Simple time-based caching
626
@timed_cache(seconds=300)  # 5 minute cache
627
def api_call(endpoint):
628
    return requests.get(endpoint).json()
629

630
# Cache iterator results
631
def data_generator():
632
    for i in range(1000000):
633
        yield expensive_transform(i)
634

635
cached_data = CachedIter(data_generator)
636
# First iteration computes and caches
637
for item in cached_data: process(item)
638
# Second iteration uses cached results
639
for item in cached_data: analyze(item)
640
```
641

642
### System Execution and Environment
643

644
```python
645
from fastcore.xtras import run, modified_env, exec_eval
646

647
# Enhanced command execution
648
result = run(["python", "script.py", "--flag"], 
649
             stderr=True, ignore_ex=True)
650

651
# Temporary environment modification
652
with modified_env(DEBUG="1", LOG_LEVEL="verbose"):
653
    run("my_app --config dev.json")
654

655
# Execute code and get result
656
result = exec_eval("""
657
x = 10
658
y = 20
659
x + y  # This value is returned
660
""")
661
print(result)  # 30
662
```
663

664
### Data Transformation
665

666
```python
667
from fastcore.xtras import dict2obj, ReindexCollection, SaveReturn
668

669
# Dictionary to object conversion
670
config = dict2obj({
671
    "model": "resnet50", 
672
    "lr": 0.01,
673
    "batch_size": 32
674
})
675
print(config.model)  # "resnet50"
676

677
# Reindexing collections
678
data = ReindexCollection([10, 20, 30, 40, 50])
679
subset = data.reindex([0, 2, 4])  # [10, 30, 50]
680

681
# Save iteration results
682
def number_generator():
683
    for i in range(5):
684
        yield i * i
685

686
with SaveReturn(number_generator()) as saver:
687
    for val in saver:
688
        print(f"Processing {val}")
689

690
results = saver.values  # [0, 1, 4, 9, 16]
691
```
692

693
### String Processing and Formatting
694

695
```python
696
from fastcore.xtras import truncstr, sparkline, partial_format
697

698
# String truncation
699
long_text = "This is a very long string that needs truncation"
700
short = truncstr(long_text, maxlen=20)  # "This is a very lo…"
701

702
# Unicode sparklines
703
data = [1, 3, 7, 4, 2, 8, 5, 3, 2, 1]
704
chart = sparkline(data)  # "▁▃▇▄▂█▅▃▂▁"
705

706
# Partial string formatting
707
template = "Hello {name}, your score is {score}"
708
partial = partial_format(template, name="Alice")
709
# "Hello Alice, your score is {score}"
710
```
711

712
### Development Tools
713

714
```python
715
from fastcore.xtras import EventTimer, trace, ContextManagers
716

717
# Precise timing
718
with EventTimer() as timer:
719
    expensive_operation()
720
print(f"Operation took {timer.elapsed:.3f} seconds")
721

722
# Function tracing
723
@trace
724
def fibonacci(n):
725
    if n <= 1: return n
726
    return fibonacci(n-1) + fibonacci(n-2)
727

728
# Managing multiple contexts
729
from contextlib import contextmanager
730

731
@contextmanager
732
def temp_file(): 
733
    # ... create temp file
734
    yield file
735
    # ... cleanup
736

737
with ContextManagers(open('file1.txt'), temp_file(), timer):
738
    # All contexts entered together
739
    process_files()
740
# All contexts properly exited
741
```