0
# Advanced Features
1

2
Advanced filesystem operations, algorithms, patching capabilities, and the extended filesystem (XFS) that provide sophisticated functionality beyond basic file operations.
3

4
## Overview
5

6
The advanced features of @yarnpkg/fslib include:
7
- **Algorithm Utilities**: Copy operations, directory handling, and file watching
8
- **Filesystem Patching**: Integration with Node.js fs module
9
- **Extended Filesystem (XFS)**: Enhanced filesystem with temporary file management
10
- **Custom Directory Operations**: Advanced directory iteration and management
11

12
## Algorithm Utilities
13

14
### Copy Operations and Strategies
15

16
Advanced copy operations with configurable link handling strategies.
17

18
```typescript { .api }
19
import { 
20
  setupCopyIndex, 
21
  type LinkStrategy, 
22
  type HardlinkFromIndexStrategy,
23
  type PortablePath,
24
  type FakeFS
25
} from '@yarnpkg/fslib';
26

27
/**
28
 * Sets up a copy index for efficient hardlink handling during copy operations
29
 * Creates the index directory structure with hex-based subdirectories
30
 */
31
function setupCopyIndex<P extends Path>(
32
  destinationFs: FakeFS<P>, 
33
  linkStrategy: Pick<HardlinkFromIndexStrategy<P>, 'indexPath'>
34
): Promise<P>;
35

36
// Link strategy types
37
type HardlinkFromIndexStrategy<P> = {
38
  type: 'HardlinkFromIndex';
39
  indexPath: P;
40
  autoRepair?: boolean;
41
  readOnly?: boolean;
42
};
43

44
type LinkStrategy<P> = HardlinkFromIndexStrategy<P>;
45

46
// Usage example
47
import { xfs, ppath } from '@yarnpkg/fslib';
48

49
const indexPath = ppath.join('/tmp' as PortablePath, 'copy-index');
50
await setupCopyIndex(xfs, { indexPath });
51
```
52

53
#### Advanced Copy Operations
54

55
```typescript { .api }
56
import { setupCopyIndex, xfs, ppath, type PortablePath } from '@yarnpkg/fslib';
57

58
// Setup efficient copy operations with hardlink tracking
59
async function performLargeCopy(
60
  sourceDir: PortablePath,
61
  targetDir: PortablePath
62
): Promise<void> {
63
  // Setup copy index for efficient hardlink detection
64
  const indexPath = ppath.join(targetDir, '.copy-index' as PortablePath);
65
  await setupCopyIndex(indexPath);
66
  
67
  // Perform recursive copy with hardlink preservation
68
  await copyDirectoryRecursive(sourceDir, targetDir, {
69
    preserveHardlinks: true,
70
    indexPath
71
  });
72
}
73

74
// Custom copy implementation with progress tracking
75
async function copyDirectoryRecursive(
76
  source: PortablePath,
77
  target: PortablePath,
78
  options: { preserveHardlinks?: boolean; indexPath?: PortablePath } = {}
79
): Promise<void> {
80
  await xfs.mkdirPromise(target, { recursive: true });
81
  
82
  const entries = await xfs.readdirPromise(source, { withFileTypes: true });
83
  
84
  for (const entry of entries) {
85
    const srcPath = ppath.join(source, entry.name);
86
    const dstPath = ppath.join(target, entry.name);
87
    
88
    if (entry.isDirectory()) {
89
      await copyDirectoryRecursive(srcPath, dstPath, options);
90
    } else if (entry.isFile()) {
91
      // Use copy with hardlink detection if available
92
      await xfs.copyFilePromise(srcPath, dstPath);
93
    } else if (entry.isSymbolicLink()) {
94
      const linkTarget = await xfs.readlinkPromise(srcPath);
95
      await xfs.symlinkPromise(linkTarget, dstPath);
96
    }
97
  }
98
}
99
```
100

101
### Custom Directory Operations
102

103
Advanced directory handling with custom iteration logic and lifecycle management.
104

105
```typescript { .api }
106
import { 
107
  opendir, CustomDir,
108
  type CustomDirOptions, type FakeFS, type PortablePath,
109
  type DirentNoPath, type Filename
110
} from '@yarnpkg/fslib';
111

112
/**
113
 * Custom directory implementation with configurable iteration behavior
114
 */
115
class CustomDir<P extends Path> implements Dir<P> {
116
  constructor(
117
    public readonly path: P,
118
    private readonly nextDirent: () => DirentNoPath | null,
119
    private readonly opts?: CustomDirOptions
120
  );
121
  
122
  // Directory entry reading
123
  read(): Promise<DirentNoPath | null>;
124
  read(cb: (err: NodeJS.ErrnoException | null, dirent: DirentNoPath | null) => void): void;
125
  readSync(): DirentNoPath | null;
126
  
127
  // Directory closing
128
  close(): Promise<void>;
129
  close(cb: NoParamCallback): void;
130
  closeSync(): void;
131
  
132
  // Async iterator support
133
  [Symbol.asyncIterator](): AsyncIterableIterator<DirentNoPath>;
134
}
135

136
// Options for custom directory behavior
137
type CustomDirOptions = {
138
  onClose?: () => void;
139
};
140

141
/**
142
 * Creates a custom directory from a list of entries
143
 */
144
function opendir<P extends Path>(
145
  fakeFs: FakeFS<P>, 
146
  path: P, 
147
  entries: Array<Filename>, 
148
  opts?: CustomDirOptions
149
): CustomDir<P>;
150

151
// Usage example
152
const entries = ['file1.txt', 'file2.txt', 'subdir'] as Filename[];
153
const customDir = opendir(xfs, '/some/path' as PortablePath, entries, {
154
  onClose: () => console.log('Directory closed')
155
});
156

157
// Use async iteration
158
for await (const entry of customDir) {
159
  console.log(`Found: ${entry.name}, isFile: ${entry.isFile()}`);
160
}
161
```
162

163
// Open directory with custom implementation
164
const customDir = await opendir(xfs, '/path/to/directory' as PortablePath, {
165
  bufferSize: 64
166
});
167

168
// Iterate through directory entries
169
for await (const entry of customDir) {
170
  console.log(`${entry.name} (${entry.isDirectory() ? 'dir' : 'file'})`);
171
}
172

173
await customDir.close();
174
```
175
176
#### Advanced Directory Processing
177

178
```typescript { .api }
179
import { opendir, CustomDir, xfs, ppath, type PortablePath } from '@yarnpkg/fslib';
180

181
// Process directory with custom filters and transformations
182
async function processDirectoryWithFilter(
183
  dirPath: PortablePath,
184
  filter: (name: string, isDirectory: boolean) => boolean,
185
  processor: (path: PortablePath, isDirectory: boolean) => Promise<void>
186
): Promise<void> {
187
  const dir = await opendir(xfs, dirPath);
188
  
189
  try {
190
    for await (const entry of dir) {
191
      if (filter(entry.name, entry.isDirectory())) {
192
        const fullPath = ppath.join(dirPath, entry.name);
193
        await processor(fullPath, entry.isDirectory());
194
      }
195
    }
196
  } finally {
197
    await dir.close();
198
  }
199
}
200

201
// Usage: process only TypeScript files
202
await processDirectoryWithFilter(
203
  '/project/src' as PortablePath,
204
  (name, isDir) => isDir || name.endsWith('.ts') || name.endsWith('.tsx'),
205
  async (path, isDir) => {
206
    if (isDir) {
207
      console.log(`Processing directory: ${path}`);
208
      await processDirectoryWithFilter(path, filter, processor); // Recursive
209
    } else {
210
      console.log(`Processing file: ${path}`);
211
      const content = await xfs.readFilePromise(path, 'utf8');
212
      // Process TypeScript content...
213
    }
214
  }
215
);
216
```
217

218
### File Watching Operations
219

220
Advanced file and directory watching with customizable behavior.
221

222
```typescript { .api }
223
import { 
224
  watchFile, unwatchFile, unwatchAllFiles,
225
  type WatchFileOptions, type WatchFileCallback,
226
  type StatWatcher, type FakeFS, type PortablePath 
227
} from '@yarnpkg/fslib';
228

229
// Watch file for changes with custom options
230
const watchOptions: WatchFileOptions = {
231
  persistent: true,
232
  interval: 1000, // Check every second
233
  bigint: false
234
};
235

236
const callback: WatchFileCallback = (current, previous) => {
237
  if (current.mtimeMs !== previous.mtimeMs) {
238
    console.log(`File modified: ${current.mtime} (was ${previous.mtime})`);
239
  }
240
  
241
  if (current.size !== previous.size) {
242
    console.log(`File size changed: ${current.size} bytes (was ${previous.size})`);
243
  }
244
};
245

246
// Start watching file
247
const watcher: StatWatcher = watchFile(xfs, '/path/to/file.txt' as PortablePath, watchOptions, callback);
248

249
// Control watcher behavior
250
watcher.ref();   // Keep process alive while watching
251
watcher.unref(); // Allow process to exit even with active watcher
252

253
// Stop watching specific file
254
unwatchFile(xfs, '/path/to/file.txt' as PortablePath, callback);
255

256
// Stop watching all files
257
unwatchAllFiles(xfs);
258
```
259

260
#### Advanced File Watching Patterns
261

262
```typescript { .api }
263
import { watchFile, unwatchFile, xfs, type PortablePath } from '@yarnpkg/fslib';
264

265
// File watcher manager for multiple files
266
class FileWatchManager {
267
  private watchers = new Map<string, StatWatcher>();
268
  private callbacks = new Map<string, WatchFileCallback>();
269
  
270
  watchFile(path: PortablePath, callback: (path: PortablePath) => void): void {
271
    const watchCallback: WatchFileCallback = (current, previous) => {
272
      if (current.mtimeMs !== previous.mtimeMs) {
273
        callback(path);
274
      }
275
    };
276
    
277
    this.callbacks.set(path, watchCallback);
278
    this.watchers.set(path, watchFile(xfs, path, { persistent: false }, watchCallback));
279
  }
280
  
281
  unwatchFile(path: PortablePath): void {
282
    const callback = this.callbacks.get(path);
283
    if (callback) {
284
      unwatchFile(xfs, path, callback);
285
      this.callbacks.delete(path);
286
      this.watchers.delete(path);
287
    }
288
  }
289
  
290
  unwatchAll(): void {
291
    for (const [path] of this.watchers) {
292
      this.unwatchFile(path as PortablePath);
293
    }
294
  }
295
}
296

297
// Usage
298
const watchManager = new FileWatchManager();
299

300
// Watch configuration files
301
watchManager.watchFile('/project/.yarnrc.yml' as PortablePath, (path) => {
302
  console.log(`Configuration changed: ${path}`);
303
  // Reload configuration...
304
});
305

306
watchManager.watchFile('/project/package.json' as PortablePath, (path) => {
307
  console.log(`Package manifest changed: ${path}`);
308
  // Reload dependencies...
309
});
310

311
// Clean up on exit
312
process.on('exit', () => {
313
  watchManager.unwatchAll();
314
});
315
```
316

317
## Filesystem Patching
318

319
Integration with Node.js fs module to replace or extend filesystem operations.
320

321
### Patching Node.js fs Module
322

323
```typescript { .api }
324
import { patchFs, extendFs, type FakeFS, type NativePath } from '@yarnpkg/fslib';
325
import * as fs from 'fs';
326

327
// Replace Node.js fs module with custom filesystem
328
const customFs: FakeFS<NativePath> = new CustomFileSystemImplementation();
329
const patchedFs = patchFs(fs, customFs);
330

331
// Now all fs operations go through the custom filesystem
332
patchedFs.readFile('/file.txt', 'utf8', (err, data) => {
333
  // This uses the custom filesystem implementation
334
});
335

336
// Extend fs module with additional functionality
337
const extendedFs = extendFs(fs, customFs);
338

339
// Extended fs has both original and custom functionality
340
```
341

342
### File Handle Implementation
343

344
```typescript { .api }
345
import { type FileHandle, type Path } from '@yarnpkg/fslib';
346

347
// File handle for patched filesystem operations
348
interface FileHandle<P extends Path> {
349
  fd: number;
350
  path: P;
351
  
352
  // File operations
353
  readFile(options?: { encoding?: BufferEncoding }): Promise<Buffer | string>;
354
  writeFile(data: string | Buffer, options?: WriteFileOptions): Promise<void>;
355
  appendFile(data: string | Buffer, options?: WriteFileOptions): Promise<void>;
356
  
357
  // Metadata operations
358
  stat(options?: StatOptions): Promise<Stats | BigIntStats>;
359
  chmod(mode: number): Promise<void>;
360
  chown(uid: number, gid: number): Promise<void>;
361
  
362
  // Stream operations
363
  createReadStream(options?: CreateReadStreamOptions): fs.ReadStream;
364
  createWriteStream(options?: CreateWriteStreamOptions): fs.WriteStream;
365
  
366
  // Handle management
367
  close(): Promise<void>;
368
}
369
```
370

371
#### Advanced Patching Patterns
372

373
```typescript { .api }
374
import { patchFs, VirtualFS, NodeFS, type NativePath } from '@yarnpkg/fslib';
375
import * as fs from 'fs';
376

377
// Create a hybrid filesystem that uses virtual fs for temp files
378
class HybridFS extends NodeFS {
379
  private virtualFs = new VirtualFS();
380
  private tempPrefix = '/tmp/virtual-';
381
  
382
  async readFilePromise(p: NativePath, encoding?: BufferEncoding): Promise<Buffer | string> {
383
    if (p.startsWith(this.tempPrefix)) {
384
      // Use virtual filesystem for temp files
385
      return this.virtualFs.readFilePromise(p as any, encoding);
386
    } else {
387
      // Use real filesystem for regular files
388
      return super.readFilePromise(p, encoding);
389
    }
390
  }
391
  
392
  async writeFilePromise(p: NativePath, content: string | Buffer): Promise<void> {
393
    if (p.startsWith(this.tempPrefix)) {
394
      return this.virtualFs.writeFilePromise(p as any, content);
395
    } else {
396
      return super.writeFilePromise(p, content);
397
    }
398
  }
399
  
400
  // Override other methods as needed...
401
}
402

403
// Patch fs module with hybrid filesystem
404
const hybridFs = new HybridFS();
405
const patchedFs = patchFs(fs, hybridFs);
406

407
// Now temp files go to memory, regular files to disk
408
await patchedFs.promises.writeFile('/tmp/virtual-cache.json', '{}');
409
await patchedFs.promises.writeFile('/real/file.txt', 'content');
410
```
411

412
## Extended Filesystem (XFS)
413

414
Enhanced filesystem with temporary file management and additional utilities.
415

416
### XFS Type Definition
417

418
```typescript { .api }
419
import { type XFS, type NodeFS, type PortablePath } from '@yarnpkg/fslib';
420

421
// Extended filesystem interface
422
interface XFS extends NodeFS {
423
  // Temporary file operations
424
  detachTemp(p: PortablePath): void;
425
  mktempSync(): PortablePath;
426
  mktempSync<T>(cb: (p: PortablePath) => T): T;
427
  mktempPromise(): Promise<PortablePath>;
428
  mktempPromise<T>(cb: (p: PortablePath) => Promise<T>): Promise<T>;
429
  rmtempPromise(): Promise<void>;
430
  rmtempSync(): void;
431
}
432
```
433

434
### Using the XFS Instance
435

436
```typescript { .api }
437
import { xfs, ppath, type PortablePath } from '@yarnpkg/fslib';
438

439
// Create temporary directories
440
const tempDir1 = await xfs.mktempPromise();
441
console.log(`Created temp directory: ${tempDir1}`);
442

443
// Create temp directory and use it with callback
444
const result = await xfs.mktempPromise(async (tempDir) => {
445
  const configFile = ppath.join(tempDir, 'config.json');
446
  await xfs.writeFilePromise(configFile, JSON.stringify({ temp: true }));
447
  
448
  const content = await xfs.readFilePromise(configFile, 'utf8');
449
  return JSON.parse(content);
450
});
451
console.log('Temp operation result:', result);
452

453
// Synchronous temporary operations
454
const syncTempDir = xfs.mktempSync();
455
const syncResult = xfs.mktempSync((tempDir) => {
456
  const filePath = ppath.join(tempDir, 'sync-file.txt');
457
  xfs.writeFileSync(filePath, 'synchronous content');
458
  return xfs.readFileSync(filePath, 'utf8');
459
});
460

461
// Clean up all temporary directories
462
await xfs.rmtempPromise();
463
// Or synchronously: xfs.rmtempSync();
464
```
465

466
### Advanced XFS Usage Patterns
467

468
```typescript { .api }
469
import { xfs, ppath, type PortablePath } from '@yarnpkg/fslib';
470

471
// Temporary workspace for complex operations
472
async function performComplexOperation(): Promise<string> {
473
  return await xfs.mktempPromise(async (workspace) => {
474
    // Create temporary project structure
475
    const srcDir = ppath.join(workspace, 'src');
476
    const buildDir = ppath.join(workspace, 'build');
477
    
478
    await xfs.mkdirPromise(srcDir);
479
    await xfs.mkdirPromise(buildDir);
480
    
481
    // Process files in temporary space
482
    await xfs.writeFilePromise(
483
      ppath.join(srcDir, 'input.txt'),
484
      'input data'
485
    );
486
    
487
    // Simulate processing
488
    const inputContent = await xfs.readFilePromise(
489
      ppath.join(srcDir, 'input.txt'),
490
      'utf8'
491
    );
492
    
493
    const processedContent = inputContent.toUpperCase();
494
    
495
    await xfs.writeFilePromise(
496
      ppath.join(buildDir, 'output.txt'),
497
      processedContent
498
    );
499
    
500
    return await xfs.readFilePromise(
501
      ppath.join(buildDir, 'output.txt'),
502
      'utf8'
503
    );
504
  });
505
  // Temporary workspace is automatically cleaned up
506
}
507

508
// Detach temporary directories to prevent automatic cleanup
509
async function createPersistentTemp(): Promise<PortablePath> {
510
  const tempDir = await xfs.mktempPromise();
511
  
512
  // Setup temporary directory with important data
513
  await xfs.writeFilePromise(
514
    ppath.join(tempDir, 'important.txt'),
515
    'This should not be auto-deleted'
516
  );
517
  
518
  // Detach from auto-cleanup
519
  xfs.detachTemp(tempDir);
520
  
521
  return tempDir; // Caller is responsible for cleanup
522
}
523

524
// Batch operations with temporary files
525
async function processBatch(items: string[]): Promise<string[]> {
526
  const results: string[] = [];
527
  
528
  // Process each item in its own temporary space
529
  for (const item of items) {
530
    const result = await xfs.mktempPromise(async (tempDir) => {
531
      const inputFile = ppath.join(tempDir, 'input.json');
532
      const outputFile = ppath.join(tempDir, 'output.json');
533
      
534
      await xfs.writeFilePromise(inputFile, JSON.stringify({ data: item }));
535
      
536
      // Simulate processing
537
      const input = JSON.parse(await xfs.readFilePromise(inputFile, 'utf8'));
538
      const output = { processed: input.data.toUpperCase(), timestamp: Date.now() };
539
      
540
      await xfs.writeFilePromise(outputFile, JSON.stringify(output));
541
      
542
      return JSON.parse(await xfs.readFilePromise(outputFile, 'utf8'));
543
    });
544
    
545
    results.push(result.processed);
546
  }
547
  
548
  return results;
549
}
550
```
551

552
## Integration and Composition
553

554
### Combining Advanced Features
555

556
```typescript { .api }
557
import { 
558
  xfs, watchFile, opendir, patchFs, 
559
  ppath, type PortablePath 
560
} from '@yarnpkg/fslib';
561
import * as fs from 'fs';
562

563
// Advanced file processing system
564
class FileProcessor {
565
  private watchers = new Map<string, StatWatcher>();
566
  
567
  async processProjectDirectory(projectPath: PortablePath): Promise<void> {
568
    // Setup file watching for the project
569
    await this.setupWatching(projectPath);
570
    
571
    // Process files in temporary workspace
572
    await xfs.mktempPromise(async (workspace) => {
573
      // Copy project to workspace for processing
574
      await this.copyProject(projectPath, workspace);
575
      
576
      // Process files with custom directory iteration
577
      await this.processFiles(workspace);
578
      
579
      // Copy results back if needed
580
      const outputDir = ppath.join(projectPath, 'dist');
581
      await this.copyResults(workspace, outputDir);
582
    });
583
  }
584
  
585
  private async setupWatching(projectPath: PortablePath): Promise<void> {
586
    const srcDir = ppath.join(projectPath, 'src');
587
    const dir = await opendir(xfs, srcDir);
588
    
589
    for await (const entry of dir) {
590
      if (entry.isFile() && (entry.name.endsWith('.ts') || entry.name.endsWith('.tsx'))) {
591
        const filePath = ppath.join(srcDir, entry.name);
592
        this.watchers.set(filePath, watchFile(xfs, filePath, {}, () => {
593
          console.log(`Source file changed: ${entry.name}`);
594
          // Trigger reprocessing...
595
        }));
596
      }
597
    }
598
    
599
    await dir.close();
600
  }
601
  
602
  private async copyProject(source: PortablePath, target: PortablePath): Promise<void> {
603
    const dir = await opendir(xfs, source);
604
    await xfs.mkdirPromise(target, { recursive: true });
605
    
606
    for await (const entry of dir) {
607
      const srcPath = ppath.join(source, entry.name);
608
      const dstPath = ppath.join(target, entry.name);
609
      
610
      if (entry.isDirectory()) {
611
        await this.copyProject(srcPath, dstPath); // Recursive
612
      } else if (entry.isFile()) {
613
        await xfs.copyFilePromise(srcPath, dstPath);
614
      }
615
    }
616
    
617
    await dir.close();
618
  }
619
  
620
  private async processFiles(workspace: PortablePath): Promise<void> {
621
    // Use patched fs for processing if needed
622
    const customFs = new ProcessingFileSystem();
623
    const patchedFs = patchFs(fs, customFs);
624
    
625
    // Process files using patched filesystem...
626
  }
627
  
628
  cleanup(): void {
629
    for (const watcher of this.watchers.values()) {
630
      // Cleanup watchers
631
    }
632
    this.watchers.clear();
633
  }
634
}
635
```
636

637
### Error Handling in Advanced Operations
638

639
```typescript { .api }
640
import { 
641
  xfs, errors, watchFile, 
642
  type PortablePath, type WatchFileCallback 
643
} from '@yarnpkg/fslib';
644

645
// Robust advanced operations with comprehensive error handling
646
class RobustProcessor {
647
  async safeProcessWithTemp<T>(
648
    operation: (tempDir: PortablePath) => Promise<T>
649
  ): Promise<T> {
650
    try {
651
      return await xfs.mktempPromise(async (tempDir) => {
652
        try {
653
          return await operation(tempDir);
654
        } catch (error) {
655
          // Log error with context
656
          console.error(`Operation failed in temp directory ${tempDir}:`, error);
657
          throw error;
658
        }
659
      });
660
    } catch (error) {
661
      if (error.code === 'ENOSPC') {
662
        throw errors.ENOSPC('Insufficient disk space for temporary operations');
663
      } else if (error.code === 'EACCES') {
664
        throw errors.EACCES('Permission denied for temporary directory operations');
665
      } else {
666
        throw error;
667
      }
668
    }
669
  }
670
  
671
  async safeWatch(path: PortablePath, callback: WatchFileCallback): Promise<void> {
672
    try {
673
      const stats = await xfs.statPromise(path);
674
      if (!stats.isFile()) {
675
        throw errors.EISDIR(`Cannot watch non-file: ${path}`);
676
      }
677
      
678
      watchFile(xfs, path, { persistent: false }, (current, previous) => {
679
        try {
680
          callback(current, previous);
681
        } catch (error) {
682
          console.error(`Watch callback failed for ${path}:`, error);
683
        }
684
      });
685
      
686
    } catch (error) {
687
      if (error.code === 'ENOENT') {
688
        throw errors.ENOENT(`Cannot watch non-existent file: ${path}`);
689
      } else {
690
        throw error;
691
      }
692
    }
693
  }
694
}
695
```
696

697
## Related Documentation
698

699
- [Filesystem Abstractions](./filesystem-abstractions.md) - Base classes these features build upon
700
- [Filesystem Implementations](./filesystem-implementations.md) - Concrete implementations using these features
701
- [Path Handling](./path-handling.md) - Path utilities used in advanced operations
702
- [Constants and Utilities](./constants-and-utilities.md) - Error handling and constants used throughout