CtrlK
BlogDocsLog inGet started
Tessl Logo

tessl/npm-yarnpkg--fslib

A TypeScript library abstracting the Node filesystem APIs with virtual filesystems and cross-platform path handling

Overview
Eval results
Files

advanced-features.mddocs/

Advanced Features

Advanced filesystem operations, algorithms, patching capabilities, and the extended filesystem (XFS) that provide sophisticated functionality beyond basic file operations.

Overview

The advanced features of @yarnpkg/fslib include:

  • Algorithm Utilities: Copy operations, directory handling, and file watching
  • Filesystem Patching: Integration with Node.js fs module
  • Extended Filesystem (XFS): Enhanced filesystem with temporary file management
  • Custom Directory Operations: Advanced directory iteration and management

Algorithm Utilities

Copy Operations and Strategies

Advanced copy operations with configurable link handling strategies.

import { 
  setupCopyIndex, 
  type LinkStrategy, 
  type HardlinkFromIndexStrategy,
  type PortablePath,
  type FakeFS
} from '@yarnpkg/fslib';

/**
 * Sets up a copy index for efficient hardlink handling during copy operations
 * Creates the index directory structure with hex-based subdirectories
 */
function setupCopyIndex<P extends Path>(
  destinationFs: FakeFS<P>, 
  linkStrategy: Pick<HardlinkFromIndexStrategy<P>, 'indexPath'>
): Promise<P>;

// Link strategy types
type HardlinkFromIndexStrategy<P> = {
  type: 'HardlinkFromIndex';
  indexPath: P;
  autoRepair?: boolean;
  readOnly?: boolean;
};

type LinkStrategy<P> = HardlinkFromIndexStrategy<P>;

// Usage example
import { xfs, ppath } from '@yarnpkg/fslib';

const indexPath = ppath.join('/tmp' as PortablePath, 'copy-index');
await setupCopyIndex(xfs, { indexPath });

Advanced Copy Operations

import { setupCopyIndex, xfs, ppath, type PortablePath } from '@yarnpkg/fslib';

// Setup efficient copy operations with hardlink tracking
async function performLargeCopy(
  sourceDir: PortablePath,
  targetDir: PortablePath
): Promise<void> {
  // Setup copy index for efficient hardlink detection
  const indexPath = ppath.join(targetDir, '.copy-index' as PortablePath);
  await setupCopyIndex(indexPath);
  
  // Perform recursive copy with hardlink preservation
  await copyDirectoryRecursive(sourceDir, targetDir, {
    preserveHardlinks: true,
    indexPath
  });
}

// Custom copy implementation with progress tracking
async function copyDirectoryRecursive(
  source: PortablePath,
  target: PortablePath,
  options: { preserveHardlinks?: boolean; indexPath?: PortablePath } = {}
): Promise<void> {
  await xfs.mkdirPromise(target, { recursive: true });
  
  const entries = await xfs.readdirPromise(source, { withFileTypes: true });
  
  for (const entry of entries) {
    const srcPath = ppath.join(source, entry.name);
    const dstPath = ppath.join(target, entry.name);
    
    if (entry.isDirectory()) {
      await copyDirectoryRecursive(srcPath, dstPath, options);
    } else if (entry.isFile()) {
      // Use copy with hardlink detection if available
      await xfs.copyFilePromise(srcPath, dstPath);
    } else if (entry.isSymbolicLink()) {
      const linkTarget = await xfs.readlinkPromise(srcPath);
      await xfs.symlinkPromise(linkTarget, dstPath);
    }
  }
}

Custom Directory Operations

Advanced directory handling with custom iteration logic and lifecycle management.

import { 
  opendir, CustomDir,
  type CustomDirOptions, type FakeFS, type PortablePath,
  type DirentNoPath, type Filename
} from '@yarnpkg/fslib';

/**
 * Custom directory implementation with configurable iteration behavior
 */
class CustomDir<P extends Path> implements Dir<P> {
  constructor(
    public readonly path: P,
    private readonly nextDirent: () => DirentNoPath | null,
    private readonly opts?: CustomDirOptions
  );
  
  // Directory entry reading
  read(): Promise<DirentNoPath | null>;
  read(cb: (err: NodeJS.ErrnoException | null, dirent: DirentNoPath | null) => void): void;
  readSync(): DirentNoPath | null;
  
  // Directory closing
  close(): Promise<void>;
  close(cb: NoParamCallback): void;
  closeSync(): void;
  
  // Async iterator support
  [Symbol.asyncIterator](): AsyncIterableIterator<DirentNoPath>;
}

// Options for custom directory behavior
type CustomDirOptions = {
  onClose?: () => void;
};

/**
 * Creates a custom directory from a list of entries
 */
function opendir<P extends Path>(
  fakeFs: FakeFS<P>, 
  path: P, 
  entries: Array<Filename>, 
  opts?: CustomDirOptions
): CustomDir<P>;

// Usage example
const entries = ['file1.txt', 'file2.txt', 'subdir'] as Filename[];
const customDir = opendir(xfs, '/some/path' as PortablePath, entries, {
  onClose: () => console.log('Directory closed')
});

// Use async iteration
for await (const entry of customDir) {
  console.log(`Found: ${entry.name}, isFile: ${entry.isFile()}`);
}

// Open directory with custom implementation const customDir = await opendir(xfs, '/path/to/directory' as PortablePath, { bufferSize: 64 });

// Iterate through directory entries for await (const entry of customDir) { console.log(${entry.name} (${entry.isDirectory() ? 'dir' : 'file'})); }

await customDir.close();

#### Advanced Directory Processing

```typescript { .api }
import { opendir, CustomDir, xfs, ppath, type PortablePath } from '@yarnpkg/fslib';

// Process directory with custom filters and transformations
async function processDirectoryWithFilter(
  dirPath: PortablePath,
  filter: (name: string, isDirectory: boolean) => boolean,
  processor: (path: PortablePath, isDirectory: boolean) => Promise<void>
): Promise<void> {
  const dir = await opendir(xfs, dirPath);
  
  try {
    for await (const entry of dir) {
      if (filter(entry.name, entry.isDirectory())) {
        const fullPath = ppath.join(dirPath, entry.name);
        await processor(fullPath, entry.isDirectory());
      }
    }
  } finally {
    await dir.close();
  }
}

// Usage: process only TypeScript files
await processDirectoryWithFilter(
  '/project/src' as PortablePath,
  (name, isDir) => isDir || name.endsWith('.ts') || name.endsWith('.tsx'),
  async (path, isDir) => {
    if (isDir) {
      console.log(`Processing directory: ${path}`);
      await processDirectoryWithFilter(path, filter, processor); // Recursive
    } else {
      console.log(`Processing file: ${path}`);
      const content = await xfs.readFilePromise(path, 'utf8');
      // Process TypeScript content...
    }
  }
);

File Watching Operations

Advanced file and directory watching with customizable behavior.

import { 
  watchFile, unwatchFile, unwatchAllFiles,
  type WatchFileOptions, type WatchFileCallback,
  type StatWatcher, type FakeFS, type PortablePath 
} from '@yarnpkg/fslib';

// Watch file for changes with custom options
const watchOptions: WatchFileOptions = {
  persistent: true,
  interval: 1000, // Check every second
  bigint: false
};

const callback: WatchFileCallback = (current, previous) => {
  if (current.mtimeMs !== previous.mtimeMs) {
    console.log(`File modified: ${current.mtime} (was ${previous.mtime})`);
  }
  
  if (current.size !== previous.size) {
    console.log(`File size changed: ${current.size} bytes (was ${previous.size})`);
  }
};

// Start watching file
const watcher: StatWatcher = watchFile(xfs, '/path/to/file.txt' as PortablePath, watchOptions, callback);

// Control watcher behavior
watcher.ref();   // Keep process alive while watching
watcher.unref(); // Allow process to exit even with active watcher

// Stop watching specific file
unwatchFile(xfs, '/path/to/file.txt' as PortablePath, callback);

// Stop watching all files
unwatchAllFiles(xfs);

Advanced File Watching Patterns

import { watchFile, unwatchFile, xfs, type PortablePath } from '@yarnpkg/fslib';

// File watcher manager for multiple files
class FileWatchManager {
  private watchers = new Map<string, StatWatcher>();
  private callbacks = new Map<string, WatchFileCallback>();
  
  watchFile(path: PortablePath, callback: (path: PortablePath) => void): void {
    const watchCallback: WatchFileCallback = (current, previous) => {
      if (current.mtimeMs !== previous.mtimeMs) {
        callback(path);
      }
    };
    
    this.callbacks.set(path, watchCallback);
    this.watchers.set(path, watchFile(xfs, path, { persistent: false }, watchCallback));
  }
  
  unwatchFile(path: PortablePath): void {
    const callback = this.callbacks.get(path);
    if (callback) {
      unwatchFile(xfs, path, callback);
      this.callbacks.delete(path);
      this.watchers.delete(path);
    }
  }
  
  unwatchAll(): void {
    for (const [path] of this.watchers) {
      this.unwatchFile(path as PortablePath);
    }
  }
}

// Usage
const watchManager = new FileWatchManager();

// Watch configuration files
watchManager.watchFile('/project/.yarnrc.yml' as PortablePath, (path) => {
  console.log(`Configuration changed: ${path}`);
  // Reload configuration...
});

watchManager.watchFile('/project/package.json' as PortablePath, (path) => {
  console.log(`Package manifest changed: ${path}`);
  // Reload dependencies...
});

// Clean up on exit
process.on('exit', () => {
  watchManager.unwatchAll();
});

Filesystem Patching

Integration with Node.js fs module to replace or extend filesystem operations.

Patching Node.js fs Module

import { patchFs, extendFs, type FakeFS, type NativePath } from '@yarnpkg/fslib';
import * as fs from 'fs';

// Replace Node.js fs module with custom filesystem
const customFs: FakeFS<NativePath> = new CustomFileSystemImplementation();
const patchedFs = patchFs(fs, customFs);

// Now all fs operations go through the custom filesystem
patchedFs.readFile('/file.txt', 'utf8', (err, data) => {
  // This uses the custom filesystem implementation
});

// Extend fs module with additional functionality
const extendedFs = extendFs(fs, customFs);

// Extended fs has both original and custom functionality

File Handle Implementation

import { type FileHandle, type Path } from '@yarnpkg/fslib';

// File handle for patched filesystem operations
interface FileHandle<P extends Path> {
  fd: number;
  path: P;
  
  // File operations
  readFile(options?: { encoding?: BufferEncoding }): Promise<Buffer | string>;
  writeFile(data: string | Buffer, options?: WriteFileOptions): Promise<void>;
  appendFile(data: string | Buffer, options?: WriteFileOptions): Promise<void>;
  
  // Metadata operations
  stat(options?: StatOptions): Promise<Stats | BigIntStats>;
  chmod(mode: number): Promise<void>;
  chown(uid: number, gid: number): Promise<void>;
  
  // Stream operations
  createReadStream(options?: CreateReadStreamOptions): fs.ReadStream;
  createWriteStream(options?: CreateWriteStreamOptions): fs.WriteStream;
  
  // Handle management
  close(): Promise<void>;
}

Advanced Patching Patterns

import { patchFs, VirtualFS, NodeFS, type NativePath } from '@yarnpkg/fslib';
import * as fs from 'fs';

// Create a hybrid filesystem that uses virtual fs for temp files
class HybridFS extends NodeFS {
  private virtualFs = new VirtualFS();
  private tempPrefix = '/tmp/virtual-';
  
  async readFilePromise(p: NativePath, encoding?: BufferEncoding): Promise<Buffer | string> {
    if (p.startsWith(this.tempPrefix)) {
      // Use virtual filesystem for temp files
      return this.virtualFs.readFilePromise(p as any, encoding);
    } else {
      // Use real filesystem for regular files
      return super.readFilePromise(p, encoding);
    }
  }
  
  async writeFilePromise(p: NativePath, content: string | Buffer): Promise<void> {
    if (p.startsWith(this.tempPrefix)) {
      return this.virtualFs.writeFilePromise(p as any, content);
    } else {
      return super.writeFilePromise(p, content);
    }
  }
  
  // Override other methods as needed...
}

// Patch fs module with hybrid filesystem
const hybridFs = new HybridFS();
const patchedFs = patchFs(fs, hybridFs);

// Now temp files go to memory, regular files to disk
await patchedFs.promises.writeFile('/tmp/virtual-cache.json', '{}');
await patchedFs.promises.writeFile('/real/file.txt', 'content');

Extended Filesystem (XFS)

Enhanced filesystem with temporary file management and additional utilities.

XFS Type Definition

import { type XFS, type NodeFS, type PortablePath } from '@yarnpkg/fslib';

// Extended filesystem interface
interface XFS extends NodeFS {
  // Temporary file operations
  detachTemp(p: PortablePath): void;
  mktempSync(): PortablePath;
  mktempSync<T>(cb: (p: PortablePath) => T): T;
  mktempPromise(): Promise<PortablePath>;
  mktempPromise<T>(cb: (p: PortablePath) => Promise<T>): Promise<T>;
  rmtempPromise(): Promise<void>;
  rmtempSync(): void;
}

Using the XFS Instance

import { xfs, ppath, type PortablePath } from '@yarnpkg/fslib';

// Create temporary directories
const tempDir1 = await xfs.mktempPromise();
console.log(`Created temp directory: ${tempDir1}`);

// Create temp directory and use it with callback
const result = await xfs.mktempPromise(async (tempDir) => {
  const configFile = ppath.join(tempDir, 'config.json');
  await xfs.writeFilePromise(configFile, JSON.stringify({ temp: true }));
  
  const content = await xfs.readFilePromise(configFile, 'utf8');
  return JSON.parse(content);
});
console.log('Temp operation result:', result);

// Synchronous temporary operations
const syncTempDir = xfs.mktempSync();
const syncResult = xfs.mktempSync((tempDir) => {
  const filePath = ppath.join(tempDir, 'sync-file.txt');
  xfs.writeFileSync(filePath, 'synchronous content');
  return xfs.readFileSync(filePath, 'utf8');
});

// Clean up all temporary directories
await xfs.rmtempPromise();
// Or synchronously: xfs.rmtempSync();

Advanced XFS Usage Patterns

import { xfs, ppath, type PortablePath } from '@yarnpkg/fslib';

// Temporary workspace for complex operations
async function performComplexOperation(): Promise<string> {
  return await xfs.mktempPromise(async (workspace) => {
    // Create temporary project structure
    const srcDir = ppath.join(workspace, 'src');
    const buildDir = ppath.join(workspace, 'build');
    
    await xfs.mkdirPromise(srcDir);
    await xfs.mkdirPromise(buildDir);
    
    // Process files in temporary space
    await xfs.writeFilePromise(
      ppath.join(srcDir, 'input.txt'),
      'input data'
    );
    
    // Simulate processing
    const inputContent = await xfs.readFilePromise(
      ppath.join(srcDir, 'input.txt'),
      'utf8'
    );
    
    const processedContent = inputContent.toUpperCase();
    
    await xfs.writeFilePromise(
      ppath.join(buildDir, 'output.txt'),
      processedContent
    );
    
    return await xfs.readFilePromise(
      ppath.join(buildDir, 'output.txt'),
      'utf8'
    );
  });
  // Temporary workspace is automatically cleaned up
}

// Detach temporary directories to prevent automatic cleanup
async function createPersistentTemp(): Promise<PortablePath> {
  const tempDir = await xfs.mktempPromise();
  
  // Setup temporary directory with important data
  await xfs.writeFilePromise(
    ppath.join(tempDir, 'important.txt'),
    'This should not be auto-deleted'
  );
  
  // Detach from auto-cleanup
  xfs.detachTemp(tempDir);
  
  return tempDir; // Caller is responsible for cleanup
}

// Batch operations with temporary files
async function processBatch(items: string[]): Promise<string[]> {
  const results: string[] = [];
  
  // Process each item in its own temporary space
  for (const item of items) {
    const result = await xfs.mktempPromise(async (tempDir) => {
      const inputFile = ppath.join(tempDir, 'input.json');
      const outputFile = ppath.join(tempDir, 'output.json');
      
      await xfs.writeFilePromise(inputFile, JSON.stringify({ data: item }));
      
      // Simulate processing
      const input = JSON.parse(await xfs.readFilePromise(inputFile, 'utf8'));
      const output = { processed: input.data.toUpperCase(), timestamp: Date.now() };
      
      await xfs.writeFilePromise(outputFile, JSON.stringify(output));
      
      return JSON.parse(await xfs.readFilePromise(outputFile, 'utf8'));
    });
    
    results.push(result.processed);
  }
  
  return results;
}

Integration and Composition

Combining Advanced Features

import { 
  xfs, watchFile, opendir, patchFs, 
  ppath, type PortablePath 
} from '@yarnpkg/fslib';
import * as fs from 'fs';

// Advanced file processing system
class FileProcessor {
  private watchers = new Map<string, StatWatcher>();
  
  async processProjectDirectory(projectPath: PortablePath): Promise<void> {
    // Setup file watching for the project
    await this.setupWatching(projectPath);
    
    // Process files in temporary workspace
    await xfs.mktempPromise(async (workspace) => {
      // Copy project to workspace for processing
      await this.copyProject(projectPath, workspace);
      
      // Process files with custom directory iteration
      await this.processFiles(workspace);
      
      // Copy results back if needed
      const outputDir = ppath.join(projectPath, 'dist');
      await this.copyResults(workspace, outputDir);
    });
  }
  
  private async setupWatching(projectPath: PortablePath): Promise<void> {
    const srcDir = ppath.join(projectPath, 'src');
    const dir = await opendir(xfs, srcDir);
    
    for await (const entry of dir) {
      if (entry.isFile() && (entry.name.endsWith('.ts') || entry.name.endsWith('.tsx'))) {
        const filePath = ppath.join(srcDir, entry.name);
        this.watchers.set(filePath, watchFile(xfs, filePath, {}, () => {
          console.log(`Source file changed: ${entry.name}`);
          // Trigger reprocessing...
        }));
      }
    }
    
    await dir.close();
  }
  
  private async copyProject(source: PortablePath, target: PortablePath): Promise<void> {
    const dir = await opendir(xfs, source);
    await xfs.mkdirPromise(target, { recursive: true });
    
    for await (const entry of dir) {
      const srcPath = ppath.join(source, entry.name);
      const dstPath = ppath.join(target, entry.name);
      
      if (entry.isDirectory()) {
        await this.copyProject(srcPath, dstPath); // Recursive
      } else if (entry.isFile()) {
        await xfs.copyFilePromise(srcPath, dstPath);
      }
    }
    
    await dir.close();
  }
  
  private async processFiles(workspace: PortablePath): Promise<void> {
    // Use patched fs for processing if needed
    const customFs = new ProcessingFileSystem();
    const patchedFs = patchFs(fs, customFs);
    
    // Process files using patched filesystem...
  }
  
  cleanup(): void {
    for (const watcher of this.watchers.values()) {
      // Cleanup watchers
    }
    this.watchers.clear();
  }
}

Error Handling in Advanced Operations

import { 
  xfs, errors, watchFile, 
  type PortablePath, type WatchFileCallback 
} from '@yarnpkg/fslib';

// Robust advanced operations with comprehensive error handling
class RobustProcessor {
  async safeProcessWithTemp<T>(
    operation: (tempDir: PortablePath) => Promise<T>
  ): Promise<T> {
    try {
      return await xfs.mktempPromise(async (tempDir) => {
        try {
          return await operation(tempDir);
        } catch (error) {
          // Log error with context
          console.error(`Operation failed in temp directory ${tempDir}:`, error);
          throw error;
        }
      });
    } catch (error) {
      if (error.code === 'ENOSPC') {
        throw errors.ENOSPC('Insufficient disk space for temporary operations');
      } else if (error.code === 'EACCES') {
        throw errors.EACCES('Permission denied for temporary directory operations');
      } else {
        throw error;
      }
    }
  }
  
  async safeWatch(path: PortablePath, callback: WatchFileCallback): Promise<void> {
    try {
      const stats = await xfs.statPromise(path);
      if (!stats.isFile()) {
        throw errors.EISDIR(`Cannot watch non-file: ${path}`);
      }
      
      watchFile(xfs, path, { persistent: false }, (current, previous) => {
        try {
          callback(current, previous);
        } catch (error) {
          console.error(`Watch callback failed for ${path}:`, error);
        }
      });
      
    } catch (error) {
      if (error.code === 'ENOENT') {
        throw errors.ENOENT(`Cannot watch non-existent file: ${path}`);
      } else {
        throw error;
      }
    }
  }
}

Related Documentation

  • Filesystem Abstractions - Base classes these features build upon
  • Filesystem Implementations - Concrete implementations using these features
  • Path Handling - Path utilities used in advanced operations
  • Constants and Utilities - Error handling and constants used throughout

Install with Tessl CLI

npx tessl i tessl/npm-yarnpkg--fslib

docs

advanced-features.md

constants-and-utilities.md

filesystem-abstractions.md

filesystem-implementations.md

index.md

path-handling.md

tile.json