0
# Commit and Pipeline Operations
1

2
Pipeline-based commit system for batch file operations with custom transforms and filtering, enabling efficient processing of file changes before writing to disk.
3

4
## Capabilities
5

6
### Commit Files to Disk
7

8
Commit all pending file changes from the in-memory store to the file system with optional transforms and filtering.
9

10
```typescript { .api }
11
/**
12
 * Commit pending file changes to disk
13
 * @param options - Pipeline options or file transform function
14
 * @param transforms - Additional file transform functions to apply
15
 * @returns Promise that resolves when all files are committed
16
 */
17
function commit<EditorFile extends MemFsEditorFile>(
18
  options?: PipelineOptions<EditorFile> | FileTransform<EditorFile>,
19
  ...transforms: FileTransform<EditorFile>[]
20
): Promise<void>;
21

22
interface PipelineOptions<T> {
23
  /** Filter function to select which files to commit */
24
  filter?: (file: T) => boolean;
25
  /** Custom error handling for individual files */
26
  onError?: (error: Error, file: T) => void;
27
  /** Additional pipeline configuration */
28
  [key: string]: any;
29
}
30

31
interface FileTransform<T> {
32
  /** Transform function applied to each file before committing */
33
  (file: T): T | Promise<T>;
34
}
35

36
interface MemFsEditorFile {
37
  path: string;
38
  stat?: { mode?: number } | null;
39
  contents: Buffer | null;
40
  committed?: boolean;
41
  isNew?: boolean;
42
  state?: 'modified' | 'deleted';
43
  stateCleared?: 'modified' | 'deleted';
44
}
45
```
46

47
**Usage Examples:**
48

49
```typescript
50
import { create as createMemFs } from "mem-fs";
51
import { create as createEditor } from "mem-fs-editor";
52

53
const store = createMemFs();
54
const fs = createEditor(store);
55

56
// Make some changes
57
fs.write("package.json", JSON.stringify({ name: "my-app" }, null, 2));
58
fs.copy("src/**/*.js", "dist/");
59
fs.delete("temp/**/*");
60

61
// Simple commit
62
await fs.commit();
63

64
// Commit with file filtering
65
await fs.commit({
66
  filter: (file) => !file.path.includes("node_modules")
67
});
68

69
// Commit with transforms
70
await fs.commit(
71
  (file) => {
72
    // Add timestamp to all files
73
    if (file.contents) {
74
      const header = `// Generated: ${new Date().toISOString()}\n`;
75
      file.contents = Buffer.concat([
76
        Buffer.from(header),
77
        file.contents
78
      ]);
79
    }
80
    return file;
81
  }
82
);
83
```
84

85
### Export File State for Debugging
86

87
Export current file states and metadata for debugging and inspection purposes.
88

89
```typescript { .api }
90
/**
91
 * Export file states for debugging
92
 * @param cwd - Current working directory (default: process.cwd())
93
 * @param filter - Filter pattern or function to select files
94
 * @returns Object mapping file paths to their dump information
95
 */
96
function dump<EditorFile extends MemFsEditorFile>(
97
  cwd?: string,
98
  filter?: string | ((file: EditorFile, cwd: string) => boolean)
99
): Record<string, MemFsEditorFileDump>;
100

101
interface MemFsEditorFileDump {
102
  /** File contents as string or null if binary/empty */
103
  contents: string | null;
104
  /** Current file state (modified, deleted, etc.) */
105
  state?: string;
106
  /** Previously cleared state information */
107
  stateCleared?: string;
108
}
109
```
110

111
**Usage Examples:**
112

113
```typescript
114
// Dump all files
115
const allFiles = fs.dump();
116
console.log("Current files:", Object.keys(allFiles));
117

118
// Dump with custom working directory
119
const srcFiles = fs.dump("/project/src");
120

121
// Dump with string filter (glob pattern)
122
const jsFiles = fs.dump(undefined, "**/*.js");
123

124
// Dump with function filter
125
const modifiedFiles = fs.dump(undefined, (file, cwd) => {
126
  return file.state === 'modified';
127
});
128

129
// Inspect specific file states
130
Object.entries(allFiles).forEach(([path, dump]) => {
131
  console.log(`${path}:`, {
132
    hasContents: dump.contents !== null,
133
    state: dump.state,
134
    stateCleared: dump.stateCleared
135
  });
136
});
137
```
138

139
## Advanced Pipeline Operations
140

141
### File Transform Functions
142

143
```typescript
144
// Content transformation
145
const addLicenseHeader = (file) => {
146
  if (file.path.endsWith('.js') && file.contents) {
147
    const license = `/*
148
 * Copyright (c) 2024 My Company
149
 * Licensed under MIT License
150
 */\n`;
151
    file.contents = Buffer.concat([
152
      Buffer.from(license),
153
      file.contents
154
    ]);
155
  }
156
  return file;
157
};
158

159
// Minification transform
160
const minifyJS = (file) => {
161
  if (file.path.endsWith('.js') && file.contents) {
162
    const minified = file.contents
163
      .toString()
164
      .replace(/\/\*[\s\S]*?\*\//g, '') // Remove comments
165
      .replace(/\s+/g, ' ') // Collapse whitespace
166
      .trim();
167
    file.contents = Buffer.from(minified);
168
  }
169
  return file;
170
};
171

172
// Path transformation
173
const updatePaths = (file) => {
174
  if (file.path.startsWith('temp/')) {
175
    file.path = file.path.replace('temp/', 'dist/');
176
  }
177
  return file;
178
};
179

180
// Apply multiple transforms
181
await fs.commit(addLicenseHeader, minifyJS, updatePaths);
182
```
183

184
### Conditional Commits
185

186
```typescript
187
// Commit only specific file types
188
await fs.commit({
189
  filter: (file) => {
190
    return file.path.match(/\.(js|ts|json)$/) && !file.path.includes('test');
191
  }
192
});
193

194
// Commit with size limits
195
await fs.commit({
196
  filter: (file) => {
197
    return !file.contents || file.contents.length < 1024 * 1024; // < 1MB
198
  }
199
});
200

201
// Commit only new files
202
await fs.commit({
203
  filter: (file) => file.isNew === true
204
});
205

206
// Commit only modified files
207
await fs.commit({
208
  filter: (file) => file.state === 'modified'
209
});
210
```
211

212
### Error Handling in Commits
213

214
```typescript
215
// Commit with error handling
216
await fs.commit({
217
  filter: (file) => file.path.endsWith('.js'),
218
  onError: (error, file) => {
219
    console.error(`Failed to commit ${file.path}:`, error.message);
220
    // Could log to external service, skip file, etc.
221
  }
222
});
223

224
// Robust commit with try-catch
225
try {
226
  await fs.commit(
227
    // Transform that might fail
228
    (file) => {
229
      if (file.path.endsWith('.json') && file.contents) {
230
        // Validate JSON
231
        JSON.parse(file.contents.toString());
232
      }
233
      return file;
234
    }
235
  );
236
} catch (error) {
237
  console.error("Commit failed:", error.message);
238
  
239
  // Inspect what files were problematic
240
  const dump = fs.dump();
241
  const pendingFiles = Object.entries(dump)
242
    .filter(([_, info]) => info.state)
243
    .map(([path]) => path);
244
  
245
  console.log("Files with pending changes:", pendingFiles);
246
}
247
```
248

249
### Stream-based Processing
250

251
```typescript
252
import { createCommitTransform } from "mem-fs-editor/transform";
253

254
// Create custom commit transform
255
const customCommitTransform = createCommitTransform();
256

257
// The transform module provides stream-based processing
258
// for high-performance batch operations
259
```
260

261
## State Management Integration
262

263
```typescript
264
import { 
265
  setFileState, 
266
  isFileNew, 
267
  isFileStateModified,
268
  isFilePending,
269
  setCommittedFile 
270
} from "mem-fs-editor/state";
271

272
// Check file states before committing
273
const dump = fs.dump();
274
Object.entries(dump).forEach(([path, info]) => {
275
  const file = store.get(path);
276
  
277
  console.log(`${path}:`, {
278
    isNew: isFileNew(file),
279
    isModified: isFileStateModified(file), 
280
    isPending: isFilePending(file)
281
  });
282
});
283

284
// Commit with state management
285
await fs.commit({
286
  filter: (file) => isFilePending(file)
287
});
288

289
// Custom state management after commit
290
const commitWithStateTracking = async () => {
291
  const filesToCommit = Object.keys(fs.dump());
292
  
293
  await fs.commit();
294
  
295
  // Mark files as committed using state utilities
296
  filesToCommit.forEach(filepath => {
297
    const file = store.get(filepath);
298
    if (file) {
299
      setCommittedFile(file);
300
    }
301
  });
302
};
303

304
await commitWithStateTracking();
305
```
306

307
## Performance Optimization
308

309
```typescript
310
// Batch commits for better performance
311
const commitInBatches = async (batchSize = 50) => {
312
  const allFiles = Object.keys(fs.dump());
313
  
314
  for (let i = 0; i < allFiles.length; i += batchSize) {
315
    const batch = allFiles.slice(i, i + batchSize);
316
    
317
    await fs.commit({
318
      filter: (file) => batch.includes(file.path)
319
    });
320
    
321
    console.log(`Committed batch ${Math.floor(i / batchSize) + 1}`);
322
  }
323
};
324

325
await commitInBatches();
326

327
// Optimized commit with minimal transforms
328
await fs.commit({
329
  filter: (file) => {
330
    // Quick checks first
331
    return file.path.endsWith('.js') && file.isNew;
332
  }
333
}, (file) => {
334
  // Minimal transform
335
  return file;
336
});
337
```
338

339
## Debugging and Monitoring
340

341
```typescript
342
// Monitor commit progress
343
const monitorCommit = async () => {
344
  const startTime = Date.now();
345
  const initialCount = Object.keys(fs.dump()).length;
346
  
347
  console.log(`Starting commit of ${initialCount} files...`);
348
  
349
  await fs.commit((file) => {
350
    console.log(`Processing: ${file.path}`);
351
    return file;
352
  });
353
  
354
  const endTime = Date.now();
355
  console.log(`Commit completed in ${endTime - startTime}ms`);
356
};
357

358
await monitorCommit();
359

360
// Debug file states during commit
361
const debugCommit = async () => {
362
  const beforeDump = fs.dump();
363
  console.log("Before commit:", Object.keys(beforeDump).length, "files");
364
  
365
  await fs.commit();
366
  
367
  const afterDump = fs.dump();
368
  console.log("After commit:", Object.keys(afterDump).length, "files");
369
  
370
  // Show what changed
371
  const committed = Object.keys(beforeDump).filter(path => 
372
    !afterDump[path] || !afterDump[path].state
373
  );
374
  console.log("Committed files:", committed);
375
};
376

377
await debugCommit();
378
```