CtrlK
BlogDocsLog inGet started
Tessl Logo

tessl/npm-google-cloud--storage

Cloud Storage Client Library for Node.js that provides comprehensive API for managing buckets, files, and metadata with authentication, streaming, and access control.

—

Pending
Quality

Pending

Does it follow best practices?

Impact

Pending

No eval scenarios have been run

SecuritybySnyk

Pending

The risk profile of this skill

Overview
Eval results
Files

transfer-manager.mddocs/

Transfer Manager

The Transfer Manager provides high-level methods for bulk upload and download operations with parallel processing, progress tracking, and error handling for efficient data transfer.

TransferManager Class

class TransferManager {
  constructor(bucket: Bucket);
  
  // Properties
  bucket: Bucket;
  
  // Bulk upload methods
  uploadManyFiles(filePathsOrDirectory: string[] | string, options?: UploadManyFilesOptions): Promise<UploadResponse[]>;
  uploadFileInChunks(filePath: string, options?: UploadFileInChunksOptions): Promise<void>;
  
  // Bulk download methods
  downloadManyFiles(files: File[], options?: DownloadManyFilesOptions): Promise<void>;
  downloadFileInChunks(file: File, options?: DownloadFileInChunksOptions): Promise<void>;
}

Creating Transfer Manager

// Create transfer manager for bucket
const transferManager = new TransferManager(bucket);

// Or import and create
import { TransferManager } from '@google-cloud/storage';
const bucket = storage.bucket('my-bucket');
const transferManager = new TransferManager(bucket);

Bulk Upload Operations

Upload Many Files

uploadManyFiles(filePathsOrDirectory: string[] | string, options?: UploadManyFilesOptions): Promise<UploadResponse[]>

interface UploadManyFilesOptions {
  concurrencyLimit?: number;
  customDestinationBuilder?: (filePath: string) => string;
  skipIfExists?: boolean;
  prefix?: string;
  passthroughOptions?: UploadOptions;
}

type UploadResponse = [File, unknown]; // [file, apiResponse]

// Upload multiple files by path
const filePaths = [
  '/local/documents/file1.pdf',
  '/local/documents/file2.pdf',
  '/local/images/photo1.jpg',
  '/local/images/photo2.jpg'
];

const uploadResponses = await transferManager.uploadManyFiles(filePaths);
uploadResponses.forEach(([file]) => {
  console.log(`Uploaded: ${file.name}`);
});

// Upload entire directory
const uploadResponses = await transferManager.uploadManyFiles('/local/documents');

// Upload with options
const uploadResponses = await transferManager.uploadManyFiles(filePaths, {
  concurrencyLimit: 10,
  skipIfExists: true,
  prefix: 'uploaded/',
  customDestinationBuilder: (filePath) => {
    // Custom logic for destination path
    const fileName = path.basename(filePath);
    const timestamp = new Date().toISOString().replace(/:/g, '-');
    return `uploads/${timestamp}-${fileName}`;
  }
});

// Upload with passthrough options
const uploadResponses = await transferManager.uploadManyFiles(filePaths, {
  concurrencyLimit: 5,
  passthroughOptions: {
    public: true,
    metadata: {
      cacheControl: 'public, max-age=3600'
    },
    resumable: true
  }
});

Upload File in Chunks

uploadFileInChunks(filePath: string, options?: UploadFileInChunksOptions): Promise<void>

interface UploadFileInChunksOptions {
  chunkSizeBytes?: number;
  concurrencyLimit?: number;
  destination?: string;
  encryptionKey?: string | Buffer;
  kmsKeyName?: string;
  maxRetries?: number;
  preconditionOpts?: PreconditionOptions;
  uploadId?: string;
  passthroughOptions?: UploadOptions;
}

// Upload large file in chunks
await transferManager.uploadFileInChunks('/local/large-video.mp4');

// Upload with custom chunk size
await transferManager.uploadFileInChunks('/local/huge-dataset.zip', {
  chunkSizeBytes: 32 * 1024 * 1024, // 32MB chunks
  concurrencyLimit: 8,
  destination: 'datasets/processed-data.zip'
});

// Upload with encryption
await transferManager.uploadFileInChunks('/local/sensitive-data.db', {
  chunkSizeBytes: 16 * 1024 * 1024, // 16MB chunks
  encryptionKey: crypto.randomBytes(32).toString('base64'),
  maxRetries: 5
});

// Upload with KMS encryption
await transferManager.uploadFileInChunks('/local/confidential.pdf', {
  kmsKeyName: 'projects/PROJECT_ID/locations/us/keyRings/ring/cryptoKeys/key',
  destination: 'confidential/document.pdf'
});

Bulk Download Operations

Download Many Files

downloadManyFiles(files: File[], options?: DownloadManyFilesOptions): Promise<void>

interface DownloadManyFilesOptions {
  concurrencyLimit?: number;
  prefix?: string;
  stripPrefix?: string;
  passthroughOptions?: DownloadOptions;
}

// Download multiple files
const files = [
  bucket.file('documents/file1.pdf'),
  bucket.file('documents/file2.pdf'),
  bucket.file('images/photo1.jpg')
];

await transferManager.downloadManyFiles(files, {
  concurrencyLimit: 10,
  prefix: '/local/downloads/'
});

// Get files and download with pattern
const [allFiles] = await bucket.getFiles({ prefix: 'reports/' });
await transferManager.downloadManyFiles(allFiles, {
  concurrencyLimit: 5,
  prefix: '/local/reports/',
  stripPrefix: 'reports/',
  passthroughOptions: {
    validation: false // Skip integrity checking for faster downloads
  }
});

// Download with custom local structure
const [files] = await bucket.getFiles({ prefix: 'backup/2023/' });
await transferManager.downloadManyFiles(files, {
  prefix: '/local/restored/',
  stripPrefix: 'backup/2023/',
  concurrencyLimit: 8
});

Download File in Chunks

downloadFileInChunks(file: File, options?: DownloadFileInChunksOptions): Promise<void>

interface DownloadFileInChunksOptions {
  chunkSizeBytes?: number;
  concurrencyLimit?: number;
  destination?: string;
  validation?: boolean;
  decompress?: boolean;
}

// Download large file in chunks
const largeFile = bucket.file('datasets/large-dataset.zip');
await transferManager.downloadFileInChunks(largeFile, {
  destination: '/local/downloads/dataset.zip'
});

// Download with custom chunk size and concurrency
await transferManager.downloadFileInChunks(largeFile, {
  chunkSizeBytes: 64 * 1024 * 1024, // 64MB chunks
  concurrencyLimit: 12,
  destination: '/local/fast-download/dataset.zip',
  validation: false // Skip validation for speed
});

// Download compressed file with decompression
const compressedFile = bucket.file('logs/compressed-logs.gz');
await transferManager.downloadFileInChunks(compressedFile, {
  destination: '/local/logs/decompressed.log',
  decompress: true
});

Progress Tracking and Events

Upload Progress Monitoring

// Track upload progress
class UploadProgressTracker {
  private totalFiles: number = 0;
  private completedFiles: number = 0;
  private totalBytes: number = 0;
  private uploadedBytes: number = 0;
  
  onUploadStart(files: string[]) {
    this.totalFiles = files.length;
    this.completedFiles = 0;
    console.log(`Starting upload of ${this.totalFiles} files`);
  }
  
  onFileComplete(filePath: string, file: File) {
    this.completedFiles++;
    console.log(`Uploaded ${filePath} -> ${file.name} (${this.completedFiles}/${this.totalFiles})`);
  }
  
  onUploadComplete() {
    console.log(`Upload complete: ${this.completedFiles} files uploaded`);
  }
}

// Custom upload with progress tracking
async function uploadWithProgress(filePaths: string[]) {
  const tracker = new UploadProgressTracker();
  tracker.onUploadStart(filePaths);
  
  const results = await transferManager.uploadManyFiles(filePaths, {
    concurrencyLimit: 5
  });
  
  results.forEach(([file], index) => {
    tracker.onFileComplete(filePaths[index], file);
  });
  
  tracker.onUploadComplete();
  
  return results;
}

Download Progress Monitoring

// Track download progress
class DownloadProgressTracker {
  private totalSize: number = 0;
  private downloadedSize: number = 0;
  private startTime: number = Date.now();
  
  onDownloadStart(files: File[]) {
    this.totalSize = files.reduce((sum, file) => {
      return sum + (parseInt(file.metadata?.size || '0', 10) || 0);
    }, 0);
    
    console.log(`Starting download of ${files.length} files (${this.formatBytes(this.totalSize)})`);
  }
  
  onChunkDownloaded(bytes: number) {
    this.downloadedSize += bytes;
    const progress = (this.downloadedSize / this.totalSize) * 100;
    const elapsed = (Date.now() - this.startTime) / 1000;
    const speed = this.downloadedSize / elapsed;
    
    console.log(`Progress: ${progress.toFixed(1)}% (${this.formatBytes(speed)}/s)`);
  }
  
  private formatBytes(bytes: number): string {
    const units = ['B', 'KB', 'MB', 'GB'];
    let size = bytes;
    let unitIndex = 0;
    
    while (size >= 1024 && unitIndex < units.length - 1) {
      size /= 1024;
      unitIndex++;
    }
    
    return `${size.toFixed(2)} ${units[unitIndex]}`;
  }
}

Error Handling and Retry Logic

Robust Upload with Error Handling

async function robustUpload(filePaths: string[]) {
  const maxRetries = 3;
  const results: Array<{ file: string; success: boolean; error?: Error }> = [];
  
  for (const filePath of filePaths) {
    let success = false;
    let lastError: Error | null = null;
    
    for (let attempt = 1; attempt <= maxRetries; attempt++) {
      try {
        const [uploadResults] = await transferManager.uploadManyFiles([filePath], {
          concurrencyLimit: 1,
          passthroughOptions: {
            timeout: 300000 // 5 minutes
          }
        });
        
        results.push({ file: filePath, success: true });
        success = true;
        break;
        
      } catch (error) {
        lastError = error as Error;
        console.warn(`Upload attempt ${attempt} failed for ${filePath}:`, error.message);
        
        if (attempt < maxRetries) {
          // Exponential backoff
          const delay = Math.pow(2, attempt) * 1000;
          await new Promise(resolve => setTimeout(resolve, delay));
        }
      }
    }
    
    if (!success) {
      results.push({ file: filePath, success: false, error: lastError! });
    }
  }
  
  // Report results
  const successful = results.filter(r => r.success);
  const failed = results.filter(r => !r.success);
  
  console.log(`Upload complete: ${successful.length} successful, ${failed.length} failed`);
  
  if (failed.length > 0) {
    console.error('Failed uploads:');
    failed.forEach(result => {
      console.error(`  ${result.file}: ${result.error?.message}`);
    });
  }
  
  return results;
}

Parallel Processing with Concurrency Control

class ConcurrentTransferManager {
  private bucket: Bucket;
  private concurrencyLimit: number;
  
  constructor(bucket: Bucket, concurrencyLimit = 10) {
    this.bucket = bucket;
    this.concurrencyLimit = concurrencyLimit;
  }
  
  async uploadDirectory(directoryPath: string, options?: UploadManyFilesOptions) {
    // Get all files in directory
    const files = await this.getFilesRecursively(directoryPath);
    
    // Process in batches
    const batches = this.createBatches(files, this.concurrencyLimit);
    const results: UploadResponse[] = [];
    
    for (const batch of batches) {
      console.log(`Processing batch of ${batch.length} files`);
      
      const batchPromises = batch.map(async (filePath) => {
        const transferManager = new TransferManager(this.bucket);
        return transferManager.uploadManyFiles([filePath], options);
      });
      
      const batchResults = await Promise.allSettled(batchPromises);
      
      batchResults.forEach((result, index) => {
        if (result.status === 'fulfilled') {
          results.push(...result.value);
          console.log(`✓ Uploaded: ${batch[index]}`);
        } else {
          console.error(`✗ Failed: ${batch[index]} - ${result.reason}`);
        }
      });
    }
    
    return results;
  }
  
  private async getFilesRecursively(dir: string): Promise<string[]> {
    const fs = require('fs').promises;
    const path = require('path');
    
    const files: string[] = [];
    const items = await fs.readdir(dir);
    
    for (const item of items) {
      const itemPath = path.join(dir, item);
      const stat = await fs.stat(itemPath);
      
      if (stat.isDirectory()) {
        files.push(...await this.getFilesRecursively(itemPath));
      } else {
        files.push(itemPath);
      }
    }
    
    return files;
  }
  
  private createBatches<T>(items: T[], batchSize: number): T[][] {
    const batches: T[][] = [];
    for (let i = 0; i < items.length; i += batchSize) {
      batches.push(items.slice(i, i + batchSize));
    }
    return batches;
  }
}

Advanced Transfer Scenarios

Incremental Backup

async function incrementalBackup(localPath: string, remotePath: string) {
  const fs = require('fs').promises;
  const path = require('path');
  
  // Get local files with metadata
  const localFiles = await getLocalFilesWithMetadata(localPath);
  
  // Get remote files
  const [remoteFiles] = await bucket.getFiles({ prefix: remotePath });
  const remoteFileMap = new Map(
    remoteFiles.map(file => [file.name, file])
  );
  
  // Find files to upload (new or modified)
  const filesToUpload: string[] = [];
  
  for (const [relativePath, localStat] of localFiles) {
    const remoteName = path.join(remotePath, relativePath).replace(/\\/g, '/');
    const remoteFile = remoteFileMap.get(remoteName);
    
    if (!remoteFile) {
      // New file
      filesToUpload.push(path.join(localPath, relativePath));
    } else {
      // Check if modified
      const [remoteMetadata] = await remoteFile.getMetadata();
      const remoteModified = new Date(remoteMetadata.updated!);
      
      if (localStat.mtime > remoteModified) {
        filesToUpload.push(path.join(localPath, relativePath));
      }
    }
  }
  
  console.log(`Found ${filesToUpload.length} files to upload`);
  
  if (filesToUpload.length > 0) {
    await transferManager.uploadManyFiles(filesToUpload, {
      concurrencyLimit: 8,
      customDestinationBuilder: (filePath) => {
        const relativePath = path.relative(localPath, filePath);
        return path.join(remotePath, relativePath).replace(/\\/g, '/');
      }
    });
  }
  
  return filesToUpload.length;
}

async function getLocalFilesWithMetadata(dirPath: string): Promise<Map<string, any>> {
  const fs = require('fs').promises;
  const path = require('path');
  const files = new Map();
  
  async function scan(currentPath: string, relativePath = '') {
    const items = await fs.readdir(currentPath);
    
    for (const item of items) {
      const itemPath = path.join(currentPath, item);
      const itemRelativePath = path.join(relativePath, item);
      const stat = await fs.stat(itemPath);
      
      if (stat.isDirectory()) {
        await scan(itemPath, itemRelativePath);
      } else {
        files.set(itemRelativePath, stat);
      }
    }
  }
  
  await scan(dirPath);
  return files;
}

Data Migration Between Buckets

async function migrateBetweenBuckets(
  sourceBucket: Bucket, 
  destinationBucket: Bucket, 
  prefix?: string
) {
  // Get all files from source bucket
  const [files] = await sourceBucket.getFiles({ prefix });
  
  console.log(`Migrating ${files.length} files from ${sourceBucket.name} to ${destinationBucket.name}`);
  
  // Process in batches to avoid memory issues
  const batchSize = 100;
  let migrated = 0;
  
  for (let i = 0; i < files.length; i += batchSize) {
    const batch = files.slice(i, i + batchSize);
    
    const migrationPromises = batch.map(async (file) => {
      try {
        // Copy file to destination bucket
        const [copiedFile] = await file.copy(destinationBucket.file(file.name));
        
        // Verify copy succeeded
        const [exists] = await copiedFile.exists();
        if (exists) {
          // Optionally delete original file
          // await file.delete();
          migrated++;
          console.log(`Migrated: ${file.name}`);
        }
      } catch (error) {
        console.error(`Failed to migrate ${file.name}:`, error);
      }
    });
    
    await Promise.allSettled(migrationPromises);
    console.log(`Progress: ${Math.min(i + batchSize, files.length)}/${files.length} files processed`);
  }
  
  console.log(`Migration complete: ${migrated}/${files.length} files migrated`);
  return migrated;
}

Resumable Large File Transfer

class ResumableTransferManager {
  private bucket: Bucket;
  private uploadStates = new Map<string, string>(); // file -> resumeUri
  
  constructor(bucket: Bucket) {
    this.bucket = bucket;
  }
  
  async resumableUpload(filePath: string, destination?: string) {
    const fs = require('fs');
    const path = require('path');
    
    destination = destination || path.basename(filePath);
    const file = this.bucket.file(destination);
    
    // Check for existing upload session
    let resumeUri = this.uploadStates.get(filePath);
    
    if (!resumeUri) {
      // Create new resumable upload session
      [resumeUri] = await file.createResumableUpload({
        metadata: {
          contentType: this.getContentType(filePath)
        }
      });
      
      this.uploadStates.set(filePath, resumeUri);
      console.log(`Created upload session for ${filePath}`);
    } else {
      console.log(`Resuming upload session for ${filePath}`);
    }
    
    // Upload with resumable stream
    return new Promise<void>((resolve, reject) => {
      const readStream = fs.createReadStream(filePath);
      const writeStream = file.createWriteStream({
        uri: resumeUri,
        resumable: true
      });
      
      writeStream.on('error', (error) => {
        console.error(`Upload failed for ${filePath}:`, error);
        // Keep resume URI for retry
        reject(error);
      });
      
      writeStream.on('finish', () => {
        console.log(`Upload completed for ${filePath}`);
        this.uploadStates.delete(filePath); // Clear successful upload
        resolve();
      });
      
      readStream.pipe(writeStream);
    });
  }
  
  private getContentType(filePath: string): string {
    const ext = require('path').extname(filePath).toLowerCase();
    const contentTypes: { [key: string]: string } = {
      '.jpg': 'image/jpeg',
      '.jpeg': 'image/jpeg', 
      '.png': 'image/png',
      '.pdf': 'application/pdf',
      '.txt': 'text/plain',
      '.json': 'application/json',
      '.zip': 'application/zip'
    };
    
    return contentTypes[ext] || 'application/octet-stream';
  }
}

Performance Optimization

Optimal Concurrency Settings

// Determine optimal concurrency based on file sizes and network
function getOptimalConcurrency(files: string[]): number {
  const fs = require('fs');
  
  // Calculate average file size
  let totalSize = 0;
  let fileCount = 0;
  
  for (const file of files) {
    try {
      const stat = fs.statSync(file);
      totalSize += stat.size;
      fileCount++;
    } catch (error) {
      console.warn(`Could not stat file ${file}`);
    }
  }
  
  if (fileCount === 0) return 5; // Default
  
  const averageSize = totalSize / fileCount;
  const sizeMB = averageSize / (1024 * 1024);
  
  // Adjust concurrency based on file size
  if (sizeMB < 1) return 20;        // Small files: high concurrency
  else if (sizeMB < 10) return 10;  // Medium files: medium concurrency  
  else if (sizeMB < 100) return 5;  // Large files: low concurrency
  else return 2;                    // Very large files: minimal concurrency
}

// Usage
const filePaths = ['/path/to/files/*'];
const concurrency = getOptimalConcurrency(filePaths);

await transferManager.uploadManyFiles(filePaths, {
  concurrencyLimit: concurrency
});

Error Handling

import { ApiError } from '@google-cloud/storage';

try {
  await transferManager.uploadManyFiles(filePaths);
} catch (error) {
  if (error instanceof ApiError) {
    if (error.code === 403) {
      console.log('Permission denied - check bucket permissions');
    } else if (error.code === 404) {
      console.log('Bucket not found');
    } else if (error.code === 413) {
      console.log('File too large - consider chunked upload');
    } else {
      console.error(`Transfer Error ${error.code}: ${error.message}`);
    }
  } else {
    console.error('Transfer error:', error);
  }
}

Callback Support

Transfer Manager methods are Promise-based, but you can use them with callbacks:

// Promise pattern (recommended)
const results = await transferManager.uploadManyFiles(filePaths);

// Callback wrapper
function uploadWithCallback(filePaths: string[], callback: (err: Error | null, results?: UploadResponse[]) => void) {
  transferManager.uploadManyFiles(filePaths)
    .then(results => callback(null, results))
    .catch(error => callback(error));
}

// Usage
uploadWithCallback(filePaths, (err, results) => {
  if (err) {
    console.error('Upload error:', err);
    return;
  }
  console.log(`Uploaded ${results!.length} files`);
});

docs

access-control.md

authentication.md

bucket-operations.md

file-operations.md

index.md

notifications.md

storage-client.md

transfer-manager.md

utilities.md

tile.json