or run

npx @tessl/cli init

Version

Tile

Overview

Evals

Files

docs

access-control.md authentication.md bucket-operations.md file-operations.md index.md notifications.md storage-client.md transfer-manager.md utilities.md

tile.json

Authentication

Google Cloud Storage supports multiple authentication methods including HMAC keys for interoperability with Amazon S3-compatible tools, service accounts, and Application Default Credentials.

HMAC Keys

HMAC keys provide interoperable authentication that works with Amazon S3 libraries and tools, enabling a smooth migration path from S3 to Cloud Storage.

HmacKey Class

class HmacKey extends ServiceObject {
  constructor(storage: Storage, accessId: string, options?: HmacKeyOptions);
  
  // Properties
  accessId: string;
  storage: Storage;
  projectId?: string;
  userProject?: string;
  
  // Methods
  getMetadata(options?: GetMetadataOptions): Promise<HmacKeyMetadataResponse>;
  setMetadata(metadata: SetHmacKeyMetadata, options?: SetHmacKeyMetadataOptions): Promise<HmacKeyMetadataResponse>;
  delete(): Promise<[unknown]>;
}

interface HmacKeyOptions {
  projectId?: string;
}

HMAC Key Metadata

interface HmacKeyMetadata {
  accessId?: string;
  etag?: string;
  id?: string;
  projectId?: string;
  serviceAccountEmail?: string;
  state?: 'ACTIVE' | 'INACTIVE' | 'DELETED';
  timeCreated?: string;
  updated?: string;
}

interface SetHmacKeyMetadata {
  state?: 'ACTIVE' | 'INACTIVE';
  etag?: string;
}

type HmacKeyMetadataResponse = [HmacKeyMetadata, unknown]; // [metadata, apiResponse]

Creating HMAC Keys

// Create HMAC key for service account
const serviceAccountEmail = 'service-account@project.iam.gserviceaccount.com';
const [hmacKey, secret] = await storage.createHmacKey(serviceAccountEmail);

console.log('Access ID:', hmacKey.accessId);
console.log('Secret Key (store securely):', secret);
console.log('Service Account:', hmacKey.metadata?.serviceAccountEmail);
console.log('State:', hmacKey.metadata?.state);

// Create with explicit project
const [hmacKey, secret] = await storage.createHmacKey(serviceAccountEmail, {
  projectId: 'my-project-id'
});

// Store credentials securely
const credentials = {
  accessKeyId: hmacKey.accessId,
  secretAccessKey: secret,
  endpoint: 'https://storage.googleapis.com'
};

Managing HMAC Keys

// Get HMAC key reference
const hmacKey = storage.hmacKey('GOOG1EXAMPLE123456789');

// Get metadata
const [metadata] = await hmacKey.getMetadata();
console.log('HMAC Key Info:');
console.log('  Access ID:', metadata.accessId);
console.log('  Service Account:', metadata.serviceAccountEmail);
console.log('  State:', metadata.state);
console.log('  Created:', metadata.timeCreated);
console.log('  Updated:', metadata.updated);

// Deactivate HMAC key
await hmacKey.setMetadata({
  state: 'INACTIVE'
});

// Reactivate HMAC key
await hmacKey.setMetadata({
  state: 'ACTIVE'
});

// Delete HMAC key (must be INACTIVE first)
await hmacKey.setMetadata({ state: 'INACTIVE' });
await hmacKey.delete();

Listing HMAC Keys

// List all HMAC keys
const [hmacKeys] = await storage.getHmacKeys();
hmacKeys.forEach(key => {
  console.log(`Access ID: ${key.accessId}`);
  console.log(`Service Account: ${key.metadata?.serviceAccountEmail}`);
  console.log(`State: ${key.metadata?.state}`);
  console.log('---');
});

// List keys for specific service account
const [hmacKeys] = await storage.getHmacKeys({
  serviceAccountEmail: 'service-account@project.iam.gserviceaccount.com'
});

// Include deleted keys
const [allKeys] = await storage.getHmacKeys({
  showDeletedKeys: true
});

// Stream HMAC keys for large lists
storage.getHmacKeysStream()
  .on('data', hmacKey => {
    console.log(`HMAC Key: ${hmacKey.accessId} - ${hmacKey.metadata?.state}`);
  })
  .on('end', () => {
    console.log('Finished listing HMAC keys');
  });

// List with pagination
const [hmacKeys, nextQuery] = await storage.getHmacKeys({
  maxResults: 10
});

if (nextQuery) {
  const [moreKeys] = await storage.getHmacKeys(nextQuery);
}

Using HMAC Keys with S3-Compatible Libraries

AWS SDK Configuration

// Using AWS SDK v3
import { S3Client, ListObjectsV2Command, PutObjectCommand } from '@aws-sdk/client-s3';

const s3Client = new S3Client({
  credentials: {
    accessKeyId: 'GOOG1EXAMPLE123456789', // HMAC access ID
    secretAccessKey: 'base64-encoded-secret' // HMAC secret
  },
  endpoint: 'https://storage.googleapis.com',
  region: 'auto', // Not used by Cloud Storage but required by AWS SDK
  forcePathStyle: true // Use path-style URLs
});

// List objects
const listCommand = new ListObjectsV2Command({
  Bucket: 'my-bucket',
  Prefix: 'uploads/'
});
const response = await s3Client.send(listCommand);

// Upload object
const putCommand = new PutObjectCommand({
  Bucket: 'my-bucket',
  Key: 'path/to/file.txt',
  Body: 'File contents'
});
await s3Client.send(putCommand);

Boto3 (Python) Configuration

# Using boto3 in Python
import boto3

client = boto3.client(
    's3',
    endpoint_url='https://storage.googleapis.com',
    aws_access_key_id='GOOG1EXAMPLE123456789',
    aws_secret_access_key='base64-encoded-secret'
)

# List objects
response = client.list_objects_v2(
    Bucket='my-bucket',
    Prefix='uploads/'
)

# Upload object
client.put_object(
    Bucket='my-bucket',
    Key='path/to/file.txt',
    Body=b'File contents'
)

curl Commands

# List bucket contents
curl -H "Authorization: AWS GOOG1EXAMPLE123456789:SIGNATURE" \
     "https://storage.googleapis.com/my-bucket/"

# Upload file
curl -X PUT \
     -H "Authorization: AWS GOOG1EXAMPLE123456789:SIGNATURE" \
     -H "Content-Type: text/plain" \
     --data "File contents" \
     "https://storage.googleapis.com/my-bucket/path/to/file.txt"

Service Account Authentication

Default Credentials

// Application Default Credentials (ADC)
// Automatically discovers credentials from environment
const storage = new Storage();

// ADC search order:
// 1. GOOGLE_APPLICATION_CREDENTIALS environment variable
// 2. gcloud user credentials
// 3. Google Cloud metadata server (on GCP)
// 4. Service account attached to the resource

Explicit Service Account

// Using service account key file
const storage = new Storage({
  projectId: 'my-project-id',
  keyFilename: '/path/to/service-account-key.json'
});

// Using credentials object
const storage = new Storage({
  projectId: 'my-project-id',
  credentials: {
    type: 'service_account',
    project_id: 'my-project-id',
    private_key_id: 'key-id',
    private_key: '-----BEGIN PRIVATE KEY-----\n...',
    client_email: 'service@project.iam.gserviceaccount.com',
    client_id: '123456789',
    auth_uri: 'https://accounts.google.com/o/oauth2/auth',
    token_uri: 'https://oauth2.googleapis.com/token'
  }
});

Environment Variables

// Set environment variables
process.env.GOOGLE_APPLICATION_CREDENTIALS = '/path/to/service-account.json';
process.env.GOOGLE_CLOUD_PROJECT = 'my-project-id';

// Storage client will automatically use these
const storage = new Storage();

// Or set in shell:
// export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json"
// export GOOGLE_CLOUD_PROJECT="my-project-id"

Custom Authentication

Using Google Auth Library

import { GoogleAuth } from 'google-auth-library';

// Custom auth client
const auth = new GoogleAuth({
  scopes: ['https://www.googleapis.com/auth/cloud-platform'],
  keyFile: '/path/to/service-account.json'
});

const authClient = await auth.getClient();

const storage = new Storage({
  authClient: authClient
});

// With impersonation
const auth = new GoogleAuth({
  scopes: ['https://www.googleapis.com/auth/cloud-platform'],
  credentials: {
    // Source credentials
    client_email: 'source@project.iam.gserviceaccount.com',
    private_key: '...'
  }
});

const client = await auth.getClient();
client.targetPrincipal = 'target@project.iam.gserviceaccount.com';

const storage = new Storage({
  authClient: client
});

OAuth 2.0 Flow

import { OAuth2Client } from 'google-auth-library';

// OAuth client for user authentication
const oauth2Client = new OAuth2Client(
  'client-id',
  'client-secret',
  'http://localhost:3000/callback'
);

// Generate auth URL
const authUrl = oauth2Client.generateAuthUrl({
  access_type: 'offline',
  scope: ['https://www.googleapis.com/auth/cloud-platform']
});

// After user authorization, exchange code for tokens
const { tokens } = await oauth2Client.getToken(authorizationCode);
oauth2Client.setCredentials(tokens);

const storage = new Storage({
  authClient: oauth2Client
});

Authentication Best Practices

Secure Credential Storage

// Don't hardcode credentials
// ❌ Bad
const storage = new Storage({
  credentials: {
    private_key: '-----BEGIN PRIVATE KEY-----\nActual key here...'
  }
});

// ✅ Good - use environment variables
const storage = new Storage({
  credentials: {
    private_key: process.env.GOOGLE_PRIVATE_KEY?.replace(/\\n/g, '\n')
  }
});

// ✅ Best - use ADC with key file
process.env.GOOGLE_APPLICATION_CREDENTIALS = '/secure/path/to/key.json';
const storage = new Storage();

Principle of Least Privilege

// Create service accounts with minimal required permissions

// For read-only access
const readOnlyServiceAccount = {
  roles: [
    'roles/storage.objectViewer'
  ]
};

// For upload-only access
const uploadServiceAccount = {
  roles: [
    'roles/storage.objectCreator'
  ]
};

// For full bucket management
const bucketAdminServiceAccount = {
  roles: [
    'roles/storage.admin'
  ]
};

Credential Rotation

// Regular HMAC key rotation
async function rotateHmacKey(serviceAccountEmail: string, oldAccessId: string) {
  // Create new HMAC key
  const [newHmacKey, newSecret] = await storage.createHmacKey(serviceAccountEmail);
  
  // Update application configuration
  await updateApplicationConfig({
    accessKeyId: newHmacKey.accessId,
    secretAccessKey: newSecret
  });
  
  // Test new credentials
  const testStorage = new Storage({
    // Use HMAC credentials with S3-compatible library
  });
  
  try {
    await testStorage.getBuckets({ maxResults: 1 });
    
    // Deactivate old key
    const oldKey = storage.hmacKey(oldAccessId);
    await oldKey.setMetadata({ state: 'INACTIVE' });
    
    // Delete old key after grace period
    setTimeout(async () => {
      await oldKey.delete();
    }, 24 * 60 * 60 * 1000); // 24 hours
    
  } catch (error) {
    // Rollback if new key doesn't work
    await newHmacKey.setMetadata({ state: 'INACTIVE' });
    await newHmacKey.delete();
    throw error;
  }
}

Monitoring and Auditing

// Enable audit logging for authentication events
// Configure in Cloud Console or via gcloud:
// gcloud logging sinks create storage-audit \
//   bigquery.googleapis.com/projects/PROJECT_ID/datasets/audit_logs \
//   --log-filter='protoPayload.serviceName="storage.googleapis.com"'

// Monitor HMAC key usage
async function auditHmacKeys() {
  const [hmacKeys] = await storage.getHmacKeys();
  
  for (const key of hmacKeys) {
    const [metadata] = await key.getMetadata();
    
    // Check for old or unused keys
    const created = new Date(metadata.timeCreated);
    const ageInDays = (Date.now() - created.getTime()) / (1000 * 60 * 60 * 24);
    
    if (ageInDays > 90 && metadata.state === 'ACTIVE') {
      console.warn(`HMAC key ${key.accessId} is ${ageInDays} days old and still active`);
    }
    
    if (metadata.state === 'INACTIVE' && ageInDays > 30) {
      console.info(`Consider deleting inactive HMAC key ${key.accessId}`);
    }
  }
}

Error Handling

import { ApiError } from '@google-cloud/storage';

try {
  const [hmacKey, secret] = await storage.createHmacKey('invalid-email');
} catch (error) {
  if (error instanceof ApiError) {
    if (error.code === 400) {
      console.log('Invalid service account email');
    } else if (error.code === 403) {
      console.log('Permission denied - check IAM roles');
    } else if (error.code === 404) {
      console.log('Service account not found');
    } else {
      console.error(`API Error ${error.code}: ${error.message}`);
    }
  }
}

// Test authentication
async function testAuthentication() {
  try {
    const [buckets] = await storage.getBuckets({ maxResults: 1 });
    console.log('Authentication successful');
    return true;
  } catch (error) {
    if (error instanceof ApiError) {
      if (error.code === 401) {
        console.error('Authentication failed - invalid credentials');
      } else if (error.code === 403) {
        console.error('Authentication successful but access denied');
      } else {
        console.error(`Authentication error: ${error.message}`);
      }
    }
    return false;
  }
}

Migration from S3

Credential Mapping

// S3 credentials
const s3Config = {
  accessKeyId: 'AKIAIOSFODNN7EXAMPLE',
  secretAccessKey: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
  region: 'us-west-2'
};

// Create equivalent HMAC key in Cloud Storage
const serviceAccount = 'migration-service@project.iam.gserviceaccount.com';
const [hmacKey, secret] = await storage.createHmacKey(serviceAccount);

const gcsConfig = {
  accessKeyId: hmacKey.accessId,           // Maps to S3 access key
  secretAccessKey: secret,                  // Maps to S3 secret key
  endpoint: 'https://storage.googleapis.com', // Cloud Storage endpoint
  region: 'auto'                           // Not used but required by S3 SDK
};

Code Migration Examples

// Original S3 code
import AWS from 'aws-sdk';
const s3 = new AWS.S3({
  accessKeyId: 'AKIA...',
  secretAccessKey: '...',
  region: 'us-west-2'
});

// Migrated to Cloud Storage with HMAC
import AWS from 'aws-sdk';
const s3 = new AWS.S3({
  accessKeyId: 'GOOG1...', // HMAC access ID
  secretAccessKey: '...',   // HMAC secret
  endpoint: 'https://storage.googleapis.com',
  s3ForcePathStyle: true,
  region: 'auto'
});

// Or migrate to native Cloud Storage client
import { Storage } from '@google-cloud/storage';
const storage = new Storage();
const bucket = storage.bucket('my-bucket');

Callback Support

All HMAC key methods support both Promise and callback patterns:

// Promise pattern (recommended)
const [hmacKey, secret] = await storage.createHmacKey(serviceAccountEmail);

// Callback pattern
storage.createHmacKey(serviceAccountEmail, (err, hmacKey, secret, apiResponse) => {
  if (err) {
    console.error('Error:', err);
    return;
  }
  console.log('Created HMAC key:', hmacKey.accessId);
  console.log('Secret:', secret);
});

// Callback types
interface CreateHmacKeyCallback {
  (err: Error | null, hmacKey?: HmacKey, secret?: string, apiResponse?: unknown): void;
}

interface HmacKeyMetadataCallback {
  (err: Error | null, metadata?: HmacKeyMetadata, apiResponse?: unknown): void;
}