AWS Storage Services Overview

Start: What type of data?

┌─ Files/Objects (unstructured data)
│  ├─ Need frequent access?
│  │  ├─ Yes → Access patterns?
│  │  │  ├─ Web/mobile apps, analytics → S3 Standard
│  │  │  ├─ Infrequent (< 1/month) → S3 Infrequent Access
│  │  │  └─ Archive (< 1/year) → S3 Glacier
│  │  └─ No → How fast retrieval?
│  │     ├─ Minutes → S3 Glacier Instant/Flexible
│  │     └─ Hours → S3 Glacier Deep Archive
│  │
│  └─ Need shared file system?
│     ├─ Linux/NFS → EFS
│     ├─ Windows SMB → FSx for Windows
│     ├─ High-performance computing → FSx for Lustre
│     ├─ Enterprise features → FSx for NetApp ONTAP
│     └─ ZFS compatibility → FSx for OpenZFS
│
┌─ Block storage (for EC2)
│  ├─ Performance needs?
│  │  ├─ Highest IOPS (> 64,000) → EBS io2 Block Express
│  │  ├─ High IOPS (< 64,000) → EBS io2 or io1
│  │  ├─ Balanced → EBS gp3 (default choice)
│  │  ├─ Throughput-optimized → EBS st1
│  │  └─ Cold storage → EBS sc1
│  │
│  ├─ Temporary data? → EC2 Instance Store
│  └─ Boot volumes → EBS gp3
│
└─ Backup/DR
   ├─ Centralized management → AWS Backup
   ├─ Application-specific → S3 + lifecycle policies
   └─ Long-term retention → S3 Glacier

import * as aws from "@pulumi/aws";

// S3 bucket for website content
const bucket = new aws.s3.BucketV2("website", {
    bucket: "my-website.com",
});

// Enable website hosting
const websiteConfig = new aws.s3.BucketWebsiteConfigurationV2("website-config", {
    bucket: bucket.id,
    indexDocument: { suffix: "index.html" },
    errorDocument: { key: "error.html" },
});

// Public access (behind CloudFront)
const publicAccess = new aws.s3.BucketPublicAccessBlock("public", {
    bucket: bucket.id,
    blockPublicAcls: false,
    blockPublicPolicy: false,
    ignorePublicAcls: false,
    restrictPublicBuckets: false,
});

// CloudFront distribution
const cdn = new aws.cloudfront.Distribution("cdn", {
    origins: [{
        domainName: bucket.bucketRegionalDomainName,
        originId: "S3Origin",
    }],
    enabled: true,
    defaultRootObject: "index.html",
    defaultCacheBehavior: {
        targetOriginId: "S3Origin",
        viewerProtocolPolicy: "redirect-to-https",
        allowedMethods: ["GET", "HEAD"],
        cachedMethods: ["GET", "HEAD"],
        forwardedValues: {
            queryString: false,
            cookies: { forward: "none" },
        },
    },
});

export const websiteUrl = cdn.domainName;

// Data lake bucket
const dataLake = new aws.s3.BucketV2("data-lake", {
    bucket: "my-data-lake",
});

// Lifecycle policy for cost optimization
const lifecycle = new aws.s3.BucketLifecycleConfigurationV2("lifecycle", {
    bucket: dataLake.id,
    rules: [{
        id: "archive-old-data",
        status: "Enabled",
        transitions: [
            {
                days: 30,
                storageClass: "STANDARD_IA",
            },
            {
                days: 90,
                storageClass: "GLACIER_INSTANT_RETRIEVAL",
            },
            {
                days: 365,
                storageClass: "DEEP_ARCHIVE",
            },
        ],
        noncurrentVersionTransitions: [{
            noncurrentDays: 30,
            storageClass: "GLACIER_FLEXIBLE_RETRIEVAL",
        }],
    }],
});

// Enable versioning for data protection
const versioning = new aws.s3.BucketVersioningV2("versioning", {
    bucket: dataLake.id,
    versioningConfiguration: { status: "Enabled" },
});

// Server-side encryption
const encryption = new aws.s3.BucketServerSideEncryptionConfigurationV2("encryption", {
    bucket: dataLake.id,
    rules: [{
        applyServerSideEncryptionByDefault: {
            sseAlgorithm: "AES256",
        },
    }],
});

// High-performance EBS volume for database
const dbVolume = new aws.ebs.Volume("db-volume", {
    availabilityZone: "us-east-1a",
    size: 100, // GB
    type: "io2",
    iops: 10000,
    encrypted: true,
    tags: { Name: "production-db", Purpose: "database" },
});

// Attach to EC2 instance
const attachment = new aws.ec2.VolumeAttachment("db-attachment", {
    deviceName: "/dev/xvdf",
    volumeId: dbVolume.id,
    instanceId: instance.id,
});

// DLM policy for automated snapshots
const snapshotRole = new aws.iam.Role("dlm-role", {
    assumeRolePolicy: JSON.stringify({
        Version: "2012-10-17",
        Statement: [{
            Effect: "Allow",
            Principal: { Service: "dlm.amazonaws.com" },
            Action: "sts:AssumeRole",
        }],
    }),
});

const snapshotPolicy = new aws.dlm.LifecyclePolicy("db-snapshots", {
    description: "Daily database snapshots",
    executionRoleArn: snapshotRole.arn,
    state: "ENABLED",
    policyDetails: {
        resourceTypes: ["VOLUME"],
        schedules: [{
            name: "Daily snapshots",
            createRule: { interval: 24, intervalUnit: "HOURS", times: ["03:00"] },
            retainRule: { count: 7 },
            tagsToAdd: { SnapshotType: "automated" },
        }],
        targetTags: { Purpose: "database" },
    },
});

// EFS file system
const fileSystem = new aws.efs.FileSystem("shared-fs", {
    encrypted: true,
    lifecyclePolicies: [{
        transitionToIa: "AFTER_30_DAYS",
    }],
    tags: { Name: "shared-storage" },
});

// Mount targets in each AZ
const mountTargets = subnetIds.map((subnetId, index) =>
    new aws.efs.MountTarget(`mount-${index}`, {
        fileSystemId: fileSystem.id,
        subnetId: subnetId,
        securityGroups: [securityGroup.id],
    })
);

// Access point for application
const accessPoint = new aws.efs.AccessPoint("app-access", {
    fileSystemId: fileSystem.id,
    posixUser: {
        uid: 1000,
        gid: 1000,
    },
    rootDirectory: {
        path: "/app-data",
        creationInfo: {
            ownerUid: 1000,
            ownerGid: 1000,
            permissions: "755",
        },
    },
});

// Use in ECS task definition
const taskDef = new aws.ecs.TaskDefinition("task", {
    family: "app",
    volumes: [{
        name: "shared-storage",
        efsVolumeConfiguration: {
            fileSystemId: fileSystem.id,
            transitEncryption: "ENABLED",
            authorizationConfig: {
                accessPointId: accessPoint.id,
            },
        },
    }],
    containerDefinitions: JSON.stringify([{
        name: "app",
        image: "nginx",
        mountPoints: [{
            sourceVolume: "shared-storage",
            containerPath: "/data",
        }],
    }]),
    requiresCompatibilities: ["FARGATE"],
    cpu: "256",
    memory: "512",
    networkMode: "awsvpc",
});

// Backup vault
const vault = new aws.backup.Vault("main-vault", {
    name: "primary-backup-vault",
    kmsKeyArn: kmsKey.arn,
});

// Backup plan
const plan = new aws.backup.Plan("daily-backup", {
    name: "daily-backup-plan",
    rules: [{
        ruleName: "DailyBackup",
        targetVaultName: vault.name,
        schedule: "cron(0 5 ? * * *)", // 5 AM UTC daily
        lifecycle: {
            deleteAfter: 30,
            coldStorageAfter: 7, // Move to cold storage after 7 days
        },
    }],
});

// Backup selection (what to back up)
const selection = new aws.backup.Selection("resources", {
    name: "production-resources",
    planId: plan.id,
    iamRoleArn: backupRole.arn,
    resources: [
        rdsInstance.arn,
        efsFileSystem.arn,
        dynamoTable.arn,
    ],
});

// Or use tags to select resources
const tagSelection = new aws.backup.Selection("by-tags", {
    name: "tag-based-selection",
    planId: plan.id,
    iamRoleArn: backupRole.arn,
    selectionTags: [{
        type: "STRINGEQUALS",
        key: "Backup",
        value: "true",
    }],
});