Migration Services

import { dms } from "@pulumi/aws";

// Create a replication subnet group
const subnetGroup = new dms.ReplicationSubnetGroup("dms-subnet", {
    replicationSubnetGroupId: "dms-subnet-group",
    replicationSubnetGroupDescription: "DMS replication subnet group",
    subnetIds: subnetIds,
});

// Create a replication instance
const replicationInstance = new dms.ReplicationInstance("migration", {
    replicationInstanceId: "db-migration",
    replicationInstanceClass: "dms.t3.medium",
    allocatedStorage: 100,
    engineVersion: "3.5.1",
    multiAz: true,
    publiclyAccessible: false,
    replicationSubnetGroupId: subnetGroup.id,
    vpcSecurityGroupIds: [securityGroup.id],
});

// Create source endpoint (on-premises MySQL)
const sourceEndpoint = new dms.Endpoint("source", {
    endpointId: "source-mysql",
    endpointType: "source",
    engineName: "mysql",
    serverName: "source-db.example.com",
    port: 3306,
    databaseName: "appdb",
    username: "admin",
    password: sourcePassword,
    sslMode: "require",
});

// Create target endpoint (RDS Aurora)
const targetEndpoint = new dms.Endpoint("target", {
    endpointId: "target-aurora",
    endpointType: "target",
    engineName: "aurora-mysql",
    serverName: auroraCluster.endpoint,
    port: 3306,
    databaseName: "appdb",
    username: "admin",
    password: targetPassword,
    sslMode: "require",
});

// Create replication task
const replicationTask = new dms.ReplicationTask("migrate", {
    replicationTaskId: "db-migration-task",
    replicationInstanceArn: replicationInstance.replicationInstanceArn,
    sourceEndpointArn: sourceEndpoint.endpointArn,
    targetEndpointArn: targetEndpoint.endpointArn,
    migrationType: "full-load-and-cdc",
    tableMappings: JSON.stringify({
        rules: [
            {
                "rule-type": "selection",
                "rule-id": "1",
                "rule-name": "1",
                "object-locator": {
                    "schema-name": "appdb",
                    "table-name": "%",
                },
                "rule-action": "include",
            },
        ],
    }),
    replicationTaskSettings: JSON.stringify({
        TargetMetadata: {
            TargetSchema: "",
            SupportLobs: true,
            FullLobMode: false,
            LobChunkSize: 64,
            LimitedSizeLobMode: true,
            LobMaxSize: 32,
        },
        FullLoadSettings: {
            TargetTablePrepMode: "DROP_AND_CREATE",
            CreatePkAfterFullLoad: false,
            StopTaskCachedChangesApplied: false,
            MaxFullLoadSubTasks: 8,
        },
        Logging: {
            EnableLogging: true,
            LogComponents: [
                {
                    Id: "TRANSFORMATION",
                    Severity: "LOGGER_SEVERITY_DEFAULT",
                },
                {
                    Id: "SOURCE_CAPTURE",
                    Severity: "LOGGER_SEVERITY_INFO",
                },
            ],
        },
    }),
});

// Create event subscription for monitoring
new dms.EventSubscription("migration-events", {
    name: "dms-migration-events",
    eventCategories: ["failure", "state change"],
    snsTopicArn: alertTopic.arn,
    sourceType: "replication-task",
    sourceIds: [replicationTask.replicationTaskId],
});

// S3 endpoint for data lake migration
const s3Endpoint = new dms.S3Endpoint("data-lake", {
    endpointId: "s3-target",
    endpointType: "target",
    bucketName: dataLakeBucket.id,
    serviceAccessRoleArn: dmsRole.arn,
    dataFormat: "parquet",
    compressionType: "GZIP",
    datePartitionEnabled: true,
    datePartitionSequence: "YYYYMMDD",
    timestampColumnName: "event_timestamp",
});

import { datasync } from "@pulumi/aws";

// Create a location for on-premises NFS
const nfsLocation = new datasync.NfsLocation("on-prem", {
    serverHostname: "fileserver.example.com",
    subdirectory: "/data/export",
    onPremConfig: {
        agentArns: [datasyncAgent.arn],
    },
});

// Create a location for S3
const s3Location = new datasync.S3Location("aws-storage", {
    s3BucketArn: storageBucket.arn,
    subdirectory: "/imported-data",
    s3Config: {
        bucketAccessRoleArn: datasyncRole.arn,
    },
});

// Create a task to transfer data
const transferTask = new datasync.Task("data-transfer", {
    name: "nfs-to-s3-transfer",
    sourceLocationArn: nfsLocation.arn,
    destinationLocationArn: s3Location.arn,
    options: {
        verifyMode: "POINT_IN_TIME_CONSISTENT",
        overwriteMode: "ALWAYS",
        atime: "BEST_EFFORT",
        mtime: "PRESERVE",
        uid: "INT_VALUE",
        gid: "INT_VALUE",
        preserveDeletedFiles: "PRESERVE",
        preserveDevices: "NONE",
        posixPermissions: "PRESERVE",
        bytesPerSecond: 104857600, // 100 MB/s
        taskQueueing: "ENABLED",
        logLevel: "TRANSFER",
    },
    schedule: {
        scheduleExpression: "cron(0 2 * * ? *)", // Daily at 2 AM
    },
    cloudwatchLogGroupArn: datasyncLogs.arn,
});

// Create EFS location for cloud-to-cloud transfer
const efsLocation = new datasync.EfsLocation("efs-target", {
    efsFileSystemArn: fileSystem.arn,
    ec2Config: {
        securityGroupArns: [securityGroup.arn],
        subnetArn: subnet.arn,
    },
});

// Create FSx for Windows location
const fsxLocation = new datasync.FsxWindowsLocation("fsx", {
    fsxFilesystemArn: fsxFileSystem.arn,
    user: "Admin",
    password: fsxPassword,
    securityGroupArns: [securityGroup.arn],
});

import { transfer } from "@pulumi/aws";

// Create a server
const ftpServer = new transfer.Server("file-transfer", {
    endpointType: "VPC",
    protocols: ["SFTP", "FTPS"],
    identityProviderType: "SERVICE_MANAGED",
    domain: "S3",
    endpointDetails: {
        vpcId: vpc.id,
        subnetIds: subnetIds,
        securityGroupIds: [securityGroup.id],
    },
    certificate: certificate.arn,
    loggingRole: transferLogsRole.arn,
});

// Create a user
const user = new transfer.User("ftp-user", {
    userName: "datauser",
    serverId: ftpServer.id,
    role: transferUserRole.arn,
    homeDirectory: "/my-bucket/users/datauser",
    homeDirectoryType: "LOGICAL",
    homeDirectoryMappings: [{
        entry: "/",
        target: `/${storageBucket.id}/users/datauser`,
    }],
});

// Add SSH key for authentication
new transfer.SshKey("user-key", {
    serverId: ftpServer.id,
    userName: user.userName,
    body: sshPublicKey,
});

// Create a connector for outbound transfers
const connector = new transfer.Connector("external-sftp", {
    url: "sftp://partner.example.com",
    accessRole: connectorRole.arn,
    as2Config: {
        localProfileId: localProfile.id,
        partnerProfileId: partnerProfile.id,
        messageSubject: "Data Transfer",
    },
});

// Create a workflow for file processing
const workflow = new transfer.Workflow("process-uploads", {
    steps: [
        {
            type: "COPY",
            copyStepDetails: {
                name: "copy-to-archive",
                destinationFileLocation: {
                    s3FileLocation: {
                        bucket: archiveBucket.id,
                        key: "archive/${transfer:UserName}/${transfer:UploadDate}/",
                    },
                },
            },
        },
        {
            type: "CUSTOM",
            customStepDetails: {
                name: "process-file",
                target: processingLambda.arn,
                timeoutSeconds: 300,
            },
        },
    ],
    onExceptionSteps: [{
        type: "CUSTOM",
        customStepDetails: {
            name: "handle-error",
            target: errorHandler.arn,
        },
    }],
});

// Associate workflow with server
new transfer.ServerWorkflow("upload-workflow", {
    serverId: ftpServer.id,
    workflowId: workflow.id,
    onUpload: true,
});

import { drs } from "@pulumi/aws";

// Create a replication configuration template
const replicationConfig = new drs.ReplicationConfigurationTemplate("disaster-recovery", {
    stagingAreaSubnetId: subnet.id,
    associateDefaultSecurityGroup: false,
    securityGroupIds: [replicationSecurityGroup.id],
    replicationServerInstanceType: "t3.small",
    replicationServersSecurityGroupsIds: [securityGroup.id],
    ebsEncryption: "DEFAULT",
    dataPlaneRouting: "PRIVATE_IP",
    createPublicIp: false,
    useDedicatedReplicationServer: false,
    tags: {
        Environment: "dr",
    },
});