AWS Glue (Services Overview)

const database = new aws.glue.CatalogDatabase("catalog", {
    name: "data_catalog",
});

const table = new aws.glue.CatalogTable("table", {
    name: "events",
    databaseName: database.name,
    storageDescriptor: {
        location: "s3://bucket/data/",
        columns: [
            { name: "id", type: "string" },
            { name: "timestamp", type: "timestamp" },
        ],
    },
});

const crawler = new aws.glue.Crawler("crawler", {
    name: "data-crawler",
    role: role.arn,
    databaseName: database.name,
    s3Targets: [{ path: "s3://bucket/data/" }],
});

const job = new aws.glue.Job("etl", {
    name: "transform-data",
    roleArn: role.arn,
    command: {
        scriptLocation: "s3://bucket/scripts/transform.py",
        pythonVersion: "3",
    },
    glueVersion: "4.0",
});