or run

tessl search

tessl/golang-cloud-google-com--go--bigquery

tessl install tessl/golang-cloud-google-com--go--bigquery@1.72.0

Google Cloud BigQuery client library providing comprehensive Go APIs for querying, loading data, managing datasets and tables, streaming inserts, and accessing BigQuery's ecosystem of services including Storage, Analytics Hub, Data Transfer, and Migration APIs

Dataset Management

This document covers creating, reading, updating, and deleting BigQuery datasets, as well as configuring access control, encryption, and other dataset properties.

Overview

A Dataset is a container for tables, views, models, and routines in BigQuery. Datasets are organized by project and location, with configurable access controls, encryption, and expiration policies.

Dataset Reference

Getting a Dataset Reference

func (c *Client) Dataset(id string) *Dataset

Get a dataset reference in the client's project:

dataset := client.Dataset("my_dataset")

func (c *Client) DatasetInProject(projectID, datasetID string) *Dataset

Get a dataset reference in another project:

dataset := client.DatasetInProject("other-project", "their_dataset")

Dataset Type

type Dataset struct {
    ProjectID string
    DatasetID string
}

The Dataset type represents a reference to a BigQuery dataset. It does not necessarily mean the dataset exists.

Dataset Identifier

func (d *Dataset) Identifier(f IdentifierFormat) (string, error)

type IdentifierFormat int

const (
    // StandardSQLID is the format required for Standard SQL queries
    StandardSQLID IdentifierFormat = iota
    // LegacySQLID is the format used in Legacy SQL queries
    LegacySQLID
    // ApiEndpointID is the format used in API requests
    ApiEndpointID
    // StorageAPIResourceID is the format used in Storage API requests
    StorageAPIResourceID
)

Get the dataset identifier in various formats:

// Standard SQL format: `project.dataset` (with backticks if needed)
sqlID, err := dataset.Identifier(bigquery.StandardSQLID)

// Legacy SQL format: [project:dataset]
legacyID, err := dataset.Identifier(bigquery.LegacySQLID)

// API endpoint format: projects/project/datasets/dataset
apiID, err := dataset.Identifier(bigquery.ApiEndpointID)

Creating Datasets

Basic Dataset Creation

func (d *Dataset) Create(ctx context.Context, md *DatasetMetadata) error

Create a dataset with default settings:

if err := dataset.Create(ctx, nil); err != nil {
    // Handle error
}

Create with metadata:

meta := &bigquery.DatasetMetadata{
    Name:        "My Dataset",
    Description: "Dataset for analytics",
    Location:    "US",
}
if err := dataset.Create(ctx, meta); err != nil {
    // Handle error
}

Create with Options

func (d *Dataset) CreateWithOptions(ctx context.Context, md *DatasetMetadata, opts ...DatasetOption) error

Create with additional options:

meta := &bigquery.DatasetMetadata{
    Name:     "My Dataset",
    Location: "US",
}
err := dataset.CreateWithOptions(ctx, meta,
    bigquery.WithAccessPolicyVersion(3))

Dataset Metadata

DatasetMetadata Type

type DatasetMetadata struct {
    // Settable fields
    Name                       string
    Description                string
    Location                   string
    DefaultTableExpiration     time.Duration
    Labels                     map[string]string
    Access                     []*AccessEntry
    DefaultEncryptionConfig    *EncryptionConfig
    DefaultPartitionExpiration time.Duration
    DefaultCollation           string
    ExternalDatasetReference   *ExternalDatasetReference
    MaxTimeTravel              time.Duration
    StorageBillingModel        string
    IsCaseInsensitive          bool

    // Read-only fields
    CreationTime     time.Time
    LastModifiedTime time.Time
    FullID           string
    Tags             []*DatasetTag
    ETag             string
}

Field descriptions:

Name: User-friendly name for the dataset
Description: Dataset description
Location: Geographic location (e.g., "US", "EU", "us-central1")
DefaultTableExpiration: Default expiration time for new tables
Labels: User-provided key-value labels
Access: Access control list
DefaultEncryptionConfig: Encryption configuration for new resources
DefaultPartitionExpiration: Default partition expiration for new partitioned tables
DefaultCollation: Default collation for new tables
ExternalDatasetReference: For external datasets
MaxTimeTravel: Time travel window in hours (up to 7 days)
StorageBillingModel: "LOGICAL" (default) or "PHYSICAL"
IsCaseInsensitive: Whether dataset and table names are case-insensitive
CreationTime: When the dataset was created (read-only)
LastModifiedTime: When dataset or its tables were modified (read-only)
FullID: Full dataset ID in format projectID:datasetID (read-only)
Tags: Resource manager tags (read-only)
ETag: Entity tag for optimistic concurrency control (read-only)

Getting Dataset Metadata

func (d *Dataset) Metadata(ctx context.Context) (*DatasetMetadata, error)

meta, err := dataset.Metadata(ctx)
if err != nil {
    // Handle error (e.g., dataset doesn't exist)
}
fmt.Printf("Dataset: %s\n", meta.Name)
fmt.Printf("Location: %s\n", meta.Location)
fmt.Printf("Created: %s\n", meta.CreationTime)

Get Metadata with Options

func (d *Dataset) MetadataWithOptions(ctx context.Context, opts ...DatasetOption) (*DatasetMetadata, error)

meta, err := dataset.MetadataWithOptions(ctx,
    bigquery.WithDatasetView(bigquery.DatasetMetadataView))

Updating Datasets

DatasetMetadataToUpdate Type

type DatasetMetadataToUpdate struct {
    Description                optional.String
    Name                       optional.String
    DefaultTableExpiration     optional.Duration
    DefaultPartitionExpiration optional.Duration
    DefaultEncryptionConfig    *EncryptionConfig
    DefaultCollation           optional.String
    ExternalDatasetReference   *ExternalDatasetReference
    MaxTimeTravel              optional.Duration
    StorageBillingModel        optional.String
    Access                     []*AccessEntry
    IsCaseInsensitive          optional.Bool
}

Note: Fields use optional.String, optional.Duration, and optional.Bool from cloud.google.com/go/internal/optional to distinguish between "not set" and "set to zero value".

Update Dataset

func (d *Dataset) Update(ctx context.Context, dm DatasetMetadataToUpdate, etag string) (*DatasetMetadata, error)

Update dataset metadata:

import "cloud.google.com/go/internal/optional"

update := bigquery.DatasetMetadataToUpdate{
    Description: optional.ToString("Updated description"),
    Name:        optional.ToString("New Name"),
}

// Blind write (always succeeds)
meta, err := dataset.Update(ctx, update, "")

// Read-modify-write with concurrency control
meta, err := dataset.Metadata(ctx)
if err != nil {
    return err
}
update := bigquery.DatasetMetadataToUpdate{
    Description: optional.ToString("Updated description"),
}
meta, err = dataset.Update(ctx, update, meta.ETag)

Update with Options

func (d *Dataset) UpdateWithOptions(ctx context.Context, dm DatasetMetadataToUpdate, etag string, opts ...DatasetOption) (*DatasetMetadata, error)

meta, err := dataset.UpdateWithOptions(ctx, update, "",
    bigquery.WithUpdateMode(bigquery.DatasetMetadataUpdateMode))

Managing Labels

func (u *DatasetMetadataToUpdate) SetLabel(name, value string)
func (u *DatasetMetadataToUpdate) DeleteLabel(name string)

update := bigquery.DatasetMetadataToUpdate{}
update.SetLabel("environment", "production")
update.SetLabel("team", "analytics")
update.DeleteLabel("old_label")

meta, err := dataset.Update(ctx, update, "")

Deleting Datasets

Delete Empty Dataset

func (d *Dataset) Delete(ctx context.Context) error

Delete a dataset (fails if not empty):

if err := dataset.Delete(ctx); err != nil {
    // Handle error (e.g., dataset not empty)
}

Delete Dataset with Contents

func (d *Dataset) DeleteWithContents(ctx context.Context) error

Delete dataset and all contained tables, views, models, and routines:

if err := dataset.DeleteWithContents(ctx); err != nil {
    // Handle error
}

Listing Datasets

Dataset Iterator

func (c *Client) Datasets(ctx context.Context) *DatasetIterator

type DatasetIterator struct {
    // ListHidden causes hidden datasets to be listed when set to true.
    // Set before the first call to Next.
    ListHidden bool

    // Filter restricts the datasets returned by label.
    // Set before the first call to Next.
    Filter string

    // The project ID of the listed datasets.
    // Set before the first call to Next.
    ProjectID string
}

func (it *DatasetIterator) Next() (*Dataset, error)
func (it *DatasetIterator) PageInfo() *iterator.PageInfo

List all datasets in the client's project:

it := client.Datasets(ctx)
for {
    dataset, err := it.Next()
    if err == iterator.Done {
        break
    }
    if err != nil {
        return err
    }
    fmt.Printf("Dataset: %s.%s\n", dataset.ProjectID, dataset.DatasetID)
}

List Datasets in Another Project

func (c *Client) DatasetsInProject(ctx context.Context, projectID string) *DatasetIterator

Deprecated: Use Client.Datasets() and set ProjectID on the iterator.

// Deprecated approach
it := client.DatasetsInProject(ctx, "other-project")

// Preferred approach
it := client.Datasets(ctx)
it.ProjectID = "other-project"

Filter by Labels

it := client.Datasets(ctx)
it.Filter = "labels.environment:production"
for {
    dataset, err := it.Next()
    if err == iterator.Done {
        break
    }
    if err != nil {
        return err
    }
    // Process production datasets
}

List Hidden Datasets

it := client.Datasets(ctx)
it.ListHidden = true

Pagination

import "google.golang.org/api/iterator"

it := client.Datasets(ctx)
pageInfo := it.PageInfo()
pageInfo.MaxSize = 100 // Max datasets per page

// Get first page
for i := 0; i < pageInfo.MaxSize; i++ {
    dataset, err := it.Next()
    if err == iterator.Done {
        break
    }
    if err != nil {
        return err
    }
    // Process dataset
}

// Get next page using token
token := pageInfo.Token
// Use token in subsequent requests

Access Control

AccessEntry Type

type AccessEntry struct {
    Role       AccessRole
    EntityType EntityType
    Entity     string
    View       *Table
    Routine    *Routine
    Dataset    *DatasetAccessEntry
    Condition  *Expr
}

type AccessRole string

const (
    OwnerRole  AccessRole = "OWNER"
    ReaderRole AccessRole = "READER"
    WriterRole AccessRole = "WRITER"
)

type EntityType int

const (
    DomainEntity EntityType = iota + 1
    GroupEmailEntity
    UserEmailEntity
    SpecialGroupEntity
    ViewEntity
    IAMMemberEntity
    RoutineEntity
    DatasetEntity
)

Setting Access Controls

meta := &bigquery.DatasetMetadata{
    Name:     "My Dataset",
    Location: "US",
    Access: []*bigquery.AccessEntry{
        {
            Role:       bigquery.OwnerRole,
            EntityType: bigquery.UserEmailEntity,
            Entity:     "user@example.com",
        },
        {
            Role:       bigquery.ReaderRole,
            EntityType: bigquery.GroupEmailEntity,
            Entity:     "analysts@example.com",
        },
        {
            Role:       bigquery.ReaderRole,
            EntityType: bigquery.SpecialGroupEntity,
            Entity:     "projectReaders",
        },
    },
}
err := dataset.Create(ctx, meta)

Authorized Views

viewDataset := client.Dataset("view_dataset")
viewTable := viewDataset.Table("authorized_view")

meta := &bigquery.DatasetMetadata{
    Access: []*bigquery.AccessEntry{
        {
            EntityType: bigquery.ViewEntity,
            View:       viewTable,
        },
    },
}

Authorized Routines

routineDataset := client.Dataset("routine_dataset")
routine := routineDataset.Routine("authorized_routine")

meta := &bigquery.DatasetMetadata{
    Access: []*bigquery.AccessEntry{
        {
            EntityType: bigquery.RoutineEntity,
            Routine:    routine,
        },
    },
}

Conditional Access

type Expr struct {
    Expression  string
    Title       string
    Description string
    Location    string
}

meta := &bigquery.DatasetMetadata{
    Access: []*bigquery.AccessEntry{
        {
            Role:       bigquery.ReaderRole,
            EntityType: bigquery.UserEmailEntity,
            Entity:     "user@example.com",
            Condition: &bigquery.Expr{
                Expression: "resource.name.endsWith('/test_table')",
                Title:      "Only test tables",
                Description: "Access restricted to test tables only",
            },
        },
    },
}

// Must use Access Policy Version 3 for conditional bindings
err := dataset.CreateWithOptions(ctx, meta,
    bigquery.WithAccessPolicyVersion(3))

Dataset Access Entry

type DatasetAccessEntry struct {
    Dataset     *Dataset
    TargetTypes []string
}

Grant access to views in another dataset:

sourceDataset := client.Dataset("source_dataset")

meta := &bigquery.DatasetMetadata{
    Access: []*bigquery.AccessEntry{
        {
            EntityType: bigquery.DatasetEntity,
            Dataset: &bigquery.DatasetAccessEntry{
                Dataset:     sourceDataset,
                TargetTypes: []string{"VIEWS"},
            },
        },
    },
}

Dataset Options

DatasetOption Type

type DatasetOption func(*dsCallOption)

Access Policy Version

func WithAccessPolicyVersion(apv int) DatasetOption

Set the IAM policy version (valid values: 0, 1, 3). Version 3 is required for conditional bindings:

meta, err := dataset.MetadataWithOptions(ctx,
    bigquery.WithAccessPolicyVersion(3))

Dataset View

func WithDatasetView(view DatasetView) DatasetOption

type DatasetView string

const (
    DatasetMetadataView          DatasetView = "METADATA"
    DatasetACLView               DatasetView = "ACL"
    DatasetFullView              DatasetView = "FULL"
    UnspecifiedDatasetView       DatasetView = "DATASET_VIEW_UNSPECIFIED"
)

Control which dataset information is returned:

// Get only metadata (no ACLs)
meta, err := dataset.MetadataWithOptions(ctx,
    bigquery.WithDatasetView(bigquery.DatasetMetadataView))

// Get only ACLs (no metadata)
meta, err := dataset.MetadataWithOptions(ctx,
    bigquery.WithDatasetView(bigquery.DatasetACLView))

Update Mode

func WithUpdateMode(mode DatasetUpdateMode) DatasetOption

type DatasetUpdateMode string

const (
    DatasetMetadataUpdateMode    DatasetUpdateMode = "UPDATE_METADATA"
    DatasetACLUpdateMode         DatasetUpdateMode = "UPDATE_ACL"
    DatasetFullUpdateMode        DatasetUpdateMode = "UPDATE_FULL"
    UnspecifiedDatasetUpdateMode DatasetUpdateMode = "UPDATE_MODE_UNSPECIFIED"
)

Control which fields are updated:

// Update only metadata fields
update := bigquery.DatasetMetadataToUpdate{
    Description: optional.ToString("New description"),
}
meta, err := dataset.UpdateWithOptions(ctx, update, "",
    bigquery.WithUpdateMode(bigquery.DatasetMetadataUpdateMode))

// Update only ACL fields
update := bigquery.DatasetMetadataToUpdate{
    Access: []*bigquery.AccessEntry{/* new ACLs */},
}
meta, err := dataset.UpdateWithOptions(ctx, update, "",
    bigquery.WithUpdateMode(bigquery.DatasetACLUpdateMode))

Encryption Configuration

type EncryptionConfig struct {
    KMSKeyName string
}

Configure Cloud KMS encryption for new tables:

meta := &bigquery.DatasetMetadata{
    DefaultEncryptionConfig: &bigquery.EncryptionConfig{
        KMSKeyName: "projects/my-project/locations/us/keyRings/my-ring/cryptoKeys/my-key",
    },
}
err := dataset.Create(ctx, meta)

External Datasets

type ExternalDatasetReference struct {
    Connection     string
    ExternalSource string
}

Create a dataset backed by an external source:

meta := &bigquery.DatasetMetadata{
    ExternalDatasetReference: &bigquery.ExternalDatasetReference{
        Connection:     "projects/my-project/locations/us/connections/my-connection",
        ExternalSource: "s3://my-bucket/data",
    },
}
err := dataset.Create(ctx, meta)

Dataset Tags

type DatasetTag struct {
    TagKey   string
    TagValue string
}

Tags are managed through the Resource Manager API and are read-only in BigQuery:

meta, err := dataset.Metadata(ctx)
if err != nil {
    return err
}

for _, tag := range meta.Tags {
    fmt.Printf("Tag: %s = %s\n", tag.TagKey, tag.TagValue)
}

Storage Billing Models

const (
    LogicalStorageBillingModel  = ""
    PhysicalStorageBillingModel = "PHYSICAL"
)

Configure storage billing:

// Create with physical storage billing
meta := &bigquery.DatasetMetadata{
    StorageBillingModel: bigquery.PhysicalStorageBillingModel,
}
err := dataset.Create(ctx, meta)

// Update to physical (cannot revert to logical)
update := bigquery.DatasetMetadataToUpdate{
    StorageBillingModel: optional.ToString(bigquery.PhysicalStorageBillingModel),
}
meta, err := dataset.Update(ctx, update, "")

Note: Once changed to physical bytes, you cannot change back to logical bytes.

Complete Example

package main

import (
    "context"
    "fmt"
    "log"
    "time"

    "cloud.google.com/go/bigquery"
    "cloud.google.com/go/internal/optional"
    "google.golang.org/api/iterator"
)

func main() {
    ctx := context.Background()
    client, err := bigquery.NewClient(ctx, "my-project")
    if err != nil {
        log.Fatal(err)
    }
    defer client.Close()

    // Create dataset
    dataset := client.Dataset("analytics_dataset")
    meta := &bigquery.DatasetMetadata{
        Name:        "Analytics Dataset",
        Description: "Dataset for analytics workloads",
        Location:    "US",
        DefaultTableExpiration: 90 * 24 * time.Hour, // 90 days
        Labels: map[string]string{
            "environment": "production",
            "team":        "analytics",
        },
        Access: []*bigquery.AccessEntry{
            {
                Role:       bigquery.OwnerRole,
                EntityType: bigquery.GroupEmailEntity,
                Entity:     "analytics-owners@example.com",
            },
            {
                Role:       bigquery.ReaderRole,
                EntityType: bigquery.GroupEmailEntity,
                Entity:     "analytics-viewers@example.com",
            },
        },
        DefaultEncryptionConfig: &bigquery.EncryptionConfig{
            KMSKeyName: "projects/my-project/locations/us/keyRings/bq-ring/cryptoKeys/bq-key",
        },
    }

    if err := dataset.Create(ctx, meta); err != nil {
        log.Fatal(err)
    }

    // Get metadata
    meta, err = dataset.Metadata(ctx)
    if err != nil {
        log.Fatal(err)
    }
    fmt.Printf("Created: %s\n", meta.CreationTime)
    fmt.Printf("Location: %s\n", meta.Location)

    // Update dataset
    update := bigquery.DatasetMetadataToUpdate{
        Description: optional.ToString("Updated analytics dataset"),
    }
    update.SetLabel("updated", "true")

    meta, err = dataset.Update(ctx, update, meta.ETag)
    if err != nil {
        log.Fatal(err)
    }

    // List all datasets
    it := client.Datasets(ctx)
    it.Filter = "labels.environment:production"
    for {
        ds, err := it.Next()
        if err == iterator.Done {
            break
        }
        if err != nil {
            log.Fatal(err)
        }
        fmt.Printf("Dataset: %s.%s\n", ds.ProjectID, ds.DatasetID)
    }
}

Version