tessl/pypi-google-cloud-dataproc-metastore

Google Cloud Dataproc Metastore API client library for managing fully managed, highly available metastore services

—

Pending

Overview

Eval results

Files

Service Management

Name: tessl/pypi-google-cloud-dataproc-metastore
Author: tessl

Comprehensive lifecycle management for Dataproc Metastore services including creation, configuration, updates, and deletion. Supports multiple service tiers, Hive metastore versions, advanced networking options, security configurations, and scaling parameters.

Capabilities

List Services

Retrieve all metastore services in a specified location with optional filtering and pagination support.

def list_services(
    self,
    request: Optional[ListServicesRequest] = None,
    *,
    parent: Optional[str] = None,
    retry: OptionalRetry = gapic_v1.method.DEFAULT,
    timeout: Union[float, object] = gapic_v1.method.DEFAULT,
    metadata: Sequence[Tuple[str, Union[str, bytes]]] = ()
) -> pagers.ListServicesPager:
    """
    Lists services in a project and location.

    Args:
        request: The request object containing list parameters
        parent: Required. The relative resource name of the location
                Format: projects/{project_id}/locations/{location_id}
        retry: Retry configuration for the request
        timeout: Request timeout in seconds
        metadata: Additional metadata for the request

    Returns:
        ListServicesPager: Pageable list of services

    Raises:
        google.api_core.exceptions.GoogleAPICallError: If the request fails
    """

Usage example:

from google.cloud import metastore

client = metastore.DataprocMetastoreClient()
parent = "projects/my-project/locations/us-central1"

# List all services
for service in client.list_services(parent=parent):
    print(f"Service: {service.name}")
    print(f"State: {service.state.name}")
    print(f"Tier: {service.tier.name}")

# With pagination control
request = metastore.ListServicesRequest(
    parent=parent,
    page_size=10,
    filter="state=ACTIVE"
)
page_result = client.list_services(request=request)

Get Service

Retrieve detailed information about a specific metastore service including configuration, state, and endpoint details.

def get_service(
    self,
    request: Optional[GetServiceRequest] = None,
    *,
    name: Optional[str] = None,
    retry: OptionalRetry = gapic_v1.method.DEFAULT,
    timeout: Union[float, object] = gapic_v1.method.DEFAULT,
    metadata: Sequence[Tuple[str, Union[str, bytes]]] = ()
) -> Service:
    """
    Gets the details of a single service.

    Args:
        request: The request object
        name: Required. The relative resource name of the metastore service
              Format: projects/{project_id}/locations/{location_id}/services/{service_id}
        retry: Retry configuration
        timeout: Request timeout in seconds
        metadata: Additional metadata

    Returns:
        Service: The service resource

    Raises:
        google.api_core.exceptions.NotFound: If the service doesn't exist
    """

Create Service

Create a new metastore service with comprehensive configuration options including networking, security, and Hive metastore settings.

def create_service(
    self,
    request: Optional[CreateServiceRequest] = None,
    *,
    parent: Optional[str] = None,
    service: Optional[Service] = None,
    service_id: Optional[str] = None,
    retry: OptionalRetry = gapic_v1.method.DEFAULT,
    timeout: Union[float, object] = gapic_v1.method.DEFAULT,
    metadata: Sequence[Tuple[str, Union[str, bytes]]] = ()
) -> operation.Operation:
    """
    Creates a metastore service in a project and location.

    Args:
        request: The request object
        parent: Required. The relative resource name of the location
        service: Required. The service configuration
        service_id: Required. The ID to use for the service
        retry: Retry configuration
        timeout: Request timeout in seconds
        metadata: Additional metadata

    Returns:
        Operation: Long-running operation for service creation

    Raises:
        google.api_core.exceptions.AlreadyExists: If service_id already exists
        google.api_core.exceptions.InvalidArgument: If configuration is invalid
    """

Usage example:

from google.cloud import metastore

client = metastore.DataprocMetastoreClient()

# Configure the service
service_config = metastore.Service(
    tier=metastore.Service.Tier.ENTERPRISE,
    hive_metastore_config=metastore.HiveMetastoreConfig(
        version="3.1.0",
        config_overrides={
            "javax.jdo.option.ConnectionURL": "jdbc:mysql://...",
            "hive.metastore.uris": "thrift://metastore:9083"
        }
    ),
    network_config=metastore.NetworkConfig(
        consumers=[
            metastore.NetworkConfig.Consumer(
                subnetwork="projects/my-project/regions/us-central1/subnetworks/default"
            )
        ]
    ),
    encryption_config=metastore.EncryptionConfig(
        kms_key="projects/my-project/locations/us-central1/keyRings/my-ring/cryptoKeys/my-key"
    ),
    maintenance_window=metastore.MaintenanceWindow(
        hour_of_day=2,
        day_of_week=dayofweek_pb2.DayOfWeek.SUNDAY
    )
)

# Create the service
operation = client.create_service(
    parent="projects/my-project/locations/us-central1",
    service_id="my-metastore",
    service=service_config
)

# Wait for completion (can take 20-30 minutes)
service = operation.result(timeout=1800)
print(f"Service created: {service.name}")
print(f"Endpoint URI: {service.endpoint_uri}")

Update Service

Update an existing metastore service configuration including scaling, network settings, and maintenance windows.

def update_service(
    self,
    request: Optional[UpdateServiceRequest] = None,
    *,
    service: Optional[Service] = None,
    update_mask: Optional[field_mask_pb2.FieldMask] = None,
    retry: OptionalRetry = gapic_v1.method.DEFAULT,
    timeout: Union[float, object] = gapic_v1.method.DEFAULT,
    metadata: Sequence[Tuple[str, Union[str, bytes]]] = ()
) -> operation.Operation:
    """
    Updates the parameters of a single service.

    Args:
        request: The request object
        service: Required. The service to update
        update_mask: Required. Field mask specifying which fields to update
        retry: Retry configuration
        timeout: Request timeout in seconds
        metadata: Additional metadata

    Returns:
        Operation: Long-running operation for service update

    Raises:
        google.api_core.exceptions.NotFound: If the service doesn't exist
        google.api_core.exceptions.InvalidArgument: If update is invalid
    """

Delete Service

Delete a metastore service and all associated data including backups and metadata.

def delete_service(
    self,
    request: Optional[DeleteServiceRequest] = None,
    *,
    name: Optional[str] = None,
    retry: OptionalRetry = gapic_v1.method.DEFAULT,
    timeout: Union[float, object] = gapic_v1.method.DEFAULT,
    metadata: Sequence[Tuple[str, Union[str, bytes]]] = ()
) -> operation.Operation:
    """
    Deletes a single service.

    Args:
        request: The request object
        name: Required. The relative resource name of the service to delete
        retry: Retry configuration
        timeout: Request timeout in seconds
        metadata: Additional metadata

    Returns:
        Operation: Long-running operation for service deletion

    Raises:
        google.api_core.exceptions.NotFound: If the service doesn't exist
        google.api_core.exceptions.FailedPrecondition: If service cannot be deleted
    """

Core Types

Service Configuration

class Service:
    name: str
    create_time: timestamp_pb2.Timestamp
    update_time: timestamp_pb2.Timestamp
    labels: Dict[str, str]
    hive_metastore_config: HiveMetastoreConfig
    network: str
    endpoint_uri: str
    port: int
    state: State
    state_message: str
    artifact_gcs_uri: str
    tier: Tier
    maintenance_window: Optional[MaintenanceWindow]
    uid: str
    metadata_management_activity: Optional[MetadataManagementActivity]
    release_channel: ReleaseChannel
    encryption_config: Optional[EncryptionConfig]
    network_config: Optional[NetworkConfig]
    database_type: DatabaseType
    telemetry_config: Optional[TelemetryConfig]
    scaling_config: Optional[ScalingConfig]

    class State(enum.Enum):
        STATE_UNSPECIFIED = 0
        CREATING = 1
        ACTIVE = 2
        SUSPENDING = 3
        SUSPENDED = 4
        UPDATING = 5
        DELETING = 6
        ERROR = 7

    class Tier(enum.Enum):
        TIER_UNSPECIFIED = 0
        DEVELOPER = 1
        ENTERPRISE = 3

    class ReleaseChannel(enum.Enum):
        RELEASE_CHANNEL_UNSPECIFIED = 0
        CANARY = 1
        STABLE = 2

    class DatabaseType(enum.Enum):
        DATABASE_TYPE_UNSPECIFIED = 0
        MYSQL = 1
        SPANNER = 2

Hive Metastore Configuration

class HiveMetastoreConfig:
    version: str
    config_overrides: Dict[str, str]
    kerberos_config: Optional[KerberosConfig]
    auxiliary_versions: MutableMapping[str, AuxiliaryVersionConfig]
    endpoint_protocol: EndpointProtocol

    class EndpointProtocol(enum.Enum):
        ENDPOINT_PROTOCOL_UNSPECIFIED = 0
        THRIFT = 1
        GRPC = 2

class KerberosConfig:
    keytab: Secret
    principal: str
    krb5_config_gcs_uri: str

class Secret:
    cloud_secret: str

class AuxiliaryVersionConfig:
    version: str
    config_overrides: Dict[str, str]
    network_config: Optional[NetworkConfig]

Network Configuration

class NetworkConfig:
    consumers: List[Consumer]

    class Consumer:
        subnetwork: str
        endpoint_uri: str
        endpoint_location: str

Other Configuration Types

class EncryptionConfig:
    kms_key: str

class MaintenanceWindow:
    hour_of_day: wrappers_pb2.Int32Value
    day_of_week: DayOfWeek

    class DayOfWeek(enum.Enum):
        DAY_OF_WEEK_UNSPECIFIED = 0
        MONDAY = 1
        TUESDAY = 2
        WEDNESDAY = 3
        THURSDAY = 4
        FRIDAY = 5
        SATURDAY = 6
        SUNDAY = 7

class TelemetryConfig:
    log_format: LogFormat

    class LogFormat(enum.Enum):
        LOG_FORMAT_UNSPECIFIED = 0
        LEGACY = 1
        JSON = 2

class ScalingConfig:
    instance_size: InstanceSize
    scaling_factor: Optional[float]

    class InstanceSize(enum.Enum):
        INSTANCE_SIZE_UNSPECIFIED = 0
        EXTRA_SMALL = 1
        SMALL = 2
        MEDIUM = 3
        LARGE = 4
        EXTRA_LARGE = 5

Request/Response Types

class ListServicesRequest:
    parent: str
    page_size: int
    page_token: str
    filter: str
    order_by: str

class ListServicesResponse:
    services: List[Service]
    next_page_token: str
    unreachable: List[str]

class GetServiceRequest:
    name: str

class CreateServiceRequest:
    parent: str
    service_id: str
    service: Service
    request_id: str

class UpdateServiceRequest:
    update_mask: field_mask_pb2.FieldMask
    service: Service
    request_id: str

class DeleteServiceRequest:
    name: str
    request_id: str

Install with Tessl CLI

npx tessl i tessl/pypi-google-cloud-dataproc-metastore

docs

async-operations.md

backup-restore.md

federation-management.md

index.md

metadata-import-export.md

metadata-query.md

service-management.md

tile.json

tessl/pypi-google-cloud-dataproc-metastore