0
# Federation Management
1
2
Manage metastore federation services that provide unified access to multiple backend metastores. Supports cross-cloud and multi-region federation scenarios for enterprise data lake architectures, enabling centralized metadata management across diverse metastore environments.
3
4
## Capabilities
5
6
### List Federations
7
8
Retrieve all federation services in a specified location with optional filtering and pagination support.
9
10
```python { .api }
11
def list_federations(
12
self,
13
request: Optional[ListFederationsRequest] = None,
14
*,
15
parent: Optional[str] = None,
16
retry: OptionalRetry = gapic_v1.method.DEFAULT,
17
timeout: Union[float, object] = gapic_v1.method.DEFAULT,
18
metadata: Sequence[Tuple[str, str]] = ()
19
) -> pagers.ListFederationsPager:
20
"""
21
Lists federations in a project and location.
22
23
Args:
24
request: The request object containing list parameters
25
parent: Required. The relative resource name of the location
26
Format: projects/{project_id}/locations/{location_id}
27
retry: Retry configuration for the request
28
timeout: Request timeout in seconds
29
metadata: Additional metadata for the request
30
31
Returns:
32
ListFederationsPager: Pageable list of federations
33
34
Raises:
35
google.api_core.exceptions.GoogleAPICallError: If the request fails
36
"""
37
```
38
39
Usage example:
40
41
```python
42
from google.cloud import metastore
43
44
# Use the federation-specific client
45
federation_client = metastore.DataprocMetastoreFederationClient()
46
parent = "projects/my-project/locations/us-central1"
47
48
# List all federations
49
for federation in federation_client.list_federations(parent=parent):
50
print(f"Federation: {federation.name}")
51
print(f"Version: {federation.version}")
52
print(f"Endpoint URI: {federation.endpoint_uri}")
53
print(f"Backend metastores: {len(federation.backend_metastores)}")
54
55
# With filtering
56
request = metastore.ListFederationsRequest(
57
parent=parent,
58
filter="state=ACTIVE",
59
order_by="create_time desc"
60
)
61
```
62
63
### Get Federation
64
65
Retrieve detailed information about a specific federation including backend metastore configurations.
66
67
```python { .api }
68
def get_federation(
69
self,
70
request: Optional[GetFederationRequest] = None,
71
*,
72
name: Optional[str] = None,
73
retry: OptionalRetry = gapic_v1.method.DEFAULT,
74
timeout: Union[float, object] = gapic_v1.method.DEFAULT,
75
metadata: Sequence[Tuple[str, str]] = ()
76
) -> Federation:
77
"""
78
Gets the details of a single federation.
79
80
Args:
81
request: The request object
82
name: Required. The relative resource name of the federation
83
Format: projects/{project_id}/locations/{location_id}/federations/{federation_id}
84
retry: Retry configuration
85
timeout: Request timeout in seconds
86
metadata: Additional metadata
87
88
Returns:
89
Federation: The federation resource
90
91
Raises:
92
google.api_core.exceptions.NotFound: If the federation doesn't exist
93
"""
94
```
95
96
### Create Federation
97
98
Create a new metastore federation with multiple backend metastores for unified metadata access.
99
100
```python { .api }
101
def create_federation(
102
self,
103
request: Optional[CreateFederationRequest] = None,
104
*,
105
parent: Optional[str] = None,
106
federation: Optional[Federation] = None,
107
federation_id: Optional[str] = None,
108
retry: OptionalRetry = gapic_v1.method.DEFAULT,
109
timeout: Union[float, object] = gapic_v1.method.DEFAULT,
110
metadata: Sequence[Tuple[str, str]] = ()
111
) -> operation.Operation:
112
"""
113
Creates a metastore federation in a project and location.
114
115
Args:
116
request: The request object
117
parent: Required. The relative resource name of the location
118
federation: Required. The federation configuration
119
federation_id: Required. The ID to use for the federation
120
retry: Retry configuration
121
timeout: Request timeout in seconds
122
metadata: Additional metadata
123
124
Returns:
125
Operation: Long-running operation for federation creation
126
127
Raises:
128
google.api_core.exceptions.AlreadyExists: If federation_id already exists
129
google.api_core.exceptions.InvalidArgument: If configuration is invalid
130
"""
131
```
132
133
Usage example:
134
135
```python
136
from google.cloud import metastore
137
138
federation_client = metastore.DataprocMetastoreFederationClient()
139
140
# Configure backend metastores
141
backend_metastores = [
142
metastore.BackendMetastore(
143
name="production-hive",
144
metastore_type=metastore.BackendMetastore.MetastoreType.DATA_METASTORE,
145
hive_metastore_config=metastore.BackendMetastore.HiveMetastoreConfig(
146
endpoint_uri="thrift://prod-metastore.company.com:9083"
147
)
148
),
149
metastore.BackendMetastore(
150
name="staging-metastore",
151
metastore_type=metastore.BackendMetastore.MetastoreType.DATAPROC_METASTORE,
152
dataproc_metastore_config=metastore.BackendMetastore.DataprocMetastoreConfig(
153
metastore_service="projects/my-project/locations/us-west1/services/staging-metastore"
154
)
155
)
156
]
157
158
# Create federation
159
federation_config = metastore.Federation(
160
version="1.0",
161
backend_metastores=backend_metastores,
162
labels={
163
"environment": "multi-region",
164
"purpose": "unified-access"
165
}
166
)
167
168
operation = federation_client.create_federation(
169
parent="projects/my-project/locations/us-central1",
170
federation_id="enterprise-federation",
171
federation=federation_config
172
)
173
174
# Wait for completion
175
federation = operation.result(timeout=600)
176
print(f"Federation created: {federation.name}")
177
print(f"Endpoint URI: {federation.endpoint_uri}")
178
```
179
180
### Update Federation
181
182
Update an existing federation configuration including backend metastore settings.
183
184
```python { .api }
185
def update_federation(
186
self,
187
request: Optional[UpdateFederationRequest] = None,
188
*,
189
federation: Optional[Federation] = None,
190
update_mask: Optional[field_mask_pb2.FieldMask] = None,
191
retry: OptionalRetry = gapic_v1.method.DEFAULT,
192
timeout: Union[float, object] = gapic_v1.method.DEFAULT,
193
metadata: Sequence[Tuple[str, str]] = ()
194
) -> operation.Operation:
195
"""
196
Updates the fields of a federation.
197
198
Args:
199
request: The request object
200
federation: Required. The federation to update
201
update_mask: Required. Field mask specifying which fields to update
202
retry: Retry configuration
203
timeout: Request timeout in seconds
204
metadata: Additional metadata
205
206
Returns:
207
Operation: Long-running operation for federation update
208
209
Raises:
210
google.api_core.exceptions.NotFound: If the federation doesn't exist
211
google.api_core.exceptions.InvalidArgument: If update is invalid
212
"""
213
```
214
215
### Delete Federation
216
217
Delete a metastore federation and remove unified access to backend metastores.
218
219
```python { .api }
220
def delete_federation(
221
self,
222
request: Optional[DeleteFederationRequest] = None,
223
*,
224
name: Optional[str] = None,
225
retry: OptionalRetry = gapic_v1.method.DEFAULT,
226
timeout: Union[float, object] = gapic_v1.method.DEFAULT,
227
metadata: Sequence[Tuple[str, str]] = ()
228
) -> operation.Operation:
229
"""
230
Deletes a single federation.
231
232
Args:
233
request: The request object
234
name: Required. The relative resource name of the federation to delete
235
retry: Retry configuration
236
timeout: Request timeout in seconds
237
metadata: Additional metadata
238
239
Returns:
240
Operation: Long-running operation for federation deletion
241
242
Raises:
243
google.api_core.exceptions.NotFound: If the federation doesn't exist
244
google.api_core.exceptions.FailedPrecondition: If federation cannot be deleted
245
"""
246
```
247
248
## Core Types
249
250
### Federation Resource
251
252
```python { .api }
253
class Federation:
254
name: str
255
create_time: timestamp_pb2.Timestamp
256
update_time: timestamp_pb2.Timestamp
257
labels: Dict[str, str]
258
version: str
259
backend_metastores: MutableMapping[int, BackendMetastore]
260
endpoint_uri: str
261
state: State
262
state_message: str
263
uid: str
264
265
class State(enum.Enum):
266
STATE_UNSPECIFIED = 0
267
CREATING = 1
268
ACTIVE = 2
269
UPDATING = 3
270
DELETING = 4
271
ERROR = 5
272
```
273
274
### Backend Metastore Configuration
275
276
```python { .api }
277
class BackendMetastore:
278
name: str
279
metastore_type: MetastoreType
280
281
class MetastoreType(enum.Enum):
282
METASTORE_TYPE_UNSPECIFIED = 0
283
DATAPROC_METASTORE = 1
284
BIGQUERY = 2
285
DATA_METASTORE = 3 # Third-party Hive metastore
286
287
# Configuration for Dataproc Metastore backend
288
class DataprocMetastoreConfig:
289
metastore_service: str
290
291
# Configuration for BigQuery backend
292
class BigQueryConfig:
293
# BigQuery-specific configuration fields
294
pass
295
296
# Configuration for external Hive metastore
297
class HiveMetastoreConfig:
298
endpoint_uri: str
299
kerberos_config: Optional[KerberosConfig]
300
```
301
302
### Request/Response Types
303
304
```python { .api }
305
class ListFederationsRequest:
306
parent: str
307
page_size: int
308
page_token: str
309
filter: str
310
order_by: str
311
312
class ListFederationsResponse:
313
federations: List[Federation]
314
next_page_token: str
315
unreachable: List[str]
316
317
class GetFederationRequest:
318
name: str
319
320
class CreateFederationRequest:
321
parent: str
322
federation_id: str
323
federation: Federation
324
request_id: str
325
326
class UpdateFederationRequest:
327
update_mask: field_mask_pb2.FieldMask
328
federation: Federation
329
request_id: str
330
331
class DeleteFederationRequest:
332
name: str
333
request_id: str
334
```
335
336
## Usage Patterns
337
338
### Multi-Cloud Federation Setup
339
340
```python
341
from google.cloud import metastore
342
343
def setup_multi_cloud_federation():
344
"""Setup federation across multiple cloud providers and on-premises."""
345
federation_client = metastore.DataprocMetastoreFederationClient()
346
347
# Define backend metastores from different sources
348
backend_metastores = [
349
# Google Cloud Dataproc Metastore
350
metastore.BackendMetastore(
351
name="gcp-production",
352
metastore_type=metastore.BackendMetastore.MetastoreType.DATAPROC_METASTORE,
353
dataproc_metastore_config=metastore.BackendMetastore.DataprocMetastoreConfig(
354
metastore_service="projects/gcp-project/locations/us-central1/services/prod-metastore"
355
)
356
),
357
358
# BigQuery as metastore backend
359
metastore.BackendMetastore(
360
name="bigquery-analytics",
361
metastore_type=metastore.BackendMetastore.MetastoreType.BIGQUERY,
362
bigquery_config=metastore.BackendMetastore.BigQueryConfig()
363
),
364
365
# On-premises Hive metastore
366
metastore.BackendMetastore(
367
name="onprem-hadoop",
368
metastore_type=metastore.BackendMetastore.MetastoreType.DATA_METASTORE,
369
hive_metastore_config=metastore.BackendMetastore.HiveMetastoreConfig(
370
endpoint_uri="thrift://onprem-metastore.company.com:9083",
371
kerberos_config=metastore.KerberosConfig(
372
keytab=metastore.Secret(cloud_secret="projects/my-project/secrets/hive-keytab/versions/latest"),
373
principal="hive/metastore@COMPANY.COM",
374
krb5_config_gcs_uri="gs://my-config/krb5.conf"
375
)
376
)
377
)
378
]
379
380
# Create federation
381
federation_config = metastore.Federation(
382
version="1.0",
383
backend_metastores=backend_metastores,
384
labels={
385
"architecture": "multi-cloud",
386
"use_case": "unified_data_lake"
387
}
388
)
389
390
operation = federation_client.create_federation(
391
parent="projects/my-project/locations/us-central1",
392
federation_id="enterprise-multi-cloud-federation",
393
federation=federation_config
394
)
395
396
return operation.result(timeout=600)
397
```
398
399
### Federation Health Monitoring
400
401
```python
402
import logging
403
from typing import Dict, List
404
from google.cloud import metastore
405
406
class FederationMonitor:
407
def __init__(self, project_id: str, location: str):
408
self.federation_client = metastore.DataprocMetastoreFederationClient()
409
self.parent = f"projects/{project_id}/locations/{location}"
410
411
def check_federation_health(self) -> Dict[str, str]:
412
"""Check health status of all federations."""
413
health_status = {}
414
415
for federation in self.federation_client.list_federations(parent=self.parent):
416
if federation.state == metastore.Federation.State.ACTIVE:
417
health_status[federation.name] = "HEALTHY"
418
elif federation.state == metastore.Federation.State.ERROR:
419
health_status[federation.name] = f"ERROR: {federation.state_message}"
420
logging.error(f"Federation {federation.name} in error state: {federation.state_message}")
421
else:
422
health_status[federation.name] = f"TRANSITIONING: {federation.state.name}"
423
424
return health_status
425
426
def validate_backend_connectivity(self, federation_name: str) -> List[Dict[str, str]]:
427
"""Validate connectivity to all backend metastores in a federation."""
428
federation = self.federation_client.get_federation(name=federation_name)
429
backend_status = []
430
431
for backend in federation.backend_metastores:
432
status = {
433
"name": backend.name,
434
"type": backend.metastore_type.name,
435
"status": "UNKNOWN"
436
}
437
438
# In a real implementation, you would test connectivity
439
# to each backend metastore based on its type and configuration
440
if backend.metastore_type == metastore.BackendMetastore.MetastoreType.DATAPROC_METASTORE:
441
# Test Dataproc Metastore connectivity
442
status["status"] = "CONNECTED"
443
elif backend.metastore_type == metastore.BackendMetastore.MetastoreType.DATA_METASTORE:
444
# Test external Hive metastore connectivity
445
status["status"] = "CONNECTED"
446
447
backend_status.append(status)
448
449
return backend_status
450
```
451
452
### Dynamic Backend Management
453
454
```python
455
def add_backend_to_federation(federation_name: str, new_backend: metastore.BackendMetastore):
456
"""Dynamically add a new backend metastore to an existing federation."""
457
federation_client = metastore.DataprocMetastoreFederationClient()
458
459
# Get current federation configuration
460
current_federation = federation_client.get_federation(name=federation_name)
461
462
# Add new backend to existing list
463
updated_backends = list(current_federation.backend_metastores)
464
updated_backends.append(new_backend)
465
466
# Update federation
467
updated_federation = metastore.Federation(
468
name=current_federation.name,
469
version=current_federation.version,
470
backend_metastores=updated_backends,
471
labels=current_federation.labels
472
)
473
474
update_request = metastore.UpdateFederationRequest(
475
federation=updated_federation,
476
update_mask=field_mask_pb2.FieldMask(paths=["backend_metastores"])
477
)
478
479
operation = federation_client.update_federation(request=update_request)
480
return operation.result(timeout=300)
481
```