0
# Catalog Operations
1

2
Complete Hive metastore integration for managing databases, tables, partitions, and metadata through the `HiveCatalog` class. Provides full compatibility with Hive metastore operations and seamless integration with Flink's catalog system.
3

4
## Capabilities
5

6
### HiveCatalog
7

8
Main catalog implementation providing connection to Hive metastore and all metadata operations.
9

10
```java { .api }
11
/**
12
 * Catalog implementation for Hive metastore integration
13
 * Extends AbstractCatalog to provide Hive-specific metadata operations
14
 */
15
public class HiveCatalog extends AbstractCatalog {
16
    /**
17
     * Creates a new HiveCatalog instance with minimal configuration
18
     * @param catalogName - Name for this catalog instance
19
     * @param defaultDatabase - Default database name (can be null, defaults to "default")
20
     * @param hiveConfDir - Path to directory containing hive-site.xml (can be null)
21
     */
22
    public HiveCatalog(String catalogName, String defaultDatabase, String hiveConfDir);
23
    
24
    /**
25
     * Creates a new HiveCatalog instance with Hive version
26
     * @param catalogName - Name for this catalog instance
27
     * @param defaultDatabase - Default database name (can be null, defaults to "default")
28
     * @param hiveConfDir - Path to directory containing hive-site.xml (can be null)
29
     * @param hiveVersion - Hive version string (e.g., "2.3.6", can be null for auto-detection)
30
     */
31
    public HiveCatalog(String catalogName, String defaultDatabase, String hiveConfDir, String hiveVersion);
32
    
33
    /**
34
     * Creates a new HiveCatalog instance with full configuration
35
     * @param catalogName - Name for this catalog instance
36
     * @param defaultDatabase - Default database name (can be null, defaults to "default")
37
     * @param hiveConfDir - Path to directory containing hive-site.xml (can be null)
38
     * @param hadoopConfDir - Path to Hadoop configuration directory (can be null)
39
     * @param hiveVersion - Hive version string (e.g., "2.3.6", can be null for auto-detection)
40
     */
41
    public HiveCatalog(String catalogName, String defaultDatabase, String hiveConfDir, String hadoopConfDir, String hiveVersion);
42
    
43
    /**
44
     * Creates a new HiveCatalog instance with pre-configured HiveConf
45
     * @param catalogName - Name for this catalog instance
46
     * @param defaultDatabase - Default database name (can be null, defaults to "default")
47
     * @param hiveConf - Pre-configured HiveConf instance (can be null)
48
     * @param hiveVersion - Hive version string (e.g., "2.3.6", can be null for auto-detection)
49
     */
50
    public HiveCatalog(String catalogName, String defaultDatabase, HiveConf hiveConf, String hiveVersion);
51
    
52
    /**
53
     * Opens connection to Hive metastore
54
     * Must be called before using catalog operations
55
     * @throws CatalogException if connection fails
56
     */
57
    public void open() throws CatalogException;
58
    
59
    /**
60
     * Closes connection to Hive metastore
61
     * Should be called when catalog is no longer needed
62
     * @throws CatalogException if close operation fails
63
     */
64
    public void close() throws CatalogException;
65
}
66
```
67

68
### Database Operations
69

70
Operations for managing Hive databases including listing, creation, and metadata retrieval.
71

72
```java { .api }
73
/**
74
 * List all databases in the Hive metastore
75
 * @return List of database names
76
 * @throws CatalogException if operation fails
77
 */
78
public List<String> listDatabases() throws CatalogException;
79

80
/**
81
 * Get database metadata by name
82
 * @param databaseName - Name of the database
83
 * @return CatalogDatabase with metadata
84
 * @throws DatabaseNotExistException if database doesn't exist
85
 * @throws CatalogException if operation fails
86
 */
87
public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException;
88

89
/**
90
 * Create a new database
91
 * @param databaseName - Name for the new database
92
 * @param database - Database metadata
93
 * @param ignoreIfExists - Whether to ignore if database already exists
94
 * @throws DatabaseAlreadyExistException if database exists and ignoreIfExists is false
95
 * @throws CatalogException if operation fails
96
 */
97
public void createDatabase(String databaseName, CatalogDatabase database, boolean ignoreIfExists) 
98
    throws DatabaseAlreadyExistException, CatalogException;
99

100
/**
101
 * Drop an existing database
102
 * @param databaseName - Name of database to drop
103
 * @param ignoreIfNotExists - Whether to ignore if database doesn't exist  
104
 * @param cascade - Whether to drop all tables in the database
105
 * @throws DatabaseNotExistException if database doesn't exist and ignoreIfNotExists is false
106
 * @throws DatabaseNotEmptyException if database has tables and cascade is false
107
 * @throws CatalogException if operation fails
108
 */
109
public void dropDatabase(String databaseName, boolean ignoreIfNotExists, boolean cascade)
110
    throws DatabaseNotExistException, DatabaseNotEmptyException, CatalogException;
111
```
112

113
### Table Operations
114

115
Comprehensive table management including listing, creation, metadata retrieval, and schema operations.
116

117
```java { .api }
118
/**
119
 * List all tables in a database
120
 * @param databaseName - Name of the database
121
 * @return List of table names
122
 * @throws DatabaseNotExistException if database doesn't exist
123
 * @throws CatalogException if operation fails
124
 */
125
public List<String> listTables(String databaseName) throws DatabaseNotExistException, CatalogException;
126

127
/**
128
 * Get table metadata by path
129
 * @param tablePath - Object path containing database and table name
130
 * @return CatalogBaseTable with complete metadata
131
 * @throws TableNotExistException if table doesn't exist
132
 * @throws CatalogException if operation fails
133
 */
134
public CatalogBaseTable getTable(ObjectPath tablePath) throws TableNotExistException, CatalogException;
135

136
/**
137
 * Create a new table
138
 * @param tablePath - Object path for the new table
139
 * @param table - Table definition with schema and properties
140
 * @param ignoreIfExists - Whether to ignore if table already exists
141
 * @throws TableAlreadyExistException if table exists and ignoreIfExists is false
142
 * @throws DatabaseNotExistException if database doesn't exist
143
 * @throws CatalogException if operation fails
144
 */
145
public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ignoreIfExists)
146
    throws TableAlreadyExistException, DatabaseNotExistException, CatalogException;
147

148
/**
149
 * Drop an existing table
150
 * @param tablePath - Object path of table to drop
151
 * @param ignoreIfNotExists - Whether to ignore if table doesn't exist
152
 * @throws TableNotExistException if table doesn't exist and ignoreIfNotExists is false
153
 * @throws CatalogException if operation fails
154
 */
155
public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists)
156
    throws TableNotExistException, CatalogException;
157

158
/**
159
 * Rename an existing table
160
 * @param tablePath - Current object path of the table
161
 * @param newTableName - New name for the table
162
 * @param ignoreIfNotExists - Whether to ignore if table doesn't exist
163
 * @throws TableNotExistException if table doesn't exist and ignoreIfNotExists is false
164
 * @throws TableAlreadyExistException if new name already exists
165
 * @throws CatalogException if operation fails
166
 */
167
public void renameTable(ObjectPath tablePath, String newTableName, boolean ignoreIfNotExists)
168
    throws TableNotExistException, TableAlreadyExistException, CatalogException;
169
```
170

171
### Partition Operations
172

173
Operations for managing table partitions including listing, creation, and metadata management.
174

175
```java { .api }
176
/**
177
 * List all partitions for a table
178
 * @param tablePath - Object path of the table
179
 * @return List of partition specifications
180
 * @throws TableNotExistException if table doesn't exist
181
 * @throws CatalogException if operation fails
182
 */
183
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath)
184
    throws TableNotExistException, CatalogException;
185

186
/**
187
 * List partitions matching a partial specification
188
 * @param tablePath - Object path of the table
189
 * @param partitionSpec - Partial partition specification to match
190
 * @return List of matching partition specifications
191
 * @throws TableNotExistException if table doesn't exist
192
 * @throws CatalogException if operation fails
193
 */
194
public List<CatalogPartitionSpec> listPartitionsByFilter(ObjectPath tablePath, List<Expression> filters)
195
    throws TableNotExistException, CatalogException;
196

197
/**
198
 * Get partition metadata
199
 * @param tablePath - Object path of the table
200
 * @param partitionSpec - Complete partition specification
201
 * @return CatalogPartition with metadata
202
 * @throws PartitionNotExistException if partition doesn't exist
203
 * @throws CatalogException if operation fails
204
 */
205
public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
206
    throws PartitionNotExistException, CatalogException;
207

208
/**
209
 * Create a new partition
210
 * @param tablePath - Object path of the table
211
 * @param partitionSpec - Partition specification
212
 * @param partition - Partition metadata
213
 * @param ignoreIfExists - Whether to ignore if partition already exists
214
 * @throws PartitionAlreadyExistException if partition exists and ignoreIfExists is false
215
 * @throws TableNotExistException if table doesn't exist
216
 * @throws CatalogException if operation fails
217
 */
218
public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists)
219
    throws PartitionAlreadyExistException, TableNotExistException, CatalogException;
220

221
/**
222
 * Drop an existing partition
223
 * @param tablePath - Object path of the table
224
 * @param partitionSpec - Partition specification to drop
225
 * @param ignoreIfNotExists - Whether to ignore if partition doesn't exist
226
 * @throws PartitionNotExistException if partition doesn't exist and ignoreIfNotExists is false
227
 * @throws CatalogException if operation fails
228
 */
229
public void dropPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists)
230
    throws PartitionNotExistException, CatalogException;
231
```
232

233
### Alter Operations
234

235
Operations for modifying existing database, table, and partition metadata.
236

237
```java { .api }
238
/**
239
 * Alter an existing database
240
 * @param databaseName - Name of the database to alter
241
 * @param newDatabase - New database metadata
242
 * @param ignoreIfNotExists - Whether to ignore if database doesn't exist
243
 * @throws DatabaseNotExistException if database doesn't exist and ignoreIfNotExists is false
244
 * @throws CatalogException if operation fails
245
 */
246
public void alterDatabase(String databaseName, CatalogDatabase newDatabase, boolean ignoreIfNotExists)
247
    throws DatabaseNotExistException, CatalogException;
248

249
/**
250
 * Alter an existing table
251
 * @param tablePath - Object path of the table to alter
252
 * @param newCatalogTable - New table metadata
253
 * @param ignoreIfNotExists - Whether to ignore if table doesn't exist
254
 * @throws TableNotExistException if table doesn't exist and ignoreIfNotExists is false
255
 * @throws CatalogException if operation fails
256
 */
257
public void alterTable(ObjectPath tablePath, CatalogBaseTable newCatalogTable, boolean ignoreIfNotExists)
258
    throws TableNotExistException, CatalogException;
259

260
/**
261
 * Alter an existing partition
262
 * @param tablePath - Object path of the table
263
 * @param partitionSpec - Partition specification to alter
264
 * @param newPartition - New partition metadata
265
 * @param ignoreIfNotExists - Whether to ignore if partition doesn't exist
266
 * @throws PartitionNotExistException if partition doesn't exist and ignoreIfNotExists is false
267
 * @throws CatalogException if operation fails
268
 */
269
public void alterPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition newPartition, boolean ignoreIfNotExists)
270
    throws PartitionNotExistException, CatalogException;
271

272
/**
273
 * Alter an existing function
274
 * @param functionPath - Object path of the function to alter
275
 * @param newFunction - New function metadata
276
 * @param ignoreIfNotExists - Whether to ignore if function doesn't exist
277
 * @throws FunctionNotExistException if function doesn't exist and ignoreIfNotExists is false
278
 * @throws CatalogException if operation fails
279
 */
280
public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, boolean ignoreIfNotExists)
281
    throws FunctionNotExistException, CatalogException;
282
```
283

284
### Existence Check Operations
285

286
Operations for checking if databases, tables, and partitions exist.
287

288
```java { .api }
289
/**
290
 * Check if a database exists
291
 * @param databaseName - Name of the database to check
292
 * @return true if database exists, false otherwise
293
 * @throws CatalogException if operation fails
294
 */
295
public boolean databaseExists(String databaseName) throws CatalogException;
296

297
/**
298
 * Check if a table exists
299
 * @param tablePath - Object path of the table to check
300
 * @return true if table exists, false otherwise
301
 * @throws CatalogException if operation fails
302
 */
303
public boolean tableExists(ObjectPath tablePath) throws CatalogException;
304

305
/**
306
 * Check if a partition exists
307
 * @param tablePath - Object path of the table
308
 * @param partitionSpec - Partition specification to check
309
 * @return true if partition exists, false otherwise
310
 * @throws CatalogException if operation fails
311
 */
312
public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException;
313
```
314

315
### Statistics Operations
316

317
Operations for managing table and partition statistics.
318

319
```java { .api }
320
/**
321
 * Get table statistics
322
 * @param tablePath - Object path of the table
323
 * @return CatalogTableStatistics with table statistics
324
 * @throws TableNotExistException if table doesn't exist
325
 * @throws CatalogException if operation fails
326
 */
327
public CatalogTableStatistics getTableStatistics(ObjectPath tablePath)
328
    throws TableNotExistException, CatalogException;
329

330
/**
331
 * Get table column statistics
332
 * @param tablePath - Object path of the table
333
 * @return CatalogColumnStatistics with column statistics
334
 * @throws TableNotExistException if table doesn't exist
335
 * @throws CatalogException if operation fails
336
 */
337
public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath)
338
    throws TableNotExistException, CatalogException;
339

340
/**
341
 * Get partition statistics
342
 * @param tablePath - Object path of the table
343
 * @param partitionSpec - Partition specification
344
 * @return CatalogTableStatistics with partition statistics
345
 * @throws PartitionNotExistException if partition doesn't exist
346
 * @throws CatalogException if operation fails
347
 */
348
public CatalogTableStatistics getPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
349
    throws PartitionNotExistException, CatalogException;
350

351
/**
352
 * Get partition column statistics
353
 * @param tablePath - Object path of the table
354
 * @param partitionSpec - Partition specification
355
 * @return CatalogColumnStatistics with partition column statistics
356
 * @throws PartitionNotExistException if partition doesn't exist
357
 * @throws CatalogException if operation fails
358
 */
359
public CatalogColumnStatistics getPartitionColumnStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
360
    throws PartitionNotExistException, CatalogException;
361

362
/**
363
 * Alter table statistics
364
 * @param tablePath - Object path of the table
365
 * @param tableStatistics - New table statistics
366
 * @param ignoreIfNotExists - Whether to ignore if table doesn't exist
367
 * @throws TableNotExistException if table doesn't exist and ignoreIfNotExists is false
368
 * @throws CatalogException if operation fails
369
 */
370
public void alterTableStatistics(ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists)
371
    throws TableNotExistException, CatalogException;
372

373
/**
374
 * Alter table column statistics
375
 * @param tablePath - Object path of the table
376
 * @param columnStatistics - New column statistics
377
 * @param ignoreIfNotExists - Whether to ignore if table doesn't exist
378
 * @throws TableNotExistException if table doesn't exist and ignoreIfNotExists is false
379
 * @throws TablePartitionedException if table is partitioned
380
 * @throws CatalogException if operation fails
381
 */
382
public void alterTableColumnStatistics(ObjectPath tablePath, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists)
383
    throws TableNotExistException, TablePartitionedException, CatalogException;
384

385
/**
386
 * Alter partition statistics
387
 * @param tablePath - Object path of the table
388
 * @param partitionSpec - Partition specification
389
 * @param partitionStatistics - New partition statistics
390
 * @param ignoreIfNotExists - Whether to ignore if partition doesn't exist
391
 * @throws PartitionNotExistException if partition doesn't exist and ignoreIfNotExists is false
392
 * @throws CatalogException if operation fails
393
 */
394
public void alterPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogTableStatistics partitionStatistics, boolean ignoreIfNotExists)
395
    throws PartitionNotExistException, CatalogException;
396

397
/**
398
 * Alter partition column statistics
399
 * @param tablePath - Object path of the table
400
 * @param partitionSpec - Partition specification
401
 * @param columnStatistics - New column statistics
402
 * @param ignoreIfNotExists - Whether to ignore if partition doesn't exist
403
 * @throws PartitionNotExistException if partition doesn't exist and ignoreIfNotExists is false
404
 * @throws CatalogException if operation fails
405
 */
406
public void alterPartitionColumnStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists)
407
    throws PartitionNotExistException, CatalogException;
408
```
409

410
### Function Operations
411

412
Management of user-defined functions in the Hive metastore.
413

414
```java { .api }
415
/**
416
 * List all functions in a database
417
 * @param databaseName - Name of the database
418
 * @return List of function names
419
 * @throws DatabaseNotExistException if database doesn't exist
420
 * @throws CatalogException if operation fails
421
 */
422
public List<String> listFunctions(String databaseName) throws DatabaseNotExistException, CatalogException;
423

424
/**
425
 * Get function metadata
426
 * @param functionPath - Object path of the function
427
 * @return CatalogFunction with metadata
428
 * @throws FunctionNotExistException if function doesn't exist
429
 * @throws CatalogException if operation fails
430
 */
431
public CatalogFunction getFunction(ObjectPath functionPath) throws FunctionNotExistException, CatalogException;
432

433
/**
434
 * Create a new function
435
 * @param functionPath - Object path for the new function
436
 * @param function - Function definition
437
 * @param ignoreIfExists - Whether to ignore if function already exists
438
 * @throws FunctionAlreadyExistException if function exists and ignoreIfExists is false
439
 * @throws DatabaseNotExistException if database doesn't exist
440
 * @throws CatalogException if operation fails
441
 */
442
public void createFunction(ObjectPath functionPath, CatalogFunction function, boolean ignoreIfExists)
443
    throws FunctionAlreadyExistException, DatabaseNotExistException, CatalogException;
444
```
445

446
### Utility Methods
447

448
Additional utility methods for Hive-specific operations.
449

450
```java { .api }
451
/**
452
 * Check if a table is a Hive table based on its properties
453
 * @param tableOptions - Map of table properties
454
 * @return true if this is a Hive table
455
 */
456
public static boolean isHiveTable(Map<String, String> tableOptions);
457

458
/**
459
 * Get the Hive configuration
460
 * @return HiveConf instance used by this catalog
461
 */
462
public HiveConf getHiveConf();
463

464
/**
465
 * Get the Hive version
466
 * @return Version string for this Hive installation
467
 */
468
public String getHiveVersion();
469
```
470

471
**Usage Examples:**
472

473
```java
474
import org.apache.flink.table.api.TableEnvironment;
475
import org.apache.flink.table.catalog.hive.HiveCatalog;
476
import org.apache.flink.table.catalog.ObjectPath;
477

478
// Create and register Hive catalog
479
HiveCatalog hiveCatalog = new HiveCatalog(
480
    "hive_catalog", 
481
    "default",
482
    "/opt/hive/conf",
483
    "/opt/hadoop/etc/hadoop", 
484
    "2.3.6"
485
);
486

487
TableEnvironment tableEnv = TableEnvironment.create(EnvironmentSettings.inBatchMode());
488
tableEnv.registerCatalog("hive_catalog", hiveCatalog);
489
tableEnv.useCatalog("hive_catalog");
490

491
// List databases
492
List<String> databases = hiveCatalog.listDatabases();
493
System.out.println("Available databases: " + databases);
494

495
// List tables in default database  
496
List<String> tables = hiveCatalog.listTables("default");
497
System.out.println("Tables in default database: " + tables);
498

499
// Get table metadata
500
ObjectPath tablePath = new ObjectPath("default", "my_table");
501
CatalogBaseTable table = hiveCatalog.getTable(tablePath);
502
System.out.println("Table schema: " + table.getSchema());
503

504
// List partitions
505
List<CatalogPartitionSpec> partitions = hiveCatalog.listPartitions(tablePath);
506
System.out.println("Available partitions: " + partitions.size());
507
```
508

509
## Types
510

511
```java { .api }
512
public class ObjectPath {
513
    public ObjectPath(String databaseName, String objectName);
514
    public String getDatabaseName();
515
    public String getObjectName();
516
}
517

518
public interface CatalogDatabase {
519
    Map<String, String> getProperties();
520
    String getComment();
521
}
522

523
public class CatalogDatabaseImpl implements CatalogDatabase {
524
    public CatalogDatabaseImpl(Map<String, String> properties, String comment);
525
}
526

527
public interface CatalogPartitionSpec {
528
    Map<String, String> getPartitionSpec(); 
529
}
530

531
public interface CatalogPartition {
532
    Map<String, String> getProperties();
533
    String getComment();
534
}
535
```