Complete Hive metastore integration for metadata operations including database, table, partition, and function management. HiveCatalog provides seamless access to existing Hive table definitions and schemas.
Main catalog implementation that integrates with Hive metastore for comprehensive metadata management.
/**
* Hive catalog implementation for metadata operations with Hive metastore
*/
public class HiveCatalog extends AbstractCatalog {
/**
* Create HiveCatalog with basic configuration
* @param catalogName - Name of the catalog
* @param defaultDatabase - Default database name
* @param hiveConfDir - Path to Hive configuration directory
*/
public HiveCatalog(String catalogName, String defaultDatabase, String hiveConfDir);
/**
* Create HiveCatalog with specific Hive version
* @param catalogName - Name of the catalog
* @param defaultDatabase - Default database name
* @param hiveConfDir - Path to Hive configuration directory
* @param hiveVersion - Hive version specification
*/
public HiveCatalog(String catalogName, String defaultDatabase, String hiveConfDir, String hiveVersion);
/**
* Create HiveCatalog with Hadoop configuration
* @param catalogName - Name of the catalog
* @param defaultDatabase - Default database name
* @param hiveConfDir - Path to Hive configuration directory
* @param hadoopConfDir - Path to Hadoop configuration directory
* @param hiveVersion - Hive version specification
*/
public HiveCatalog(String catalogName, String defaultDatabase, String hiveConfDir, String hadoopConfDir, String hiveVersion);
/**
* Create HiveCatalog with pre-configured HiveConf
* @param catalogName - Name of the catalog
* @param defaultDatabase - Default database name
* @param hiveConf - Pre-configured Hive configuration object
* @param hiveVersion - Hive version specification
*/
public HiveCatalog(String catalogName, String defaultDatabase, HiveConf hiveConf, String hiveVersion);
/** Open catalog connection */
public void open() throws CatalogException;
/** Close catalog connection */
public void close() throws CatalogException;
/** Get Hive configuration object */
public HiveConf getHiveConf();
}Usage Examples:
import org.apache.flink.table.catalog.hive.HiveCatalog;
import org.apache.flink.table.api.TableEnvironment;
import org.apache.hadoop.hive.conf.HiveConf;
// Basic catalog setup
HiveCatalog catalog = new HiveCatalog("myhive", "default", "/opt/hive/conf");
// With specific Hive version
HiveCatalog catalog = new HiveCatalog("myhive", "default", "/opt/hive/conf", "3.1.2");
// With Hadoop configuration
HiveCatalog catalog = new HiveCatalog(
"myhive",
"default",
"/opt/hive/conf",
"/opt/hadoop/conf",
"3.1.2"
);
// Register catalog with table environment
TableEnvironment tableEnv = TableEnvironment.create(settings);
tableEnv.registerCatalog("myhive", catalog);
tableEnv.useCatalog("myhive");Manage Hive databases through the catalog interface.
/** Create a new database */
public void createDatabase(String databaseName, CatalogDatabase database, boolean ignoreIfExists)
throws DatabaseAlreadyExistException, CatalogException;
/** Drop an existing database */
public void dropDatabase(String databaseName, boolean ignoreIfNotExists, boolean cascade)
throws DatabaseNotExistException, DatabaseNotEmptyException, CatalogException;
/** List all databases */
public List<String> listDatabases() throws CatalogException;
/** Check if database exists */
public boolean databaseExists(String databaseName) throws CatalogException;
/** Get database metadata */
public CatalogDatabase getDatabase(String databaseName)
throws DatabaseNotExistException, CatalogException;Comprehensive table metadata management including creation, modification, and discovery.
/** Create a new table */
public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ignoreIfExists)
throws TableAlreadyExistException, DatabaseNotExistException, CatalogException;
/** Alter an existing table */
public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean ignoreIfNotExists)
throws TableNotExistException, CatalogException;
/** Drop an existing table */
public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists)
throws TableNotExistException, CatalogException;
/** List all tables in a database */
public List<String> listTables(String databaseName)
throws DatabaseNotExistException, CatalogException;
/** Check if table exists */
public boolean tableExists(ObjectPath tablePath) throws CatalogException;
/** Get table metadata */
public CatalogBaseTable getTable(ObjectPath tablePath)
throws TableNotExistException, CatalogException;Manage Hive table partitions for optimized data access.
/** Create a new partition */
public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec,
CatalogPartition partition, boolean ignoreIfExists)
throws TableNotExistException, PartitionAlreadyExistsException, CatalogException;
/** Drop an existing partition */
public void dropPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists)
throws PartitionNotExistException, CatalogException;
/** List all partitions for a table */
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath)
throws TableNotExistException, TableNotPartitionedException, CatalogException;
/** List partitions matching specification */
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
throws TableNotExistException, TableNotPartitionedException, CatalogException;
/** Check if partition exists */
public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
throws CatalogException;
/** Get partition metadata */
public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
throws PartitionNotExistException, CatalogException;Manage Hive user-defined functions through the catalog.
/** Create a new function */
public void createFunction(ObjectPath functionPath, CatalogFunction function, boolean ignoreIfExists)
throws FunctionAlreadyExistException, DatabaseNotExistException, CatalogException;
/** Alter an existing function */
public void alterFunction(ObjectPath functionPath, CatalogFunction newFunction, boolean ignoreIfNotExists)
throws FunctionNotExistException, CatalogException;
/** Drop an existing function */
public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists)
throws FunctionNotExistException, CatalogException;
/** List all functions in a database */
public List<String> listFunctions(String databaseName)
throws DatabaseNotExistException, CatalogException;
/** Check if function exists */
public boolean functionExists(ObjectPath functionPath) throws CatalogException;
/** Get function metadata */
public CatalogFunction getFunction(ObjectPath functionPath)
throws FunctionNotExistException, CatalogException;Factory for creating HiveCatalog instances from configuration properties.
public class HiveCatalogFactory implements CatalogFactory {
/** Create catalog from configuration */
public Catalog createCatalog(String name, Map<String, String> properties);
/** Get factory identifier */
public String factoryIdentifier();
/** Get required configuration options */
public Set<ConfigOption<?>> requiredOptions();
/** Get optional configuration options */
public Set<ConfigOption<?>> optionalOptions();
}Table and partition statistics management for query optimization and planning.
/** Get table statistics */
public CatalogTableStatistics getTableStatistics(ObjectPath tablePath)
throws TableNotExistException, CatalogException;
/** Get table column statistics */
public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath)
throws TableNotExistException, CatalogException;
/** Update table statistics */
public void alterTableStatistics(ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists)
throws TableNotExistException, CatalogException;
/** Update table column statistics */
public void alterTableColumnStatistics(ObjectPath tablePath, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists)
throws TableNotExistException, CatalogException, TablePartitionedException;
/** Get partition statistics */
public CatalogTableStatistics getPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
throws PartitionNotExistException, CatalogException;
/** Get partition column statistics */
public CatalogColumnStatistics getPartitionColumnStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
throws PartitionNotExistException, CatalogException;
/** Update partition statistics */
public void alterPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogTableStatistics partitionStatistics, boolean ignoreIfNotExists)
throws PartitionNotExistException, CatalogException;
/** Update partition column statistics */
public void alterPartitionColumnStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists)
throws PartitionNotExistException, CatalogException;Configuration constants for HiveCatalog factory and behavior.
public class HiveCatalogFactoryOptions {
/** Factory identifier constant */
public static final String IDENTIFIER = "hive";
/** Default database configuration option */
public static final ConfigOption<String> DEFAULT_DATABASE;
/** Hive configuration directory option */
public static final ConfigOption<String> HIVE_CONF_DIR;
/** Hive version specification option */
public static final ConfigOption<String> HIVE_VERSION;
/** Hadoop configuration directory option */
public static final ConfigOption<String> HADOOP_CONF_DIR;
}
public class HiveCatalogConfig {
/** Comment metadata key */
public static final String COMMENT = "comment";
/** Column types separator */
public static final String DEFAULT_LIST_COLUMN_TYPES_SEPARATOR = ":";
/** Partition location metadata key */
public static final String PARTITION_LOCATION = "partition.location";
}