Flink Version | Description |
1.11 | Not supported |
1.13 | Hive v2.2.0, v2.3.2, v2.3.5, and v3.1.1 supported |
1.14 | Not supported |
hive --service metastore
: Activate the Hive metastore service.ps -ef|grep metastore
: Check whether the service is successfully activated._dc
directory, and click Create Hive catalog.
Upload the configuration files hive-site.xml
(adding urls
to it), hdfs-site.xml
, hivemetastore-site.xml
, and hiveserver2-site.xml
(download them here).catalog_name.database_name
.CREATE DATABASE IF NOT EXISTS `hiveCatalogName`.`databaseName`;
catalog_name.database_name.table_name
.CREATE TABLE IF NOT EXISTS `hiveCatalogName`.`databaseName`.`tableName` (user_id INT,item_id INT,category_id INT,-- ts AS localtimestamp,-- WATERMARK FOR ts AS ts,behavior VARCHAR) WITH ('connector' = 'datagen','rows-per-second' = '1', -- The number of records generated per second'fields.user_id.kind' = 'sequence', -- Whether a bounded sequence (if yes, the output automatically stops after the sequence ends)'fields.user_id.start' = '1', -- The start value of the sequence'fields.user_id.end' = '10000', -- The end value of the sequence'fields.item_id.kind' = 'random', -- A random number without range'fields.item_id.min' = '1', -- The minimum random number'fields.item_id.max' = '1000', -- The maximum random number'fields.category_id.kind' = 'random', -- A random number without range'fields.category_id.min' = '1', -- The minimum random number'fields.category_id.max' = '1000', -- The maximum random number'fields.behavior.length' = '5' -- Random string length);
INSERT INTO`hiveCatalogName`.`databaseName`.`sink_tableName`SELECT*FROM`hiveCatalogName`.`databaseName`.`source_tableName`;
useradd flinkgroupadd supergroupusermod -a -G supergroup flinkhdfs dfsadmin -refreshUserToGroupsMappings
hive-site.xml
:<property><name>hive.metastore.authorization.storage.checks</name><value>true</value><description>Should the metastore do authorization checks againstthe underlying storage for operations like drop-partition (disallowthe drop-partition if the user in question doesn't have permissionsto delete the corresponding directory on the storage).</description><property>
Was this page helpful?