diff --git a/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql b/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql index cc9aeefc63..e913fca77e 100644 --- a/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql +++ b/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql @@ -1429,6 +1429,26 @@ SELECT DISTINCT HL.`HL_BLOCKEDBY_INT_ID` FROM SYS.`HIVE_LOCKS` AS HL; +CREATE EXTERNAL TABLE `REPLICATION_METRICS` ( + `SCHEDULED_EXECUTION_ID` bigint, + `POLICY_NAME` string, + `DUMP_EXECUTION_ID` bigint, + `METADATA` string, + `PROGRESS` string +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"RM_SCHEDULED_EXECUTION_ID\", + \"RM_POLICY\", + \"RM_DUMP_EXECUTION_ID\", + \"RM_METADATA\", + \"RM_PROGRESS\" +FROM \"REPLICATION_METRICS\"" +); + CREATE DATABASE IF NOT EXISTS INFORMATION_SCHEMA; USE INFORMATION_SCHEMA; diff --git a/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql b/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql index 09c95d7b17..18b1d2aeb0 100644 --- a/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql +++ b/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql @@ -490,6 +490,26 @@ SELECT DISTINCT HL.`HL_BLOCKEDBY_INT_ID` FROM SYS.`HIVE_LOCKS` AS HL; +CREATE EXTERNAL TABLE `REPLICATION_METRICS` ( + `SCHEDULED_EXECUTION_ID` bigint, + `POLICY_NAME` string, + `DUMP_EXECUTION_ID` bigint, + `METADATA` string, + `PROGRESS` string +) +STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' +TBLPROPERTIES ( +"hive.sql.database.type" = "METASTORE", +"hive.sql.query" = +"SELECT + \"RM_SCHEDULED_EXECUTION_ID\", + \"RM_POLICY\", + \"RM_DUMP_EXECUTION_ID\", + \"RM_METADATA\", + \"RM_PROGRESS\" +FROM \"REPLICATION_METRICS\"" +); + DROP TABLE IF EXISTS `VERSION`; CREATE OR REPLACE VIEW `VERSION` AS SELECT 1 AS `VER_ID`, '4.0.0' AS `SCHEMA_VERSION`, @@ -766,4 +786,5 @@ WHERE OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) AND P.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer()); + SELECT 'Finished upgrading MetaStore schema from 3.1.0 to 4.0.0'; diff --git a/ql/src/test/queries/clientpositive/replication_metrics_ingest.q b/ql/src/test/queries/clientpositive/replication_metrics_ingest.q new file mode 100644 index 0000000000..7f0ef81857 --- /dev/null +++ b/ql/src/test/queries/clientpositive/replication_metrics_ingest.q @@ -0,0 +1,40 @@ +--! qt:authorizer +--! qt:scheduledqueryservice +--! qt:sysdb + +set hive.repl.rootdir=${system:test.tmp.dir}/repl; + +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/repl/sentinel; +dfs -rmr ${system:test.tmp.dir}/repl; +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/repl; + +set user.name=hive_admin_user; +set role ADMIN; + +drop database if exists src cascade; +drop database if exists destination cascade; + +create database src with DBPROPERTIES ('repl.source.for' = '1,2,3'); + +create table src.t(id int, cnt int); + +-- add data to table +insert into src.t values(1,1); + +create scheduled query repl1 every 15 minutes as repl dump src +with ('hive.repl.rootdir'= '${system:test.tmp.dir}/repl'); + +alter scheduled query repl1 execute; + +!sleep 50; + +create scheduled query repl2 every 15 minutes as repl load src into destination +with ('hive.repl.rootdir'= '${system:test.tmp.dir}/repl'); + +alter scheduled query repl2 execute; + +!sleep 50; + +show databases; + +select scheduled_execution_id, policy_name, dump_execution_id from sys.replication_metrics; diff --git a/ql/src/test/results/clientpositive/replication_metrics_ingest.q.out b/ql/src/test/results/clientpositive/replication_metrics_ingest.q.out new file mode 100644 index 0000000000..1380ccd0cf --- /dev/null +++ b/ql/src/test/results/clientpositive/replication_metrics_ingest.q.out @@ -0,0 +1,76 @@ +#### A masked pattern was here #### +PREHOOK: query: set role ADMIN +PREHOOK: type: SHOW_ROLES +POSTHOOK: query: set role ADMIN +POSTHOOK: type: SHOW_ROLES +PREHOOK: query: drop database if exists src cascade +PREHOOK: type: DROPDATABASE +POSTHOOK: query: drop database if exists src cascade +POSTHOOK: type: DROPDATABASE +PREHOOK: query: drop database if exists destination cascade +PREHOOK: type: DROPDATABASE +POSTHOOK: query: drop database if exists destination cascade +POSTHOOK: type: DROPDATABASE +PREHOOK: query: create database src with DBPROPERTIES ('repl.source.for' = '1,2,3') +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:src +POSTHOOK: query: create database src with DBPROPERTIES ('repl.source.for' = '1,2,3') +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:src +PREHOOK: query: create table src.t(id int, cnt int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:src +PREHOOK: Output: src@t +POSTHOOK: query: create table src.t(id int, cnt int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:src +POSTHOOK: Output: src@t +PREHOOK: query: insert into src.t values(1,1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: src@t +POSTHOOK: query: insert into src.t values(1,1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: src@t +POSTHOOK: Lineage: t.cnt SCRIPT [] +POSTHOOK: Lineage: t.id SCRIPT [] +PREHOOK: query: create scheduled query repl1 every 15 minutes as repl dump src +#### A masked pattern was here #### +PREHOOK: type: CREATE SCHEDULED QUERY +POSTHOOK: query: create scheduled query repl1 every 15 minutes as repl dump src +#### A masked pattern was here #### +POSTHOOK: type: CREATE SCHEDULED QUERY +PREHOOK: query: alter scheduled query repl1 execute +PREHOOK: type: ALTER SCHEDULED QUERY +POSTHOOK: query: alter scheduled query repl1 execute +POSTHOOK: type: ALTER SCHEDULED QUERY +PREHOOK: query: create scheduled query repl2 every 15 minutes as repl load src into destination +#### A masked pattern was here #### +PREHOOK: type: CREATE SCHEDULED QUERY +POSTHOOK: query: create scheduled query repl2 every 15 minutes as repl load src into destination +#### A masked pattern was here #### +POSTHOOK: type: CREATE SCHEDULED QUERY +PREHOOK: query: alter scheduled query repl2 execute +PREHOOK: type: ALTER SCHEDULED QUERY +POSTHOOK: query: alter scheduled query repl2 execute +POSTHOOK: type: ALTER SCHEDULED QUERY +PREHOOK: query: show databases +PREHOOK: type: SHOWDATABASES +POSTHOOK: query: show databases +POSTHOOK: type: SHOWDATABASES +default +destination +information_schema +src +sys +PREHOOK: query: select scheduled_execution_id, policy_name, dump_execution_id from sys.replication_metrics +PREHOOK: type: QUERY +PREHOOK: Input: sys@replication_metrics +#### A masked pattern was here #### +POSTHOOK: query: select scheduled_execution_id, policy_name, dump_execution_id from sys.replication_metrics +POSTHOOK: type: QUERY +POSTHOOK: Input: sys@replication_metrics +#### A masked pattern was here #### +1 repl1 0 +2 repl2 1