diff --git a/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql b/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql index cc9aeefc63..1c7354d61e 100644 --- a/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql +++ b/metastore/scripts/upgrade/hive/hive-schema-4.0.0.hive.sql @@ -1834,5 +1834,20 @@ WHERE SE.SCHEDULED_QUERY_ID=SQ.SCHEDULED_QUERY_ID; +CREATE OR REPLACE VIEW `REPLICATION_METRICS` ( + `SCHEDULED_EXECUTION_ID`, + `POLICY_NAME`, + `DUMP_EXECUTION_ID`, + `METADATA`, + `PROGRESS` +) AS +SELECT + RM_SCHEDULED_EXECUTION_ID, + RM_POLICY, + RM_DUMP_EXECUTION_ID, + RM_METADATA, + RM_PROGRESS +FROM REPLICATION_METRICS; + diff --git a/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql b/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql index 09c95d7b17..06267ff1c0 100644 --- a/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql +++ b/metastore/scripts/upgrade/hive/upgrade-3.1.0-to-4.0.0.hive.sql @@ -766,4 +766,19 @@ WHERE OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) AND P.`TBL_PRIV`='SELECT' AND P.`AUTHORIZER`=current_authorizer()); +CREATE OR REPLACE VIEW `REPLICATION_METRICS` ( + `SCHEDULED_EXECUTION_ID`, + `POLICY_NAME`, + `DUMP_EXECUTION_ID`, + `METADATA`, + `PROGRESS` +) AS +SELECT + RM_SCHEDULED_EXECUTION_ID, + RM_POLICY, + RM_DUMP_EXECUTION_ID, + RM_METADATA, + RM_PROGRESS +FROM REPLICATION_METRICS; + SELECT 'Finished upgrading MetaStore schema from 3.1.0 to 4.0.0'; diff --git a/ql/src/test/queries/clientpositive/replication_metrics_ingest.q b/ql/src/test/queries/clientpositive/replication_metrics_ingest.q new file mode 100644 index 0000000000..879f15fe90 --- /dev/null +++ b/ql/src/test/queries/clientpositive/replication_metrics_ingest.q @@ -0,0 +1,24 @@ +set user.name=hive_admin_user; +set role admin; + +drop database if exists src cascade; + +create database src with DBPROPERTIES ('repl.source.for' = '1,2,3'); + +create table src.t(id int, cnt int); + +-- add data to table +insert into src.t values(1,1); + +-- configure to run ingestion - in the far future +create scheduled query repl1 every 1 seconds as repl dump src; + +!sleep 10; + +create scheduled query repl2 every 1 seconds as repl load src into tgt; + +show databases; + +select * from tgt.t; + +select * from sys.replication_metrics; diff --git a/ql/src/test/results/clientpositive/replication_metrics_ingest.q.out b/ql/src/test/results/clientpositive/replication_metrics_ingest.q.out new file mode 100644 index 0000000000..8e5c1230b6 --- /dev/null +++ b/ql/src/test/results/clientpositive/replication_metrics_ingest.q.out @@ -0,0 +1,124 @@ +PREHOOK: query: set role admin +PREHOOK: type: SHOW_ROLES +POSTHOOK: query: set role admin +POSTHOOK: type: SHOW_ROLES +PREHOOK: query: drop table if exists t +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists t +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table if exists s +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists s +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table s(id integer, cnt integer) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@s +POSTHOOK: query: create table s(id integer, cnt integer) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@s +PREHOOK: query: create table t(id integer, cnt integer) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: create table t(id integer, cnt integer) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: create table t_offset(offset integer) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t_offset +POSTHOOK: query: create table t_offset(offset integer) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t_offset +PREHOOK: query: insert into t_offset values(0) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t_offset +POSTHOOK: query: insert into t_offset values(0) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t_offset +POSTHOOK: Lineage: t_offset.offset SCRIPT [] +PREHOOK: query: insert into s values(1,1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@s +POSTHOOK: query: insert into s values(1,1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@s +POSTHOOK: Lineage: s.cnt SCRIPT [] +POSTHOOK: Lineage: s.id SCRIPT [] +Warning: Shuffle Join MERGEJOIN[37][tables = [$hdt$_0, $hdt$_1]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: from (select id==offset as first,* from s +join t_offset on id>=offset) s1 +insert into t select id,cnt where not first +insert overwrite table t_offset select max(s1.id) +PREHOOK: type: QUERY +PREHOOK: Input: default@s +PREHOOK: Input: default@t_offset +PREHOOK: Output: default@t +PREHOOK: Output: default@t_offset +POSTHOOK: query: from (select id==offset as first,* from s +join t_offset on id>=offset) s1 +insert into t select id,cnt where not first +insert overwrite table t_offset select max(s1.id) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@s +POSTHOOK: Input: default@t_offset +POSTHOOK: Output: default@t +POSTHOOK: Output: default@t_offset +POSTHOOK: Lineage: t.cnt SIMPLE [(s)s.FieldSchema(name:cnt, type:int, comment:null), ] +POSTHOOK: Lineage: t.id SIMPLE [(s)s.FieldSchema(name:id, type:int, comment:null), ] +POSTHOOK: Lineage: t_offset.offset EXPRESSION [(s)s.FieldSchema(name:id, type:int, comment:null), ] +Warning: Shuffle Join MERGEJOIN[34][tables = [s, t_offset]] in Stage 'Reducer 2' is a cross product +PREHOOK: query: create scheduled query ingest cron '0 0 0 1 * ? 2030' defined as +from (select id==offset as first,* from s +join t_offset on id>=offset) s1 +insert into t select id,cnt where not first +insert overwrite table t_offset select max(s1.id) +PREHOOK: type: CREATE SCHEDULED QUERY +POSTHOOK: query: create scheduled query ingest cron '0 0 0 1 * ? 2030' defined as +from (select id==offset as first,* from s +join t_offset on id>=offset) s1 +insert into t select id,cnt where not first +insert overwrite table t_offset select max(s1.id) +POSTHOOK: type: CREATE SCHEDULED QUERY +PREHOOK: query: insert into s values(2,2),(3,3) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@s +POSTHOOK: query: insert into s values(2,2),(3,3) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@s +POSTHOOK: Lineage: s.cnt SCRIPT [] +POSTHOOK: Lineage: s.id SCRIPT [] +PREHOOK: query: alter scheduled query ingest execute +PREHOOK: type: ALTER SCHEDULED QUERY +POSTHOOK: query: alter scheduled query ingest execute +POSTHOOK: type: ALTER SCHEDULED QUERY +PREHOOK: query: select state,error_message from sys.scheduled_executions +PREHOOK: type: QUERY +PREHOOK: Input: sys@scheduled_executions +#### A masked pattern was here #### +POSTHOOK: query: select state,error_message from sys.scheduled_executions +POSTHOOK: type: QUERY +POSTHOOK: Input: sys@scheduled_executions +#### A masked pattern was here #### +FINISHED NULL +PREHOOK: query: select * from t order by id +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: select * from t order by id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +1 1 +2 2 +3 3