diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index 34fb318911..a4b2f52b2c 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -1389,6 +1389,11 @@ public ReplicationMetricList getReplicationMetrics(GetReplicationMetricsRequest throw new RuntimeException("unimplemented"); } + @Override + public int deleteReplicationMetrics(int maxRetainSecs) { + return objectStore.deleteReplicationMetrics(maxRetainSecs); + } + @Override public ScheduledQuery getScheduledQuery(ScheduledQueryKey scheduleKey) throws MetaException, NoSuchObjectException { throw new RuntimeException("unimplemented"); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java index 017607e87b..fd0a214eca 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java @@ -58,7 +58,7 @@ * TestScheduledReplicationScenarios - test scheduled replication . */ public class TestScheduledReplicationScenarios extends BaseReplicationScenariosAcidTables { - private static final long DEFAULT_PROBE_TIMEOUT = 2 * 60 * 1000L; // 2 minutes + private static final long DEFAULT_PROBE_TIMEOUT = 5 * 60 * 1000L; // 5 minutes @BeforeClass public static void classLevelSetup() throws Exception { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/metric/MetricSink.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/metric/MetricSink.java index 88a7b077dd..a856c76319 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/metric/MetricSink.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/metric/MetricSink.java @@ -71,7 +71,7 @@ public synchronized void init(HiveConf conf) { long getFrequencyInSecs() { //Metastore conf is in minutes - return MetastoreConf.getLongVar(conf, MetastoreConf.ConfVars.REPL_METRICS_UPDATE_FREQUENCY) * 60; + return MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.REPL_METRICS_UPDATE_FREQUENCY, TimeUnit.MINUTES) * 60; } public synchronized void tearDown() { diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 02fb9cfc27..ca9cd7d40d 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -981,9 +981,16 @@ public static ConfVars getMetaConf(String name) { + " and the frequency of persisting the metrics to persistent storage. " ), REPL_METRICS_UPDATE_FREQUENCY("metastore.repl.metrics.update.frequency", - "hive.repl.metrics.update.frequency", 1 /*1 minute */, + "hive.repl.metrics.update.frequency", 1L, TimeUnit.MINUTES /*1 minute */, "Frequency at which replication Metrics will be stored in persistent storage. " ), + REPL_METRICS_CLEANUP_FREQUENCY("metastore.repl.metrics.cleanup.frequency", + "hive.metastore.repl.metrics.cleanup.frequency", 1, TimeUnit.DAYS, + "Interval of scheduled metrics clean up task which removes metrics above max age; Max age is" + + " defined by the config metastore.repl.metrics.max.age. The max age should be greater than this frequency"), + REPL_METRICS_MAX_AGE("metastore.repl.metrics.max.age", + "hive.metastore.repl.metrics.max.age", 7, TimeUnit.DAYS, + "Maximal age of a replication metrics entry before it is removed."), SCHEMA_INFO_CLASS("metastore.schema.info.class", "hive.metastore.schema.info.class", "org.apache.hadoop.hive.metastore.MetaStoreSchemaInfo", "Fully qualified class name for the metastore schema information class \n" @@ -1077,7 +1084,8 @@ public static ConfVars getMetaConf(String name) { EVENT_CLEANER_TASK_CLASS + "," + RUNTIME_STATS_CLEANER_TASK_CLASS + "," + "org.apache.hadoop.hive.metastore.repl.DumpDirCleanerTask" + "," + "org.apache.hadoop.hive.metastore.HiveProtoEventsCleanerTask" + "," - + "org.apache.hadoop.hive.metastore.ScheduledQueryExecutionsMaintTask", + + "org.apache.hadoop.hive.metastore.ScheduledQueryExecutionsMaintTask" + "," + + "org.apache.hadoop.hive.metastore.ReplicationMetricsMaintTask", "Comma separated list of tasks that will be started in separate threads. These will " + "always be started, regardless of whether the metastore is running in embedded mode " + "or in server mode. They must implement " + METASTORE_TASK_THREAD_CLASS), diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index 903b531be7..d6bfb899e7 100644 --- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -2039,7 +2039,7 @@ struct ReplicationMetrics{ 2: required string policy, 3: required i64 dumpExecutionId, 4: optional string metadata, - 5: optional string progress, + 5: optional string progress } struct ReplicationMetricList{ diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 5af71691e3..c472e93126 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -13104,6 +13104,7 @@ public void addReplicationMetrics(ReplicationMetricList replicationMetricList) { mReplicationMetrics.setDumpExecutionId(replicationMetric.getDumpExecutionId()); mReplicationMetrics.setScheduledExecutionId(replicationMetric.getScheduledExecutionId()); mReplicationMetrics.setPolicy(replicationMetric.getPolicy()); + mReplicationMetrics.setStartTime((int) (System.currentTimeMillis()/1000)); } if (!StringUtils.isEmpty(replicationMetric.getMetadata())) { mReplicationMetrics.setMetadata(replicationMetric.getMetadata()); @@ -13142,6 +13143,28 @@ public ReplicationMetricList getReplicationMetrics(GetReplicationMetricsRequest } } + @Override + public int deleteReplicationMetrics(int maxRetainSecs) { + if (maxRetainSecs < 0) { + LOG.debug("replication metrics deletion is disabled"); + return 0; + } + boolean committed = false; + Query q = null; + try { + openTransaction(); + int maxCreateTime = (int) ((System.currentTimeMillis() / 1000) - maxRetainSecs); + q = pm.newQuery(MReplicationMetrics.class); + q.setFilter("startTime <= maxCreateTime"); + q.declareParameters("long maxCreateTime"); + long deleted = q.deletePersistentAll(maxCreateTime); + committed = commitTransaction(); + return (int) deleted; + } finally { + rollbackAndCleanup(committed, q); + } + } + private ReplicationMetricList getMReplicationMetrics(String policy) { ReplicationMetricList ret = new ReplicationMetricList(); if (StringUtils.isEmpty(policy)) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 6534750855..d45625542f 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -1857,6 +1857,8 @@ void scheduledQueryProgress(ScheduledQueryProgressInfo info) */ ReplicationMetricList getReplicationMetrics(GetReplicationMetricsRequest replicationMetricsRequest); + int deleteReplicationMetrics(int maxRetainSecs); + int deleteScheduledExecutions(int maxRetainSecs); int markScheduledExecutionsTimedOut(int timeoutSecs) throws InvalidOperationException, MetaException; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ReplicationMetricsMaintTask.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ReplicationMetricsMaintTask.java new file mode 100644 index 0000000000..4ba968f76b --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ReplicationMetricsMaintTask.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.TimeUnit; + +/** + * Metastore task to remove old replication metrics. + */ +public class ReplicationMetricsMaintTask implements MetastoreTaskThread { + private static final Logger LOG = LoggerFactory.getLogger(ReplicationMetricsMaintTask.class); + + private Configuration conf; + + @Override + public long initialDelay(TimeUnit unit) { + // no delay before the first execution; + // after an ungracefull shutdown it might take time to notice that in-flight scheduled queries are not running anymore + return 0; + } + + @Override + public long runFrequency(TimeUnit unit) { + return MetastoreConf.getTimeVar(conf, ConfVars.REPL_METRICS_CLEANUP_FREQUENCY, + TimeUnit.DAYS); + } + + @Override + public void setConf(Configuration configuration) { + conf = configuration; + } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void run() { + try { + if (!MetastoreConf.getBoolVar(conf, ConfVars.SCHEDULED_QUERIES_ENABLED)) { + return; + } + RawStore ms = HiveMetaStore.HMSHandler.getMSForConf(conf); + int maxRetainSecs = (int) TimeUnit.DAYS.toSeconds(MetastoreConf.getTimeVar(conf, + ConfVars.REPL_METRICS_MAX_AGE, TimeUnit.DAYS)); + int deleteCnt = ms.deleteScheduledExecutions(maxRetainSecs); + if (deleteCnt > 0L){ + LOG.info("Number of deleted entries: " + deleteCnt); + } + } catch (Exception e) { + LOG.error("Exception while trying to delete: " + e.getMessage(), e); + } + } +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index c8e230b62f..16df3c3415 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -2899,6 +2899,11 @@ public ReplicationMetricList getReplicationMetrics(GetReplicationMetricsRequest return rawStore.getReplicationMetrics(replicationMetricsRequest); } + @Override + public int deleteReplicationMetrics(int maxRetainSecs) { + return rawStore.deleteReplicationMetrics(maxRetainSecs); + } + @Override public ScheduledQuery getScheduledQuery(ScheduledQueryKey scheduleKey) throws MetaException, NoSuchObjectException { return rawStore.getScheduledQuery(scheduleKey); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MReplicationMetrics.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MReplicationMetrics.java index 463be25d69..5fe3129c1a 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MReplicationMetrics.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MReplicationMetrics.java @@ -29,6 +29,7 @@ private long dumpExecutionId; private String metadata; private String progress; + private int startTime; public MReplicationMetrics() { } @@ -82,4 +83,12 @@ public String getProgress() { public void setProgress(String progress) { this.progress = progress; } + + public int getStartTime() { + return startTime; + } + + public void setStartTime(int startTime) { + this.startTime = startTime; + } } diff --git a/standalone-metastore/metastore-server/src/main/resources/package.jdo b/standalone-metastore/metastore-server/src/main/resources/package.jdo index c88ed32a78..0252dd03db 100644 --- a/standalone-metastore/metastore-server/src/main/resources/package.jdo +++ b/standalone-metastore/metastore-server/src/main/resources/package.jdo @@ -1539,6 +1539,9 @@ + + + diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql index 811f2b13fe..c64805dc4f 100644 --- a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql +++ b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.0.0.derby.sql @@ -779,6 +779,7 @@ CREATE TABLE "APP"."REPLICATION_METRICS" ( "RM_DUMP_EXECUTION_ID" bigint NOT NULL, "RM_METADATA" varchar(4000), "RM_PROGRESS" varchar(4000), + "RM_START_TIME" integer not null, PRIMARY KEY("RM_SCHEDULED_EXECUTION_ID") ); diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql index aeea23f23d..a9c2cab1f7 100644 --- a/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql +++ b/standalone-metastore/metastore-server/src/main/sql/derby/upgrade-3.2.0-to-4.0.0.derby.sql @@ -94,6 +94,7 @@ CREATE TABLE "APP"."REPLICATION_METRICS" ( "RM_DUMP_EXECUTION_ID" bigint NOT NULL, "RM_METADATA" varchar(4000), "RM_PROGRESS" varchar(4000), + "RM_START_TIME" integer not null, PRIMARY KEY("RM_SCHEDULED_EXECUTION_ID") ); diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index cba72cd0fd..a59b646fcf 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -1326,7 +1326,8 @@ CREATE TABLE "REPLICATION_METRICS" ( "RM_POLICY" varchar(256) NOT NULL, "RM_DUMP_EXECUTION_ID" bigint NOT NULL, "RM_METADATA" varchar(max), - "RM_PROGRESS" varchar(max) + "RM_PROGRESS" varchar(max), + "RM_START_TIME" integer NOT NULL ); -- Create indexes for the replication metrics table diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql index fa1250ce08..0826c1e123 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mssql/upgrade-3.2.0-to-4.0.0.mssql.sql @@ -129,7 +129,8 @@ CREATE TABLE "REPLICATION_METRICS" ( "RM_POLICY" varchar(256) NOT NULL, "RM_DUMP_EXECUTION_ID" bigint NOT NULL, "RM_METADATA" varchar(max), - "RM_PROGRESS" varchar(max) + "RM_PROGRESS" varchar(max), + "RM_START_TIME" integer NOT NULL ); -- Create indexes for the replication metrics table diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index e036b31a9d..0d0374c08a 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -1251,6 +1251,7 @@ CREATE TABLE REPLICATION_METRICS ( RM_DUMP_EXECUTION_ID bigint NOT NULL, RM_METADATA varchar(4000), RM_PROGRESS varchar(4000), + RM_START_TIME integer NOT NULL, PRIMARY KEY(RM_SCHEDULED_EXECUTION_ID) ); diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql index c1c888bb63..acf09c1ff1 100644 --- a/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql +++ b/standalone-metastore/metastore-server/src/main/sql/mysql/upgrade-3.2.0-to-4.0.0.mysql.sql @@ -101,6 +101,7 @@ CREATE TABLE REPLICATION_METRICS ( RM_DUMP_EXECUTION_ID bigint NOT NULL, RM_METADATA varchar(4000), RM_PROGRESS varchar(4000), + RM_START_TIME integer NOT NULL, PRIMARY KEY(RM_SCHEDULED_EXECUTION_ID) ); diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index c7bf0bae22..bc757bf076 100644 --- a/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ b/standalone-metastore/metastore-server/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -1224,7 +1224,8 @@ CREATE TABLE "REPLICATION_METRICS" ( "RM_POLICY" varchar2(256) NOT NULL, "RM_DUMP_EXECUTION_ID" number NOT NULL, "RM_METADATA" varchar2(4000), - "RM_PROGRESS" varchar2(4000) + "RM_PROGRESS" varchar2(4000), + "RM_START_TIME" integer NOT NULL ); --Create indexes for the replication metrics table diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql index 61467105d0..84db436aa1 100644 --- a/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql +++ b/standalone-metastore/metastore-server/src/main/sql/oracle/upgrade-3.2.0-to-4.0.0.oracle.sql @@ -98,7 +98,8 @@ CREATE TABLE "REPLICATION_METRICS" ( "RM_POLICY" varchar2(256) NOT NULL, "RM_DUMP_EXECUTION_ID" number NOT NULL, "RM_METADATA" varchar2(4000), - "RM_PROGRESS" varchar2(4000) + "RM_PROGRESS" varchar2(4000), + "RM_START_TIME" integer NOT NULL ); --Create indexes for the replication metrics table diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index bdbe88f168..9b03d730d0 100644 --- a/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ b/standalone-metastore/metastore-server/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -1913,6 +1913,7 @@ CREATE TABLE "REPLICATION_METRICS" ( "RM_DUMP_EXECUTION_ID" bigint NOT NULL, "RM_METADATA" varchar(4000), "RM_PROGRESS" varchar(4000), + "RM_START_TIME" integer NOT NULL, PRIMARY KEY("RM_SCHEDULED_EXECUTION_ID") ); diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql index f6caac8e5e..53f671cb17 100644 --- a/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql +++ b/standalone-metastore/metastore-server/src/main/sql/postgres/upgrade-3.2.0-to-4.0.0.postgres.sql @@ -225,6 +225,7 @@ CREATE TABLE "REPLICATION_METRICS" ( "RM_DUMP_EXECUTION_ID" bigint NOT NULL, "RM_METADATA" varchar(4000), "RM_PROGRESS" varchar(4000), + "RM_START_TIME" integer NOT NULL, PRIMARY KEY("RM_SCHEDULED_EXECUTION_ID") ); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index d7b2d65622..a779813e43 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -1338,6 +1338,11 @@ public ReplicationMetricList getReplicationMetrics(GetReplicationMetricsRequest return objectStore.getReplicationMetrics(replicationMetricsRequest); } + @Override + public int deleteReplicationMetrics(int maxRetainSecs) { + return objectStore.deleteReplicationMetrics(maxRetainSecs); + } + @Override public ScheduledQuery getScheduledQuery(ScheduledQueryKey scheduleKey) throws NoSuchObjectException { return objectStore.getScheduledQuery(scheduleKey); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 19586075ae..1cbc86f847 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -1321,6 +1321,11 @@ public ReplicationMetricList getReplicationMetrics(GetReplicationMetricsRequest throw new RuntimeException("unimplemented"); } + @Override + public int deleteReplicationMetrics(int maxRetainSecs) { + throw new RuntimeException("unimplemented"); + } + @Override public ScheduledQuery getScheduledQuery(ScheduledQueryKey scheduleKey) { throw new RuntimeException("unimplemented"); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestReplicationMetrics.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestReplicationMetrics.java index f8c8e1f66c..d151d3aae0 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestReplicationMetrics.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestReplicationMetrics.java @@ -70,7 +70,7 @@ public void tearDown() throws Exception { public void testAddMetrics() throws Exception { ObjectStore objStore = new ObjectStore(); objStore.setConf(metaStore.getConf()); - objStore.deleteRuntimeStats(0); + objStore.deleteReplicationMetrics(0); ReplicationMetricList replicationMetricList = new ReplicationMetricList(); List replicationMetrics = new ArrayList<>(); replicationMetrics.add(createReplicationMetric("repl1", 1L)); @@ -128,7 +128,7 @@ public void testAddMetrics() throws Exception { public void testUpdateMetrics() throws Exception { ObjectStore objStore = new ObjectStore(); objStore.setConf(metaStore.getConf()); - objStore.deleteRuntimeStats(0); + objStore.deleteReplicationMetrics(0); ReplicationMetricList replicationMetricList = new ReplicationMetricList(); List replicationMetrics = new ArrayList<>(); replicationMetrics.add(createReplicationMetric("repl1", 1L)); @@ -193,7 +193,7 @@ public void testUpdateMetrics() throws Exception { public void testGetMetricsByScheduleId() throws Exception { ObjectStore objStore = new ObjectStore(); objStore.setConf(metaStore.getConf()); - objStore.deleteRuntimeStats(0); + objStore.deleteReplicationMetrics(0); ReplicationMetricList replicationMetricList = new ReplicationMetricList(); List replicationMetrics = new ArrayList<>(); replicationMetrics.add(createReplicationMetric("repl1", 1L)); @@ -244,6 +244,45 @@ public void testGetMetricsByScheduleId() throws Exception { } + @Test + public void testDeleteMetrics() throws Exception { + ObjectStore objStore = new ObjectStore(); + objStore.setConf(metaStore.getConf()); + objStore.deleteReplicationMetrics(0); + ReplicationMetricList replicationMetricList = new ReplicationMetricList(); + List replicationMetrics = new ArrayList<>(); + replicationMetrics.add(createReplicationMetric("repl1", 1L)); + replicationMetrics.add(createReplicationMetric("repl1", 2L)); + replicationMetricList.setReplicationMetricList(replicationMetrics); + objStore.addReplicationMetrics(replicationMetricList); + Thread.sleep(2000); + replicationMetrics = new ArrayList<>(); + replicationMetrics.add(createReplicationMetric("repl1", 3L)); + replicationMetricList.setReplicationMetricList(replicationMetrics); + objStore.addReplicationMetrics(replicationMetricList); + Thread.sleep(500); + + GetReplicationMetricsRequest getReplicationMetricsRequest = new GetReplicationMetricsRequest(); + getReplicationMetricsRequest.setPolicy("repl1"); + ReplicationMetricList actualList = client.getReplicationMetrics(getReplicationMetricsRequest); + assertEquals(3, actualList.getReplicationMetricListSize()); + //delete older metrics + objStore.deleteReplicationMetrics(2); + + getReplicationMetricsRequest = new GetReplicationMetricsRequest(); + getReplicationMetricsRequest.setPolicy("repl1"); + actualList = client.getReplicationMetrics(getReplicationMetricsRequest); + assertEquals(1, actualList.getReplicationMetricListSize()); + List actualMetrics = actualList.getReplicationMetricList(); + ReplicationMetrics actualMetric0 = actualMetrics.get(0); + assertEquals("repl1", actualMetric0.getPolicy()); + assertEquals(3L, actualMetric0.getScheduledExecutionId()); + assertEquals(1, actualMetric0.getDumpExecutionId()); + assertEquals("metadata", actualMetric0.getMetadata()); + assertEquals("progress", actualMetric0.getProgress()); + + } + private ReplicationMetrics createReplicationMetric(String policyName, Long scheduleId) { ReplicationMetrics replicationMetrics = new ReplicationMetrics(); replicationMetrics.setPolicy(policyName);