diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java index 95e2bcf..2b683ad 100644 --- a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java @@ -39,4 +39,8 @@ public static final String OPERATION_PREFIX = "hs2_operation_"; public static final String COMPLETED_OPERATION_PREFIX = "hs2_completed_operation_"; + + public static final String TOTAL_DATABASES = "total_count_dbs"; + public static final String TOTAL_TABLES = "total_count_tables"; + public static final String TOTAL_PARTITIONS = "total_count_partitions"; } diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 4d881ba..638f89d 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -701,6 +701,8 @@ public void setSparkConfigUpdated(boolean isSparkConfigUpdated) { METASTORE_AGGREGATE_STATS_CACHE_CLEAN_UNTIL("hive.metastore.aggregate.stats.cache.clean.until", (float) 0.8, "The cleaner thread cleans until cache reaches this % full size."), METASTORE_METRICS("hive.metastore.metrics.enabled", false, "Enable metrics on the metastore."), + METASTORE_METRICS_METADATA_COUNT_FREQUENCY("hive.metastore.metrics.metadata.count.frequency", "300s", + new TimeValidator(TimeUnit.SECONDS), "Time interval between updating metadata count metrics on the metastore"), // Parameters for exporting metadata on table drop (requires the use of the) // org.apache.hadoop.hive.ql.parse.MetaDataExportListener preevent listener diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java b/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java index fd420f7..f21b431 100644 --- a/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java +++ b/common/src/test/org/apache/hadoop/hive/common/metrics/MetricsTestUtils.java @@ -45,9 +45,9 @@ } public static void verifyMetricFile(File jsonReportFile, MetricsCategory category, String metricsName, - Object value) throws Exception { + Object expectedValue) throws Exception { JsonNode jsonNode = getJsonNode(jsonReportFile, category, metricsName); - Assert.assertEquals(jsonNode.asText(), value.toString()); + Assert.assertEquals(expectedValue.toString(), jsonNode.asText()); } private static JsonNode getJsonNode(File jsonReportFile, MetricsCategory category, String metricsName) throws Exception { diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java index f571c7c..3eb77be 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java @@ -19,27 +19,21 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import junit.framework.TestCase; import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.common.metrics.MetricsTestUtils; +import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.Driver; -import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.ShimLoader; -import org.apache.hive.service.server.HiveServer2; -import org.junit.After; -import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import java.io.File; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.Map; /** * Tests Hive Metastore Metrics. @@ -69,6 +63,7 @@ public static void before() throws Exception { hiveConf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name()); hiveConf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString()); hiveConf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms"); + hiveConf.setVar(HiveConf.ConfVars.METASTORE_METRICS_METADATA_COUNT_FREQUENCY, "100ms"); MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge(), hiveConf); @@ -77,13 +72,12 @@ public static void before() throws Exception { } @Test - public void testMetricsFile() throws Exception { + public void testMethodCounts() throws Exception { driver.run("show databases"); //give timer thread a chance to print the metrics Thread.sleep(2000); - //As the file is being written, try a few times. //This can be replaced by CodahaleMetrics's JsonServlet reporter once it is exposed. byte[] jsonData = Files.readAllBytes(Paths.get(jsonReportFile.getAbsolutePath())); ObjectMapper objectMapper = new ObjectMapper(); @@ -95,6 +89,30 @@ public void testMetricsFile() throws Exception { Assert.assertTrue(methodCountNode.asInt() > 0); } + @Test + public void testMetaDataCounts() throws Exception { + //2 databases (1 + default) + driver.run("create database testdb1"); + + //3 tables + driver.run("create table testtbl1 (key string)"); + driver.run("create table testtbl2 (key string)"); + driver.run("create table testtblpart (key string) partitioned by (partkey string)"); + + //4 partitions + driver.run("alter table testtblpart add partition (partkey='a')"); + driver.run("alter table testtblpart add partition (partkey='b')"); + driver.run("alter table testtblpart add partition (partkey='c')"); + driver.run("alter table testtblpart add partition (partkey='d')"); + + //give timer thread a chance to print the metrics + Thread.sleep(2000); + + MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.TOTAL_DATABASES, 2); + MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.TOTAL_TABLES, 3); + MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.TOTAL_PARTITIONS, 4); + } + @Test public void testConnections() throws Exception { diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java new file mode 100644 index 0000000..e2a5ba0 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hive.metastore.hbase; + +import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.common.metrics.MetricsTestUtils; +import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; +import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +/** + * Test HMS Metrics on HBase Metastore + */ +public class TestHBaseMetastoreMetrics extends HBaseIntegrationTests { + + private static File jsonReportFile; + + @BeforeClass + public static void startup() throws Exception { + File workDir = new File(System.getProperty("test.tmp.dir")); + jsonReportFile = new File(workDir, "json_reporting"); + jsonReportFile.delete(); + + HBaseIntegrationTests.startMiniCluster(); + } + + @AfterClass + public static void shutdown() throws Exception { + HBaseIntegrationTests.shutdownMiniCluster(); + } + + @Before + public void before() throws IOException { + HBaseReadWrite.setConf(conf); + conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, + "org.apache.hadoop.hive.metastore.hbase.HBaseStore"); + conf.setBoolVar(HiveConf.ConfVars.METASTORE_FASTPATH, true); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + + conf.setBoolVar(HiveConf.ConfVars.METASTORE_METRICS, true); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms"); + conf.setVar(HiveConf.ConfVars.METASTORE_METRICS_METADATA_COUNT_FREQUENCY, "100ms"); + SessionState.start(new CliSessionState(conf)); + driver = new Driver(conf); + } + + @Test + public void testMetaDataCounts() throws Exception { + //2 databases (1 + default) + driver.run("create database testdb1"); + + //3 tables + driver.run("create table testtbl1 (key string)"); + driver.run("create table testtbl2 (key string)"); + driver.run("create table testtblpart (key string) partitioned by (partkey string)"); + + //4 partitions + driver.run("alter table testtblpart add partition (partkey='a')"); + driver.run("alter table testtblpart add partition (partkey='b')"); + driver.run("alter table testtblpart add partition (partkey='c')"); + driver.run("alter table testtblpart add partition (partkey='d')"); + + //give timer thread a chance to print the metrics + Thread.sleep(2000); + + MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.TOTAL_DATABASES, 2); + MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.TOTAL_TABLES, 3); + MetricsTestUtils.verifyMetricFile(jsonReportFile, MetricsTestUtils.GAUGE, MetricsConstant.TOTAL_PARTITIONS, 4); + } +} diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 00602e1..4ff8bd7 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -28,6 +28,7 @@ import com.google.common.collect.Multimaps; import org.apache.commons.cli.OptionBuilder; +import org.apache.hadoop.hive.common.metrics.common.MetricsVariable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -129,6 +130,7 @@ import java.util.Properties; import java.util.Set; import java.util.Timer; +import java.util.TimerTask; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Condition; @@ -202,6 +204,7 @@ public TTransport getTransport(TTransport trans) { // right now they come from jpox.properties private static String currentUrl; + private volatile int databaseCount, tableCount, partCount; //For Metrics private Warehouse wh; // hdfs warehouse private static final ThreadLocal threadLocalMS = @@ -385,6 +388,45 @@ public void init() throws MetaException { } } + Metrics metrics = MetricsFactory.getInstance(); + if (metrics != null) { + metrics.addGauge(MetricsConstant.TOTAL_DATABASES, new MetricsVariable() { + @Override + public Object getValue() { + return databaseCount; + } + }); + + metrics.addGauge(MetricsConstant.TOTAL_TABLES, new MetricsVariable() { + @Override + public Object getValue() { + return tableCount; + } + }); + + metrics.addGauge(MetricsConstant.TOTAL_PARTITIONS, new MetricsVariable() { + @Override + public Object getValue() { + return partCount; + } + }); + + Timer timer = new java.util.Timer(true); + timer.schedule(new TimerTask() { + @Override + public void run() { + try { + LOG.info("Starting metadata count metrics collection"); + updateMetrics(); + LOG.info("Completed metadata count metrics collection: " + databaseCount + " databases, " + + tableCount + " tables, and " + partCount + " partitions."); + } catch (MetaException e) { + LOG.warn("Error updating metadata count metrics", e); + } + } + }, 0, HiveConf.getTimeVar(hiveConf, ConfVars.METASTORE_METRICS_METADATA_COUNT_FREQUENCY, TimeUnit.MILLISECONDS)); + } + preListeners = MetaStoreUtils.getMetaStoreListeners(MetaStorePreEventListener.class, hiveConf, hiveConf.getVar(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS)); @@ -5730,6 +5772,12 @@ public ClearFileMetadataResult clear_file_metadata(ClearFileMetadataRequest req) getMS().putFileMetadata(req.getFileIds(), null); return new ClearFileMetadataResult(); } + + private void updateMetrics() throws MetaException { + tableCount = getMS().getTableCount(); + partCount = getMS().getPartitionCount(); + databaseCount = getMS().getDatabaseCount(); + } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 1c0ab6d..abfe2b8 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -37,8 +37,11 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.regex.Pattern; @@ -213,7 +216,6 @@ private volatile int openTrasactionCalls = 0; private Transaction currentTransaction = null; private TXN_STATUS transactionStatus = TXN_STATUS.NO_STATE; - private Pattern partitionValidationPattern; /** @@ -1049,6 +1051,40 @@ public Table getTable(String dbName, String tableName) throws MetaException { return tbls; } + public int getDatabaseCount() throws MetaException { + return getObjectCount("name", MDatabase.class.getName()); + } + + public int getPartitionCount() throws MetaException { + return getObjectCount("partitionName", MPartition.class.getName()); + } + + public int getTableCount() throws MetaException { + return getObjectCount("tableName", MTable.class.getName()); + } + + private int getObjectCount(String fieldName, String objName) { + Long result = 0L; + boolean commited = false; + Query query = null; + try { + openTransaction(); + String queryStr = + "select count(" + fieldName + ") from " + objName; + query = pm.newQuery(queryStr); + result = (Long) query.execute(); + commited = commitTransaction(); + } finally { + if (!commited) { + rollbackTransaction(); + } + if (query != null) { + query.closeAll(); + } + } + return result.intValue(); + } + @Override public List getTableMeta(String dbNames, String tableNames, List tableTypes) throws MetaException { diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java index 5b36b03..e118a3b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -27,6 +27,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; @@ -633,4 +634,22 @@ public AggrStats get_aggr_stats_for(String dbName, String tblName, void getFileMetadataByExpr(List fileIds, FileMetadataExprType type, byte[] expr, ByteBuffer[] metadatas, ByteBuffer[] exprResults, boolean[] eliminated) throws MetaException; + + /** + * Gets total number of tables. + */ + @InterfaceStability.Evolving + int getTableCount() throws MetaException; + + /** + * Gets total number of partitions. + */ + @InterfaceStability.Evolving + int getPartitionCount() throws MetaException; + + /** + * Gets total number of databases. + */ + @InterfaceStability.Evolving + int getDatabaseCount() throws MetaException; } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java index 2fb3e8f..287394e 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java @@ -19,8 +19,10 @@ package org.apache.hadoop.hive.metastore.hbase; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Iterators; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -421,6 +423,12 @@ String printDatabase(String name) throws IOException, TException { } } + int getDatabaseCount() throws IOException { + Filter fil = new FirstKeyOnlyFilter(); + Iterator iter = scan(DB_TABLE, fil); + return Iterators.size(iter); + } + /********************************************************************************************** * Function related methods *********************************************************************************************/ @@ -865,6 +873,12 @@ String printPartition(String partKey) throws IOException, TException { return lines; } + int getPartitionCount() throws IOException { + Filter fil = new FirstKeyOnlyFilter(); + Iterator iter = scan(PART_TABLE, fil); + return Iterators.size(iter); + } + private String printOnePartition(Result result) throws IOException, TException { byte[] key = result.getRow(); HBaseUtils.StorageDescriptorParts sdParts = @@ -1671,6 +1685,12 @@ String printTable(String name) throws IOException, TException { return lines; } + int getTableCount() throws IOException { + Filter fil = new FirstKeyOnlyFilter(); + Iterator iter = scan(TABLE_TABLE, fil); + return Iterators.size(iter); + } + private String printOneTable(Result result) throws IOException, TException { byte[] key = result.getRow(); HBaseUtils.StorageDescriptorParts sdParts = diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java index 98e6c75..b9509ab 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java @@ -240,6 +240,16 @@ public boolean alterDatabase(String dbname, Database db) throws NoSuchObjectExce } @Override + public int getDatabaseCount() throws MetaException { + try { + return getHBase().getDatabaseCount(); + } catch (IOException e) { + LOG.error("Unable to get database count", e); + throw new MetaException("Error scanning databases"); + } + } + + @Override public boolean createType(Type type) { throw new UnsupportedOperationException(); } @@ -561,6 +571,16 @@ public void alterTable(String dbName, String tableName, Table newTable) throws I } @Override + public int getTableCount() throws MetaException { + try { + return getHBase().getTableCount(); + } catch (IOException e) { + LOG.error("Unable to get table count", e); + throw new MetaException("Error scanning tables"); + } + } + + @Override public List listTableNamesByFilter(String dbName, String filter, short max_tables) throws MetaException, UnknownDBException { // TODO needs to wait until we support pushing filters into HBase. @@ -883,6 +903,16 @@ public boolean isPartitionMarkedForEvent(String dbName, String tblName, */ @Override + public int getPartitionCount() throws MetaException { + try { + return getHBase().getPartitionCount(); + } catch (IOException e) { + LOG.error("Unable to get partition count", e); + throw new MetaException("Error scanning partitions"); + } + } + + @Override public boolean addRole(String roleName, String ownerName) throws InvalidObjectException, MetaException, NoSuchObjectException { int now = (int)(System.currentTimeMillis()/1000); diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 9a1d159..c1156b3 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -788,4 +788,19 @@ public boolean isFileMetadataSupported() { public void getFileMetadataByExpr(List fileIds, FileMetadataExprType type, byte[] expr, ByteBuffer[] metadatas, ByteBuffer[] stripeBitsets, boolean[] eliminated) { } + + @Override + public int getTableCount() throws MetaException { + return objectStore.getTableCount(); + } + + @Override + public int getPartitionCount() throws MetaException { + return objectStore.getPartitionCount(); + } + + @Override + public int getDatabaseCount() throws MetaException { + return objectStore.getDatabaseCount(); + } } diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 8dde0af..bf20e99 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -804,6 +804,21 @@ public boolean isFileMetadataSupported() { public void getFileMetadataByExpr(List fileIds, FileMetadataExprType type, byte[] expr, ByteBuffer[] metadatas, ByteBuffer[] stripeBitsets, boolean[] eliminated) { } + + @Override + public int getTableCount() throws MetaException { + return 0; + } + + @Override + public int getPartitionCount() throws MetaException { + return 0; + } + + @Override + public int getDatabaseCount() throws MetaException { + return 0; + } }