commit cbf6e5e45e38a580bfcea35734cb8bcb832937a1 Author: liyintang Date: 3 hours ago HBASE-4689 Make the table level metrics work with rpc* metrics Summary: Previously the rpc metric cannot work with table level metric and does not generate correct metric key. Fix this problem in this diff. diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 6c821c0..0dc1753 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -268,43 +268,6 @@ public class HRegion implements HeapSize { // , Writable{ Pair>(); /** - * Method to transform a set of column families in byte[] format into a - * signature for printing out in metrics - * - * @param families - * the ordered set of column families - * @return a string to print out in metrics - */ - public static String createMutationSignature(Set families) { - int limit = families.size(); - if (1 == limit) { - return SchemaMetrics.CF_PREFIX + - Bytes.toString(families.iterator().next()); - } - - List sortedFamilies = new ArrayList(families); - Collections.sort(sortedFamilies, Bytes.BYTES_COMPARATOR); - - StringBuilder sb = new StringBuilder(SchemaMetrics.CF_PREFIX); - - int MAX_SIZE = 256; - for (byte[] family : sortedFamilies) { - if (sb.length() > MAX_SIZE) { - sb.append("__more"); - break; - } - - --limit; - sb.append(Bytes.toString(family)); - if (0 != limit) { - sb.append("~"); - } - } - - return sb.toString(); - } - - /** * Method to transform a single column family in byte[] format into a * signature for printing out in metrics. Used as overloading so as to not * create an extra Set. Could have gone further and imposed restriction on the @@ -1646,8 +1609,9 @@ public class HRegion implements HeapSize { // , Writable{ // do after lock long after = EnvironmentEdgeManager.currentTimeMillis(); - String signature = HRegion.createMutationSignature(familyMap.keySet()); - HRegion.incrTimeVaryingMetric(signature + ".delete_", after - now); + String signature = SchemaMetrics.generateSchemaMetricsPrefix( + this.getTableDesc().getNameAsString(), familyMap.keySet()); + HRegion.incrTimeVaryingMetric(signature + "delete_", after - now); if (flush) { // Request a cache flush. Do it outside update lock. @@ -1840,12 +1804,14 @@ public class HRegion implements HeapSize { // , Writable{ // else, if all have been consistent so far, check if it still holds // all else, designate failure signature and mark as unclear if (null == signature) { - signature = HRegion.createMutationSignature(put.getFamilyMap() + signature = SchemaMetrics.generateSchemaMetricsPrefix( + this.getTableDesc().getNameAsString(), put.getFamilyMap() .keySet()); } else { if (isSignatureClear) { - if (!signature.equals(HRegion.createMutationSignature(put - .getFamilyMap().keySet()))) { + if (!signature.equals(SchemaMetrics.generateSchemaMetricsPrefix( + this.getTableDesc().getNameAsString(), + put.getFamilyMap().keySet()))) { isSignatureClear = false; signature = SchemaMetrics.CF_UNKNOWN_PREFIX; } @@ -1911,7 +1877,7 @@ public class HRegion implements HeapSize { // , Writable{ if (null == signature) { signature = SchemaMetrics.CF_BAD_FAMILY_PREFIX; } - HRegion.incrTimeVaryingMetric(signature + ".multiput_", after - now); + HRegion.incrTimeVaryingMetric(signature + "multiput_", after - now); if (!success) { for (int i = firstIndex; i < lastIndexExclusive; i++) { @@ -2130,8 +2096,9 @@ public class HRegion implements HeapSize { // , Writable{ // do after lock long after = EnvironmentEdgeManager.currentTimeMillis(); - String signature = HRegion.createMutationSignature(familyMap.keySet()); - HRegion.incrTimeVaryingMetric(signature + ".put_", after - now); + String signature = SchemaMetrics.generateSchemaMetricsPrefix( + this.getTableDesc().getNameAsString(), familyMap.keySet()); + HRegion.incrTimeVaryingMetric(signature + "put_", after - now); if (flush) { // Request a cache flush. Do it outside update lock. @@ -3406,8 +3373,9 @@ public class HRegion implements HeapSize { // , Writable{ // do after lock long after = EnvironmentEdgeManager.currentTimeMillis(); - String signature = HRegion.createMutationSignature(get.familySet()); - HRegion.incrTimeVaryingMetric(signature + ".get_", after - now); + String signature = SchemaMetrics.generateSchemaMetricsPrefix( + this.getTableDesc().getNameAsString(), get.familySet()); + HRegion.incrTimeVaryingMetric(signature + "get_", after - now); return results; } diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java index 8776ffe..ad0172c 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java @@ -23,6 +23,7 @@ package org.apache.hadoop.hbase.regionserver.metrics; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -37,6 +38,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; /** @@ -161,12 +163,11 @@ public class SchemaMetrics { * per-CF/table metrics. */ public static final String UNKNOWN = "__unknown"; - - private static final String TABLE_PREFIX = "tab."; + public static final String TABLE_PREFIX = "tbl."; public static final String CF_PREFIX = "cf."; public static final String BLOCK_TYPE_PREFIX = "bt."; - public static final String CF_UNKNOWN_PREFIX = CF_PREFIX + UNKNOWN; - public static final String CF_BAD_FAMILY_PREFIX = CF_PREFIX + "__badfamily"; + public static final String CF_UNKNOWN_PREFIX = CF_PREFIX + UNKNOWN + "."; + public static final String CF_BAD_FAMILY_PREFIX = CF_PREFIX + "__badfamily."; /** * A special schema metric value that means "all tables aggregated" or @@ -231,8 +232,7 @@ public class SchemaMetrics { private SchemaMetrics(final String tableName, final String cfName) { String metricPrefix = - tableName.equals(TOTAL_KEY) ? "" : TABLE_PREFIX + tableName + "."; - metricPrefix += cfName.equals(TOTAL_KEY) ? "" : CF_PREFIX + cfName + "."; + SchemaMetrics.generateSchemaMetricsPrefix(tableName, cfName); for (BlockCategory blockCategory : BlockCategory.values()) { for (boolean isCompaction : BOOL_VALUES) { @@ -292,25 +292,12 @@ public class SchemaMetrics { tableName = UNKNOWN; } - if (!tableName.equals(TOTAL_KEY)) { - // We are provided with a non-trivial table name (including "unknown"). - // We need to know whether table name should be included into metrics. - if (useTableNameGlobally == null) { - throw new IllegalStateException("The value of the " - + SHOW_TABLE_NAME_CONF_KEY + " conf option has not been specified " - + "in SchemaMetrics"); - } - final boolean useTableName = useTableNameGlobally; - if (!useTableName) { - // Don't include table name in metric keys. - tableName = TOTAL_KEY; - } - } - if (cfName == null) { cfName = UNKNOWN; } + tableName = getEffectiveTableName(tableName); + final String instanceKey = tableName + "\t" + cfName; SchemaMetrics schemaMetrics = cfToMetrics.get(instanceKey); if (schemaMetrics != null) { @@ -482,6 +469,79 @@ public class SchemaMetrics { } } + private static String getEffectiveTableName(String tableName) { + if (!tableName.equals(TOTAL_KEY)) { + // We are provided with a non-trivial table name (including "unknown"). + // We need to know whether table name should be included into metrics. + if (useTableNameGlobally == null) { + throw new IllegalStateException("The value of the " + + SHOW_TABLE_NAME_CONF_KEY + " conf option has not been specified " + + "in SchemaMetrics"); + } + final boolean useTableName = useTableNameGlobally; + if (!useTableName) { + // Don't include table name in metric keys. + tableName = TOTAL_KEY; + } + } + return tableName; + } + + /** + * Method to transform the column family with the table name + * into a schemaMetrics prefix which is used for printing out in metrics + * + * @param tableName + * @param cfName the column family name + * @return schemaMetricsPrefix + */ + private static String generateSchemaMetricsPrefix(final String tableName, + final String cfName){ + String schemaMetricPrefix = + tableName.equals(TOTAL_KEY) ? "" : TABLE_PREFIX + tableName + "."; + schemaMetricPrefix += + cfName.equals(TOTAL_KEY) ? "" : CF_PREFIX + cfName + "."; + + return schemaMetricPrefix; + } + + /** + * Method to transform a set of column families in byte[] format with table name + * into a schemaMetrics prefix which is used for printing out in metrics + * + * @param tableName + * @param families the ordered set of column families + * @return schemaMetricsPrefix + */ + public static String generateSchemaMetricsPrefix(String tableName, + Set families) { + if (families == null || families.size() == 0 || + tableName == null || tableName.length() == 0) + return ""; + + tableName = getEffectiveTableName(tableName); + List sortedFamilies = new ArrayList(families); + Collections.sort(sortedFamilies, Bytes.BYTES_COMPARATOR); + + StringBuilder sb = new StringBuilder(); + + int MAX_SIZE = 256; + int limit = families.size(); + for (byte[] family : sortedFamilies) { + if (sb.length() > MAX_SIZE) { + sb.append("__more"); + break; + } + --limit; + sb.append(Bytes.toString(family)); + if (0 != limit) { + sb.append("~"); + } + } + + return SchemaMetrics.generateSchemaMetricsPrefix(tableName, sb.toString()); + } + /** * Sets the flag of whether to use table name in metric names. This flag * is specified in configuration and is not expected to change at runtime, diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java b/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java index c03532a..f30bd7d 100644 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java @@ -20,30 +20,29 @@ package org.apache.hadoop.hbase.regionserver.metrics; +import static org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BOOL_VALUES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; import org.apache.hadoop.hbase.io.hfile.BlockType; import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; - -import static org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics. - BOOL_VALUES; -import static org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics. - BlockMetricType; - import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; -import static org.junit.Assert.*; - @RunWith(Parameterized.class) public class TestSchemaMetrics { @@ -73,8 +72,8 @@ public class TestSchemaMetrics { @Test public void testNaming() { - final String metricPrefix = (useTableName ? "tab." + - TABLE_NAME + "." : "") + "cf." + CF_NAME + "."; + final String metricPrefix = (useTableName ? SchemaMetrics.TABLE_PREFIX + + TABLE_NAME + "." : "") + SchemaMetrics.CF_PREFIX + CF_NAME + "."; SchemaMetrics schemaMetrics = SchemaMetrics.getInstance(TABLE_NAME, CF_NAME); SchemaMetrics ALL_CF_METRICS = SchemaMetrics.ALL_SCHEMA_METRICS; @@ -223,4 +222,33 @@ public class TestSchemaMetrics { sc.heapSize()); } + @Test + public void testGenerateSchemaMetricsPrefix() { + String tableName = "table1"; + int numCF = 3; + + StringBuilder expected = new StringBuilder(); + if (useTableName) { + expected.append(SchemaMetrics.TABLE_PREFIX); + expected.append(tableName); + expected.append("."); + } + expected.append(SchemaMetrics.CF_PREFIX); + Set families = new HashSet(); + for (int i = 1; i <= numCF; i++) { + String cf = "cf" + i; + families.add(Bytes.toBytes(cf)); + expected.append(cf); + if (i == numCF) { + expected.append("."); + } else { + expected.append("~"); + } + } + + String result = SchemaMetrics.generateSchemaMetricsPrefix(tableName, + families); + assertEquals(expected.toString(), result); + } + }