diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/CompatibilitySingletonFactory.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/CompatibilitySingletonFactory.java index 97e887f..2b2c53d 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/CompatibilitySingletonFactory.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/CompatibilitySingletonFactory.java @@ -44,6 +44,7 @@ public class CompatibilitySingletonFactory extends CompatibilityFactory { * * @return the singleton */ + @SuppressWarnings("unchecked") public static synchronized T getInstance(Class klass) { T instance = (T) instances.get(klass); if (instance == null) { diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetricsSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetricsSource.java index 8fcfaf0..5d932ed 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetricsSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/metrics/MasterMetricsSource.java @@ -28,7 +28,7 @@ public interface MasterMetricsSource extends BaseMetricsSource { /** * The name of the metrics */ - static final String METRICS_NAME = "HMaster"; + static final String METRICS_NAME = "General"; /** * The context metrics will be under. @@ -38,7 +38,7 @@ public interface MasterMetricsSource extends BaseMetricsSource { /** * The name of the metrics context that metrics will be under in jmx */ - static final String METRICS_JMX_CONTEXT = "HMaster"; + static final String METRICS_JMX_CONTEXT = "HMaster,sub=General"; /** * Description diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSource.java index e8cefef..747df28 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSource.java @@ -53,11 +53,11 @@ public interface BaseMetricsSource { void decGauge(String gaugeName, long delta); /** - * Remove a gauge and no longer announce it. + * Remove a metric and no longer announce it. * * @param key Name of the gauge to remove. */ - void removeGauge(String key); + void removeMetric(String key); /** * Add some amount to a counter. @@ -84,12 +84,4 @@ public interface BaseMetricsSource { */ void updateQuantile(String name, long value); - - /** - * Remove a counter and stop announcing it to metrics2. - * - * @param key - */ - void removeCounter(String key); - } diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/HRegionMetricsWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/HRegionMetricsWrapper.java new file mode 100644 index 0000000..3de2c29 --- /dev/null +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/HRegionMetricsWrapper.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +/** + * Interface of class that will wrap an HRegion and export metrics so that they can + */ +public interface HRegionMetricsWrapper { + + /** + * Get the name of the table the region belongs to. + * @return The string version of the table name. + */ + String getTableName(); + + /** + * Get the name of the region. + * @return The encoded name of the region. + */ + String getRegionName(); + + /** + * Get the number of stores hosted on this region server. + */ + long getNumStores(); + + /** + * Get the number of store files hosted on this region server. + */ + long getNumStoreFiles(); + + /** + * Get the size of the memstore on this region server. + */ + long getMemstoreSize(); + + /** + * Get the total size of the store files this region server is serving from. + */ + long getStoreFileSize(); + +} diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/HRegionServerMetricsWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/HRegionServerMetricsWrapper.java new file mode 100644 index 0000000..f066fdc --- /dev/null +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/HRegionServerMetricsWrapper.java @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +/** + * This is the interface that will expose RegionServer information to hadoop1/hadoop2 implementations of the + * RegionServerGeneralMetricsSource. + */ +public interface HRegionServerMetricsWrapper { + + /** + * Get ServerName + */ + public String getServerName(); + + /** + * Get the Cluster ID + * @return Cluster ID + */ + public String getClusterId(); + + /** + * Get the Zookeeper Quorum Info + * @return Zookeeper Quorum Info + */ + public String getZookeeperQuorum(); + + /** + * Get the co-processors + * @return Co-processors + */ + public String getCoprocessors(); + + /** + * Get HRegionServer start time + * @return Start time of RegionServer in milliseconds + */ + public long getRegionServerStartTime(); + + /** + * The number of online regions + */ + long getNumOnlineRegions(); + + /** + * Get the number of stores hosted on this region server. + */ + long getNumStores(); + + /** + * Get the number of store files hosted on this region server. + */ + long getNumStoreFiles(); + + /** + * Get the size of the memstore on this region server. + */ + long getMemstoreSize(); + + /** + * Get the total size of the store files this region server is serving from. + */ + long getStoreFileSize(); + + /** + * Get the number of requests per second. + */ + double getRequestsPerSecond(); + + /** + * Get the total number of requests per second. + */ + long getTotalRequestCount(); + + /** + * Get the number of read requests to regions hosted on this region server. + */ + long getReadRequestsCount(); + + /** + * Get the number of write requests to regions hosted on this region server. + */ + long getWriteRequestsCount(); + + /** + * Get the number of CAS operations that failed. + */ + long getCheckAndMutateChecksFailed(); + + /** + * Get the number of CAS operations that passed. + */ + long getCheckAndMutateChecksPassed(); + + /** + * Get the Size of indexes in storefiles on disk. + */ + long getStoreFileIndexSize(); + + /** + * Get the size of of the static indexes including the roots. + */ + long getTotalStaticIndexSize(); + + /** + * Get the size of the static bloom filters. + */ + long getTotalStaticBloomSize(); + + /** + * Number of mutations received with WAL explicitly turned off. + */ + long getNumPutsWithoutWAL(); + + /** + * Ammount of data in the memstore but not in the WAL because mutations explicitly had their + * WAL turned off. + */ + long getDataInMemoryWithoutWAL(); + + /** + * Get the percent of HFiles' that are local. + */ + int getPercentFileLocal(); + + /** + * Get the size of the compaction queue + */ + int getCompactionQueueSize(); + + /** + * Get the size of the flush queue. + */ + int getFlushQueueSize(); + + /** + * Get the size of the block cache that is free. + */ + long getBlockCacheFreeSize(); + + /** + * Get the number of items in the block cache. + */ + long getBlockCacheCount(); + + /** + * Get the total size of the block cache. + */ + long getBlockCacheSize(); + + /** + * Get the count of hits to the block cache + */ + long getBlockCacheHitCount(); + + /** + * Get the count of misses to the block cache. + */ + long getBlockCacheMissCount(); + + /** + * Get the number of items evicted from the block cache. + */ + long getBlockCacheEvictedCount(); + + /** + * Get the percent of all requests that hit the block cache. + */ + int getBlockCacheHitPercent(); + + /** + * Get the percent of requests with the block cache turned on that hit the block cache. + */ + int getBlockCacheHitCachingPercent(); + + /** + * Force a recomputation of the metrics. + */ + void forceRecompute(); +} diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionAggregateMetricsSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionAggregateMetricsSource.java new file mode 100644 index 0000000..4fc9d0e --- /dev/null +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionAggregateMetricsSource.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.hadoop.hbase.metrics.BaseMetricsSource; + +/** + * This interface will be implemented by a MetricsSource that will export metrics from + * multiple regions into the hadoop metrics system. + */ +public interface RegionAggregateMetricsSource extends BaseMetricsSource { + + /** + * The name of the metrics + */ + static final String METRICS_NAME = "Detailed"; + + /** + * The name of the metrics context that metrics will be under. + */ + static final String METRICS_CONTEXT = "hregionserver"; + + /** + * Description + */ + static final String METRICS_DESCRIPTION = "Metrics about HBase RegionServer regions and tables"; + + /** + * The name of the metrics context that metrics will be under in jmx + */ + static final String METRICS_JMX_CONTEXT = "HRegionServer,sub=Detailed"; + + /** + * Register a RegionMetricSource as being open. + * @param regionSource the source for the region being opened. + */ + void register(RegionMetricSource regionSource); + + /** + * Remove a region's source. This is called when a region is closed. + * @param regionSource The region to remove. + */ + void deregister(RegionMetricSource regionSource); +} diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricSource.java new file mode 100644 index 0000000..ba8cfe5 --- /dev/null +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricSource.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + + +/** + * This interface will be implemented to allow single regions to push metrics into + * RegionAffregateMetricsSource that will in turn push data to the Hadoop metrics system. + */ +public interface RegionMetricSource extends Comparable { + + /** + * Close the region's metrics as this region is closing. + */ + void close(); + + /** + * Update related counts of puts. + */ + void updatePut(); + + /** + * Update related counts of deletes. + */ + void updateDelete(); + + /** + * Update related counts of gets. + */ + void updateGet(); + + /** + * Update related counts of increments. + */ + void updateIncrement(); + + /** + * Update related counts of appends. + */ + void updateAppend(); + + /** + * Get the metrics wrapper around the region. + */ + HRegionMetricsWrapper getRegionWrapper(); + +} diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerGeneralMetricsSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerGeneralMetricsSource.java new file mode 100644 index 0000000..e7f63cd --- /dev/null +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerGeneralMetricsSource.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.hadoop.hbase.metrics.BaseMetricsSource; + +/** + * Interface that classes that expose metrics about the regionserver will implement. + */ +public interface RegionServerGeneralMetricsSource extends BaseMetricsSource { + + /** + * The name of the metrics + */ + static final String METRICS_NAME = "General"; + + /** + * The name of the metrics context that metrics will be under. + */ + static final String METRICS_CONTEXT = "hregionserver"; + + /** + * Description + */ + static final String METRICS_DESCRIPTION = "Metrics about HBase RegionServer"; + + /** + * The name of the metrics context that metrics will be under in jmx + */ + static final String METRICS_JMX_CONTEXT = "HRegionServer,sub=General"; + + + void updatePut(long t); + + void updateDelete(long t); + + void updateGet(long t); + + void updateIncrement(long t); + + void updateAppend(long t); + + + // Strings used for exporting to metrics system. + static final String REGION_COUNT = "regionCount"; + static final String REGION_COUNT_DESC = "Number of regions"; + static final String STORE_COUNT = "storeCount"; + static final String STORE_COUNT_DESC = "Number of Stores"; + static final String STOREFILE_COUNT = "storeFileCount"; + static final String STOREFILE_COUNT_DESC = "Number of Store Files"; + static final String MEMSTORE_SIZE = "memStoreSize"; + static final String MEMSTORE_SIZE_DESC = "Size of the memstore"; + static final String STOREFILE_SIZE = "storeFileSize"; + static final String STOREFILE_SIZE_DESC = "Size of storefiles being served."; + static final String TOTAL_REQUEST_COUNT = "totalRequestCount"; + static final String TOTAL_REQUEST_COUNT_DESC = + "Total number of requests this RegionServer has answered."; + static final String READ_REQUEST_COUNT = "readRequestCount"; + static final String READ_REQUEST_COUNT_DESC = + "Number of read requests this region server has answered."; + static final String WRITE_REQUEST_COUNT = "writeRequestCount"; + static final String WRITE_REQUEST_COUNT_DESC = + "Number of mutation requests this region server has answered."; + static final String CHECK_MUTATE_FAILED_COUNT = "checkMutateFailedCount"; + static final String CHECK_MUTATE_FAILED_COUNT_DESC = + "Number of Check and Mutate calls that failed the checks."; + static final String CHECK_MUTATE_PASSED_COUNT = "checkMutatePassedCount"; + static final String CHECK_MUTATE_PASSED_COUNT_DESC = + "Number of Check and Mutate calls that passed the checks."; + static final String STOREFILE_INDEX_SIZE = "storeFileIndexSize"; + static final String STOREFILE_INDEX_SIZE_DESC = "Size of indexes in storefiles on disk."; + static final String STATIC_INDEX_SIZE = "staticIndexSize"; + static final String STATIC_INDEX_SIZE_DESC = "Uncompressed size of the static indexes."; + static final String STATIC_BLOOM_SIZE = "staticBloomSize"; + static final String STATIC_BLOOM_SIZE_DESC = + "Uncompressed size of the static bloom filters."; + static final String NUMBER_OF_PUTS_WITHOUT_WAL = "putsWithoutWALCount"; + static final String NUMBER_OF_PUTS_WITHOUT_WAL_DESC = + "Number of mutations that have been sent by clients with the write ahead logging turned off."; + static final String DATA_SIZE_WITHOUT_WAL = "putsWithoutWALSize"; + static final String DATA_SIZE_WITHOUT_WAL_DESC = + "Size of data that has been sent by clients with the write ahead logging turned off."; + static final String PERCENT_FILES_LOCAL = "percentFilesLocal"; + static final String PERCENT_FILES_LOCAL_DESC = + "The percent of HFiles that are stored on the local hdfs data node."; + static final String COMPACTION_QUEUE_LENGTH = "compactionQueueLength"; + static final String COMPACTION_QUEUE_LENGTH_DESC = "Length of the queue for compactions."; + static final String FLUSH_QUEUE_LENGTH = "flushQueueLength"; + static final String FLUSH_QUEUE_LENGTH_DESC = "Length of the queue for region flushes"; + static final String BLOCK_CACHE_FREE_SIZE = "blockCacheFreeSize"; + static final String BLOCK_CACHE_FREE_DESC = + "Size of the block cache that is not occupied."; + static final String BLOCK_CACHE_COUNT = "blockCacheCount"; + static final String BLOCK_CACHE_COUNT_DESC = "Number of block in the block cache."; + static final String BLOCK_CACHE_SIZE = "blockCacheSize"; + static final String BLOCK_CACHE_SIZE_DESC = "Size of the block cache."; + static final String BLOCK_CACHE_HIT_COUNT = "blockCacheHitCount"; + static final String BLOCK_CACHE_HIT_COUNT_DESC = "Count of the hit on the block cache."; + static final String BLOCK_CACHE_MISS_COUNT = "blockCacheMissCount"; + static final String BLOCK_COUNT_MISS_COUNT_DESC = + "Number of requests for a block that missed the block cache."; + static final String BLOCK_CACHE_EVICTION_COUNT = "blockCacheEvictionCount"; + static final String BLOCK_CACHE_EVICTION_COUNT_DESC = + "Count of the number of blocks evicted from the block cache."; + static final String BLOCK_CACHE_HIT_PERCENT = "blockCountHitPercent"; + static final String BLOCK_CACHE_HIT_PERCENT_DESC = + "Percent of block cache requests that are hits"; + static final String BLOCK_CACHE_EXPRESS_HIT_PERCENT = "blockCacheExpressHitPercent"; + static final String BLOCK_CACHE_EXPRESS_HIT_PERCENT_DESC = + "The percent of the time that requests with the cache turned on hit the cache."; + static final String RS_START_TIME_NAME = "regionServerStartTime"; + static final String ZOOKEEPER_QUORUM_NAME = "zookeeperQuorum"; + static final String SERVER_NAME_NAME = "serverName"; + static final String CLUSTER_ID_NAME = "clusterId"; + static final String RS_START_TIME_DESC = "RegionServer Start Time"; + static final String ZOOKEEPER_QUORUM_DESC = "Zookeeper Quorum"; + static final String SERVER_NAME_DESC = "Server Name"; + static final String CLUSTER_ID_DESC = "Cluster Id"; + + + static final String DELETE_KEY = "delete"; + static final String GET_KEY = "get"; + static final String INCREMENT_KEY = "increment"; + static final String PUT_KEY = "multiput"; + static final String APPEND_KEY = "append"; +} diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceFactory.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceFactory.java new file mode 100644 index 0000000..0313b30 --- /dev/null +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceFactory.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +/** + * Interface of a factory to create Metrics Sources used inside of regionservers. + */ +public interface RegionServerMetricsSourceFactory { + + /** + * Given a wrapper create a RegionServerGeneralMetricsSource. + * @param wrapper The wrapped region server + * @return a Metrics Source. + */ + RegionServerGeneralMetricsSource createGeneral(HRegionServerMetricsWrapper wrapper); + + /** + * Create a RegionMetricSource from a HRegionMetricsWrapper. + * @param wrapper + * @return + */ + RegionMetricSource createRegion(HRegionMetricsWrapper wrapper); +} diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSource.java index 0090f49..713c38c 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationMetricsSource.java @@ -33,12 +33,12 @@ public interface ReplicationMetricsSource extends BaseMetricsSource { /** * The name of the metrics context that metrics will be under. */ - static final String METRICS_CONTEXT = "replicationmetrics"; + static final String METRICS_CONTEXT = "hregionserver"; /** * The name of the metrics context that metrics will be under. */ - static final String METRICS_JMX_CONTEXT = "ReplicationMetrics"; + static final String METRICS_JMX_CONTEXT = "HRegionServer,sub=Replication"; /** * A description. diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics/MetricHistogram.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics/MetricHistogram.java deleted file mode 100644 index c5d6e49..0000000 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics/MetricHistogram.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metrics; - -/** - * - */ -public interface MetricHistogram { - - static final String NUM_OPS_METRIC_NAME = "_num_ops"; - static final String MIN_METRIC_NAME = "_min"; - static final String MAX_METRIC_NAME = "_max"; - static final String MEAN_METRIC_NAME = "_mean"; - static final String MEDIAN_METRIC_NAME = "_median"; - static final String SEVENTY_FIFTH_PERCENTILE_METRIC_NAME = "_75th_percentile"; - static final String NINETY_FIFTH_PERCENTILE_METRIC_NAME = "_95th_percentile"; - static final String NINETY_NINETH_PERCENTILE_METRIC_NAME = "_99th_percentile"; - - void add(long value); - -} diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics/MetricsExecutor.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics/MetricsExecutor.java deleted file mode 100644 index 4094922..0000000 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics/MetricsExecutor.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metrics; - -import java.util.concurrent.ScheduledExecutorService; - -/** - * - */ -public interface MetricsExecutor { - - ScheduledExecutorService getExecutor(); - - void stop(); - -} diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java new file mode 100644 index 0000000..5e94ac3 --- /dev/null +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricHistogram.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2; + +/** + * + */ +public interface MetricHistogram { + + static final String NUM_OPS_METRIC_NAME = "_num_ops"; + static final String MIN_METRIC_NAME = "_min"; + static final String MAX_METRIC_NAME = "_max"; + static final String MEAN_METRIC_NAME = "_mean"; + static final String MEDIAN_METRIC_NAME = "_median"; + static final String SEVENTY_FIFTH_PERCENTILE_METRIC_NAME = "_75th_percentile"; + static final String NINETY_FIFTH_PERCENTILE_METRIC_NAME = "_95th_percentile"; + static final String NINETY_NINETH_PERCENTILE_METRIC_NAME = "_99th_percentile"; + + void add(long value); + +} diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricsExecutor.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricsExecutor.java new file mode 100644 index 0000000..d102658 --- /dev/null +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/metrics2/MetricsExecutor.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2; + +import java.util.concurrent.ScheduledExecutorService; + +/** + * + */ +public interface MetricsExecutor { + + ScheduledExecutorService getExecutor(); + + void stop(); + +} diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestRegionServerMetricsSourceFactory.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestRegionServerMetricsSourceFactory.java new file mode 100644 index 0000000..5950659 --- /dev/null +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestRegionServerMetricsSourceFactory.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.junit.Test; + +/** + * Test for the CompatibilitySingletonFactory and building RegionServerGeneralMetricsSource + */ +public class TestRegionServerMetricsSourceFactory { + + @Test(expected=RuntimeException.class) + public void testGetInstanceNoHadoopCompat() throws Exception { + //This should throw an exception because there is no compat lib on the class path. + CompatibilitySingletonFactory.getInstance(RegionServerMetricsSourceFactory.class); + + } +} diff --git a/hbase-hadoop1-compat/pom.xml b/hbase-hadoop1-compat/pom.xml index eacde23..6b55471 100644 --- a/hbase-hadoop1-compat/pom.xml +++ b/hbase-hadoop1-compat/pom.xml @@ -98,6 +98,10 @@ limitations under the License. metrics-core + log4j + log4j + + org.apache.hadoop hadoop-test ${hadoop-one.version} diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java index 0943370..b49114b 100644 --- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java @@ -19,13 +19,10 @@ package org.apache.hadoop.hbase.metrics; import org.apache.hadoop.metrics2.MetricsBuilder; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsSource; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry; -import org.apache.hadoop.metrics2.lib.MetricMutableCounterLong; -import org.apache.hadoop.metrics2.lib.MetricMutableGaugeLong; -import org.apache.hadoop.metrics2.lib.MetricMutableHistogram; -import org.apache.hadoop.metrics2.lib.MetricMutableQuantiles; +import org.apache.hadoop.metrics2.impl.JmxCacheBuster; +import org.apache.hadoop.metrics2.lib.*; import org.apache.hadoop.metrics2.source.JvmMetricsSource; /** @@ -137,22 +134,15 @@ public class BaseMetricsSourceImpl implements BaseMetricsSource, MetricsSource { } /** - * Remove a named gauge. + * Remove a named metric. * * @param key */ - public void removeGauge(String key) { + public void removeMetric(String key) { metricsRegistry.removeMetric(key); + JmxCacheBuster.clearJmxCache(); } - /** - * Remove a named counter. - * - * @param key - */ - public void removeCounter(String key) { - metricsRegistry.removeMetric(key); - } /** * Method to export all the metrics. @@ -162,14 +152,16 @@ public class BaseMetricsSourceImpl implements BaseMetricsSource, MetricsSource { */ @Override public void getMetrics(MetricsBuilder metricsBuilder, boolean all) { - metricsRegistry.snapshot(metricsBuilder.addRecord(metricsRegistry.name()), all); + MetricsRecordBuilder mrb = metricsBuilder.addRecord(metricsName) + .setContext(metricsContext); + metricsRegistry.snapshot(mrb, all); } /** * Used to get at the DynamicMetricsRegistry. * @return DynamicMetricsRegistry */ - protected DynamicMetricsRegistry getMetricsRegistry() { + public DynamicMetricsRegistry getMetricsRegistry() { return metricsRegistry; } } diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionAggregateMetricsSourceImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionAggregateMetricsSourceImpl.java new file mode 100644 index 0000000..d5fed0f --- /dev/null +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionAggregateMetricsSourceImpl.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.metrics.BaseMetricsSourceImpl; +import org.apache.hadoop.metrics2.MetricsBuilder; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; + +import java.util.TreeSet; + +public class RegionAggregateMetricsSourceImpl extends BaseMetricsSourceImpl + implements RegionAggregateMetricsSource { + private enum SingleTonStorage { + INSTANCE; + RegionAggregateMetricsSourceImpl instance = new RegionAggregateMetricsSourceImpl(); + } + + private final Log LOG = LogFactory.getLog(this.getClass()); + + public static RegionAggregateMetricsSourceImpl getInstance() { + return SingleTonStorage.INSTANCE.instance; + } + + private final + TreeSet + regionSources = + new TreeSet(); + + public RegionAggregateMetricsSourceImpl() { + this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); + } + + + public RegionAggregateMetricsSourceImpl(String metricsName, + String metricsDescription, + String metricsContext, + String metricsJmxContext) { + super(metricsName, metricsDescription, metricsContext, metricsJmxContext); + } + + @Override + public void register(RegionMetricSource regionSource) { + regionSources.add((RegionMetricSourceImpl) regionSource); + } + + @Override + public void deregister(RegionMetricSource regionSource) { + regionSources.remove(regionSource); + } + + @Override + public void getMetrics(MetricsBuilder metricsBuilder, boolean all) { + + + MetricsRecordBuilder mrb = metricsBuilder.addRecord(metricsName) + .setContext(metricsContext); + + if (regionSources != null) { + for (RegionMetricSourceImpl regionMetricSource : regionSources) { + regionMetricSource.snapshot(mrb, all); + } + } + + + metricsRegistry.snapshot(mrb, all); + + } +} diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricSourceImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricSourceImpl.java new file mode 100644 index 0000000..20ec315 --- /dev/null +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricSourceImpl.java @@ -0,0 +1,205 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.impl.JmxCacheBuster; +import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry; +import org.apache.hadoop.metrics2.lib.MetricMutableCounterLong; + +public class RegionMetricSourceImpl implements RegionMetricSource { + + private final HRegionMetricsWrapper regionWrapper; + private boolean closed = false; + private RegionAggregateMetricsSourceImpl agg; + private DynamicMetricsRegistry registry; + private static final Log LOG = LogFactory.getLog(RegionMetricSourceImpl.class); + + private String regionNamePrefix; + private String tableNamePrefix; + + private String tablePutKey; + private String tableDeleteKey; + private String tableGetKey; + private String tableIncrementKey; + private String tableAppendKey; + private MetricMutableCounterLong tablePut; + private MetricMutableCounterLong tableDelete; + private MetricMutableCounterLong tableGet; + private MetricMutableCounterLong tableIncrement; + private MetricMutableCounterLong tableAppend; + + + private String regionPutKey; + private String regionDeleteKey; + private String regionGetKey; + private String regionIncrementKey; + private String regionAppendKey; + private MetricMutableCounterLong regionPut; + private MetricMutableCounterLong regionDelete; + private MetricMutableCounterLong regionGet; + private MetricMutableCounterLong regionIncrement; + private MetricMutableCounterLong regionAppend; + + + public RegionMetricSourceImpl(HRegionMetricsWrapper regionWrapper, + RegionAggregateMetricsSourceImpl aggregate) { + this.regionWrapper = regionWrapper; + agg = aggregate; + agg.register(this); + + LOG.debug("Creating new RegionMetricSourceImpl for table " + + regionWrapper.getTableName() + + " " + + regionWrapper.getRegionName()); + + registry = agg.getMetricsRegistry(); + + tableNamePrefix = "table." + regionWrapper.getTableName() + "."; + + regionNamePrefix = tableNamePrefix + "region." + regionWrapper.getRegionName() + "."; + String suffix = "Count"; + + + tablePutKey = tableNamePrefix + RegionServerGeneralMetricsSource.PUT_KEY + suffix; + tablePut = registry.getLongCounter(tablePutKey, 0l); + + tableDeleteKey = tableNamePrefix + RegionServerGeneralMetricsSource.DELETE_KEY + suffix; + tableDelete = registry.getLongCounter(tableDeleteKey, 0l); + + tableGetKey = tableNamePrefix + RegionServerGeneralMetricsSource.GET_KEY + suffix; + tableGet = registry.getLongCounter(tableGetKey, 0l); + + tableIncrementKey = tableNamePrefix + RegionServerGeneralMetricsSource.INCREMENT_KEY + suffix; + tableIncrement = registry.getLongCounter(tableIncrementKey, 0l); + + tableAppendKey = tableNamePrefix + RegionServerGeneralMetricsSource.APPEND_KEY + suffix; + tableAppend = registry.getLongCounter(tableAppendKey, 0l); + + + regionPutKey = regionNamePrefix + RegionServerGeneralMetricsSource.PUT_KEY + suffix; + regionPut = registry.getLongCounter(regionPutKey, 0l); + + regionDeleteKey = regionNamePrefix + RegionServerGeneralMetricsSource.DELETE_KEY + suffix; + regionDelete = registry.getLongCounter(regionDeleteKey, 0l); + + regionGetKey = regionNamePrefix + RegionServerGeneralMetricsSource.GET_KEY + suffix; + regionGet = registry.getLongCounter(regionGetKey, 0l); + + regionIncrementKey = regionNamePrefix + RegionServerGeneralMetricsSource.INCREMENT_KEY + suffix; + regionIncrement = registry.getLongCounter(regionIncrementKey, 0l); + + regionAppendKey = regionNamePrefix + RegionServerGeneralMetricsSource.APPEND_KEY + suffix; + regionAppend = registry.getLongCounter(regionAppendKey, 0l); + + + } + + @Override + public void close() { + closed = true; + agg.deregister(this); + + + LOG.trace("Removing Table Metrics: " + regionWrapper.getTableName()); + registry.removeMetric(tablePutKey); + registry.removeMetric(tableDeleteKey); + registry.removeMetric(tableGetKey); + registry.removeMetric(tableIncrementKey); + + registry.removeMetric(tableAppendKey); + + + LOG.trace("Removing region Metrics: " + regionWrapper.getRegionName()); + registry.removeMetric(regionPutKey); + registry.removeMetric(regionDeleteKey); + registry.removeMetric(regionGetKey); + registry.removeMetric(regionIncrementKey); + + registry.removeMetric(regionAppendKey); + + JmxCacheBuster.clearJmxCache(); + } + + @Override + public void updatePut() { + tablePut.incr(); + regionPut.incr(); + } + + + @Override + public void updateDelete() { + tableDelete.incr(); + regionDelete.incr(); + } + + + @Override + public void updateGet() { + tableGet.incr(); + regionGet.incr(); + } + + @Override + public void updateIncrement() { + tableIncrement.incr(); + regionIncrement.incr(); + } + + @Override + public void updateAppend() { + tableAppend.incr(); + regionAppend.incr(); + } + + @Override + public HRegionMetricsWrapper getRegionWrapper() { + return regionWrapper; + } + + @Override + public int compareTo(RegionMetricSource regionMetricSource) { + return this.regionWrapper + .getRegionName() + .compareTo(regionMetricSource.getRegionWrapper().getRegionName()); + } + + void snapshot(MetricsRecordBuilder mrb, boolean ignored) { + if (closed) return; + + mrb.addGauge(regionNamePrefix + RegionServerGeneralMetricsSource.STORE_COUNT, + RegionServerGeneralMetricsSource.STORE_COUNT_DESC, + this.regionWrapper.getNumStores()); + mrb.addGauge(regionNamePrefix + RegionServerGeneralMetricsSource.STOREFILE_COUNT, + RegionServerGeneralMetricsSource.STOREFILE_COUNT_DESC, + this.regionWrapper.getNumStoreFiles()); + mrb.addGauge(regionNamePrefix + RegionServerGeneralMetricsSource.STOREFILE_COUNT, + RegionServerGeneralMetricsSource.STOREFILE_COUNT_DESC, + this.regionWrapper.getNumStoreFiles()); + mrb.addGauge(regionNamePrefix + RegionServerGeneralMetricsSource.MEMSTORE_SIZE, + RegionServerGeneralMetricsSource.MEMSTORE_SIZE_DESC, + this.regionWrapper.getMemstoreSize()); + mrb.addGauge(regionNamePrefix + RegionServerGeneralMetricsSource.STOREFILE_SIZE, + RegionServerGeneralMetricsSource.STOREFILE_SIZE_DESC, + this.regionWrapper.getStoreFileSize()); + } +} diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerGeneralMetricsSourceImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerGeneralMetricsSourceImpl.java new file mode 100644 index 0000000..f53faf7 --- /dev/null +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerGeneralMetricsSourceImpl.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.hadoop.hbase.metrics.BaseMetricsSourceImpl; +import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.metrics2.MetricsBuilder; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; + +/** + * Hadoop1 implementation of RegionServerGeneralMetricsSource. + */ +public class RegionServerGeneralMetricsSourceImpl + extends BaseMetricsSourceImpl implements RegionServerGeneralMetricsSource { + + final HRegionServerMetricsWrapper rsWrap; + private final MetricHistogram putHisto; + private final MetricHistogram deleteHisto; + private final MetricHistogram getHisto; + private final MetricHistogram incrementHisto; + private final MetricHistogram appendHisto; + + public RegionServerGeneralMetricsSourceImpl(HRegionServerMetricsWrapper rsWrap) { + this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT, rsWrap); + } + + public RegionServerGeneralMetricsSourceImpl(String metricsName, + String metricsDescription, + String metricsContext, + String metricsJmxContext, + HRegionServerMetricsWrapper rsWrap) { + super(metricsName, metricsDescription, metricsContext, metricsJmxContext); + this.rsWrap = rsWrap; + + putHisto = getMetricsRegistry().getHistogram(PUT_KEY); + deleteHisto = getMetricsRegistry().getHistogram(DELETE_KEY); + getHisto = getMetricsRegistry().getHistogram(GET_KEY); + incrementHisto = getMetricsRegistry().getHistogram(INCREMENT_KEY); + appendHisto = getMetricsRegistry().getHistogram(APPEND_KEY); + } + + @Override + public void init() { + super.init(); + } + + @Override + public void updatePut(long t) { + putHisto.add(t); + } + + @Override + public void updateDelete(long t) { + deleteHisto.add(t); + } + + @Override + public void updateGet(long t) { + getHisto.add(t); + } + + @Override + public void updateIncrement(long t) { + incrementHisto.add(t); + } + + @Override + public void updateAppend(long t) { + appendHisto.add(t); + } + + /** + * Method to export all the metrics. + * + * @param metricsBuilder Builder to accept metrics + * @param all push all or only changed? + */ + @Override + public void getMetrics(MetricsBuilder metricsBuilder, boolean all) { + + MetricsRecordBuilder mrb = metricsBuilder.addRecord(metricsName) + .setContext(metricsContext); + + // rsWrap can be null because this function is called inside of init. + if (rsWrap != null) { + mrb.addGauge(REGION_COUNT, REGION_COUNT_DESC, rsWrap.getNumOnlineRegions()) + .addGauge(STORE_COUNT, STORE_COUNT_DESC, rsWrap.getNumStores()) + .addGauge(STOREFILE_COUNT, STOREFILE_COUNT_DESC, rsWrap.getNumStoreFiles()) + .addGauge(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC, rsWrap.getMemstoreSize()) + .addGauge(STOREFILE_SIZE, STOREFILE_SIZE_DESC, rsWrap.getStoreFileSize()) + .addGauge(RS_START_TIME_NAME, RS_START_TIME_DESC, rsWrap.getRegionServerStartTime()) + .addCounter(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC, rsWrap.getTotalRequestCount()) + .addCounter(READ_REQUEST_COUNT, READ_REQUEST_COUNT_DESC, rsWrap.getReadRequestsCount()) + .addCounter(WRITE_REQUEST_COUNT, WRITE_REQUEST_COUNT_DESC, rsWrap.getWriteRequestsCount()) + .addCounter(CHECK_MUTATE_FAILED_COUNT, + CHECK_MUTATE_FAILED_COUNT_DESC, + rsWrap.getCheckAndMutateChecksFailed()) + .addCounter(CHECK_MUTATE_PASSED_COUNT, + CHECK_MUTATE_PASSED_COUNT_DESC, + rsWrap.getCheckAndMutateChecksPassed()) + .addGauge(STOREFILE_INDEX_SIZE, STOREFILE_INDEX_SIZE_DESC, rsWrap.getStoreFileIndexSize()) + .addGauge(STATIC_INDEX_SIZE, STATIC_INDEX_SIZE_DESC, rsWrap.getTotalStaticIndexSize()) + .addGauge(STATIC_BLOOM_SIZE, STATIC_BLOOM_SIZE_DESC, rsWrap.getTotalStaticBloomSize()) + .addCounter(NUMBER_OF_PUTS_WITHOUT_WAL, + NUMBER_OF_PUTS_WITHOUT_WAL_DESC, + rsWrap.getNumPutsWithoutWAL()) + .addGauge(DATA_SIZE_WITHOUT_WAL, + DATA_SIZE_WITHOUT_WAL_DESC, + rsWrap.getDataInMemoryWithoutWAL()) + .addGauge(PERCENT_FILES_LOCAL, PERCENT_FILES_LOCAL_DESC, rsWrap.getPercentFileLocal()) + .addGauge(COMPACTION_QUEUE_LENGTH, + COMPACTION_QUEUE_LENGTH_DESC, + rsWrap.getCompactionQueueSize()) + .addGauge(FLUSH_QUEUE_LENGTH, FLUSH_QUEUE_LENGTH_DESC, rsWrap.getFlushQueueSize()) + .addGauge(BLOCK_CACHE_FREE_SIZE, BLOCK_CACHE_FREE_DESC, rsWrap.getBlockCacheFreeSize()) + .addGauge(BLOCK_CACHE_COUNT, BLOCK_CACHE_COUNT_DESC, rsWrap.getBlockCacheCount()) + .addGauge(BLOCK_CACHE_SIZE, BLOCK_CACHE_SIZE_DESC, rsWrap.getBlockCacheSize()) + .addCounter(BLOCK_CACHE_HIT_COUNT, + BLOCK_CACHE_HIT_COUNT_DESC, + rsWrap.getBlockCacheHitCount()) + .addCounter(BLOCK_CACHE_MISS_COUNT, + BLOCK_COUNT_MISS_COUNT_DESC, + rsWrap.getBlockCacheMissCount()) + .addCounter(BLOCK_CACHE_EVICTION_COUNT, + BLOCK_CACHE_EVICTION_COUNT_DESC, + rsWrap.getBlockCacheEvictedCount()) + .addGauge(BLOCK_CACHE_HIT_PERCENT, + BLOCK_CACHE_HIT_PERCENT_DESC, + rsWrap.getBlockCacheHitPercent()) + .addGauge(BLOCK_CACHE_EXPRESS_HIT_PERCENT, + BLOCK_CACHE_EXPRESS_HIT_PERCENT_DESC, + rsWrap.getBlockCacheHitCachingPercent()) + .tag(ZOOKEEPER_QUORUM_NAME, ZOOKEEPER_QUORUM_DESC, rsWrap.getZookeeperQuorum()) + .tag(SERVER_NAME_NAME, SERVER_NAME_DESC, rsWrap.getServerName()) + .tag(CLUSTER_ID_NAME, CLUSTER_ID_DESC, rsWrap.getClusterId()); + } + + metricsRegistry.snapshot(mrb, all); + } + + +} diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceFactoryImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceFactoryImpl.java new file mode 100644 index 0000000..0c07d81 --- /dev/null +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceFactoryImpl.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +/** + * Factory to create RegionServerGeneralMetricsSource when given a HRegionServerMetricsWrapper + */ +public class RegionServerMetricsSourceFactoryImpl implements RegionServerMetricsSourceFactory { + private static enum FactoryStorage { + INSTANCE; + private RegionServerGeneralMetricsSource sourceGeneral; + private RegionAggregateMetricsSourceImpl aggImpl; + } + + private synchronized RegionAggregateMetricsSourceImpl getAggregate() { + if (FactoryStorage.INSTANCE.aggImpl == null) { + FactoryStorage.INSTANCE.aggImpl = new RegionAggregateMetricsSourceImpl(); + } + return FactoryStorage.INSTANCE.aggImpl; + } + + + @Override + public synchronized RegionServerGeneralMetricsSource createGeneral(HRegionServerMetricsWrapper beanWrapper) { + if (FactoryStorage.INSTANCE.sourceGeneral == null) { + FactoryStorage.INSTANCE.sourceGeneral = new RegionServerGeneralMetricsSourceImpl(beanWrapper); + } + return FactoryStorage.INSTANCE.sourceGeneral; + } + + @Override + public RegionMetricSource createRegion(HRegionMetricsWrapper wrapper) { + return new RegionMetricSourceImpl(wrapper, getAggregate()); + } +} diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/thrift/metrics/ThriftServerMetricsSourceImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/thrift/metrics/ThriftServerMetricsSourceImpl.java index 7e5d0c4..1f6abd4 100644 --- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/thrift/metrics/ThriftServerMetricsSourceImpl.java +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/thrift/metrics/ThriftServerMetricsSourceImpl.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.thrift.metrics; import org.apache.hadoop.hbase.metrics.BaseMetricsSourceImpl; -import org.apache.hadoop.hbase.thrift.metrics.ThriftServerMetricsSource; import org.apache.hadoop.metrics2.lib.MetricMutableGaugeLong; import org.apache.hadoop.metrics2.lib.MetricMutableStat; diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java new file mode 100644 index 0000000..63a773c --- /dev/null +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.impl; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; + + +public class JmxCacheBuster { + private static final Log LOG = LogFactory.getLog(JmxCacheBuster.class); + + public static void clearJmxCache() { + LOG.trace("Clearing JMX mbean cache."); + + // This is pretty extreme but it's the best way that + // I could find to get metrics to be removed. + + try { + DefaultMetricsSystem.INSTANCE.stop(); + DefaultMetricsSystem.INSTANCE.start(); + } catch (Exception exception ) { + LOG.debug("error clearing the jmx it appears the metrics system hasn't been started", exception); + } + } +} diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java index 04fb2a9..3f0bc47 100644 --- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java @@ -23,6 +23,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.metrics2.MetricsException; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.MetricsTag; @@ -39,6 +41,8 @@ import org.apache.hadoop.metrics2.MetricsTag; */ public class DynamicMetricsRegistry { + private final Log LOG = LogFactory.getLog(this.getClass()); + /** key for the context tag */ public static final String CONTEXT_KEY = "context"; /** description for the context tag */ @@ -284,6 +288,7 @@ public class DynamicMetricsRegistry { * @param all get all the metrics even if the values are not changed. */ public void snapshot(MetricsRecordBuilder builder, boolean all) { + for (Entry entry : tags()) { builder.add(entry.getValue()); } diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableHistogram.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableHistogram.java index 166af08..b7c24dd 100644 --- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableHistogram.java +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableHistogram.java @@ -21,9 +21,8 @@ package org.apache.hadoop.metrics2.lib; import com.yammer.metrics.stats.ExponentiallyDecayingSample; import com.yammer.metrics.stats.Sample; import com.yammer.metrics.stats.Snapshot; -import org.apache.hadoop.metrics.MetricHistogram; +import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsRecordBuilder; -import org.apache.hadoop.metrics2.lib.MetricMutable; import java.util.concurrent.atomic.AtomicLong; diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java index 7f4b71b..e80095f 100644 --- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java @@ -20,8 +20,8 @@ package org.apache.hadoop.metrics2.lib; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.metrics.MetricHistogram; -import org.apache.hadoop.metrics.MetricsExecutor; +import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.metrics2.MetricsExecutor; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.util.MetricQuantile; import org.apache.hadoop.metrics2.util.MetricSampleQuantiles; diff --git a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java index 3135758..d47912c 100644 --- a/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java +++ b/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java @@ -18,7 +18,7 @@ package org.apache.hadoop.metrics2.lib; -import org.apache.hadoop.metrics.MetricsExecutor; +import org.apache.hadoop.metrics2.MetricsExecutor; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; diff --git a/hbase-hadoop1-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetricsSourceFactory b/hbase-hadoop1-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetricsSourceFactory new file mode 100644 index 0000000..650c6c3 --- /dev/null +++ b/hbase-hadoop1-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetricsSourceFactory @@ -0,0 +1 @@ +org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetricsSourceFactoryImpl diff --git a/hbase-hadoop1-compat/src/main/resources/META-INF/services/org.apache.hadoop.metrics2.MetricsExecutor b/hbase-hadoop1-compat/src/main/resources/META-INF/services/org.apache.hadoop.metrics2.MetricsExecutor new file mode 100644 index 0000000..dc12052 --- /dev/null +++ b/hbase-hadoop1-compat/src/main/resources/META-INF/services/org.apache.hadoop.metrics2.MetricsExecutor @@ -0,0 +1 @@ +org.apache.hadoop.metrics2.lib.MetricsExecutorImpl diff --git a/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseMetricsSourceImplTest.java b/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseMetricsSourceImplTest.java index 095cb14..a68f9ce 100644 --- a/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseMetricsSourceImplTest.java +++ b/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseMetricsSourceImplTest.java @@ -81,17 +81,11 @@ public class TestBaseMetricsSourceImplTest { } @Test - public void testRemoveGauge() throws Exception { + public void testRemoveMetric() throws Exception { bmsi.setGauge("testrm", 100); - bmsi.removeGauge("testrm"); + bmsi.removeMetric("testrm"); assertNull(bmsi.metricsRegistry.get("testrm")); } - @Test - public void testRemoveCounter() throws Exception { - bmsi.incCounters("testrm", 100); - bmsi.removeCounter("testrm"); - assertNull(bmsi.metricsRegistry.get("testrm")); - } } diff --git a/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceImplTest.java b/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceImplTest.java new file mode 100644 index 0000000..0f7944b --- /dev/null +++ b/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceImplTest.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.junit.Test; + +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * Test for RegionServerGeneralMetricsSourceImpl + */ +public class RegionServerMetricsSourceImplTest { + + @Test + public void testGetInstance() throws Exception { + RegionServerMetricsSourceFactory regionServerMetricsSourceFactory = + CompatibilitySingletonFactory.getInstance(RegionServerMetricsSourceFactory.class); + RegionServerGeneralMetricsSource regionServerGeneralMetricsSource = + regionServerMetricsSourceFactory.createGeneral(null); + assertTrue(regionServerGeneralMetricsSource instanceof RegionServerGeneralMetricsSourceImpl); + assertSame(regionServerMetricsSourceFactory, + CompatibilitySingletonFactory.getInstance(RegionServerMetricsSourceFactory.class)); + } + + + @Test(expected = RuntimeException.class) + public void testNoGetRegionServerMetricsSourceImpl() throws Exception { + // This should throw an exception because RegionServerGeneralMetricsSourceImpl should only + // be created by a factory. + CompatibilitySingletonFactory.getInstance(RegionServerGeneralMetricsSourceImpl.class); + } + +} diff --git a/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java b/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java index 346047c..f383c2c 100644 --- a/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java +++ b/hbase-hadoop1-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java @@ -137,7 +137,7 @@ public class MetricsAssertHelperImpl implements MetricsAssertHelper { @Override public void assertGauge(String name, double expected, BaseMetricsSource source) { double found = getGaugeDouble(name, source); - assertEquals("Metrics Should be equal", (double) Double.valueOf(expected), found); + assertEquals("Metrics Should be equal", (double) Double.valueOf(expected), found, 0.01); } @Override diff --git a/hbase-hadoop1-compat/src/test/resources/hadoop-metrics2.properties b/hbase-hadoop1-compat/src/test/resources/hadoop-metrics2.properties new file mode 100644 index 0000000..e69de29 diff --git a/hbase-hadoop2-compat/pom.xml b/hbase-hadoop2-compat/pom.xml index df8e764..754ea10 100644 --- a/hbase-hadoop2-compat/pom.xml +++ b/hbase-hadoop2-compat/pom.xml @@ -138,6 +138,10 @@ limitations under the License. com.yammer.metrics metrics-core + + log4j + log4j + diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java index 7e37089..9654b2e 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/metrics/BaseMetricsSourceImpl.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.metrics; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.impl.JmxCacheBuster; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry; import org.apache.hadoop.metrics2.lib.MetricMutableQuantiles; @@ -141,20 +142,12 @@ public class BaseMetricsSourceImpl implements BaseMetricsSource, MetricsSource { * * @param key */ - public void removeGauge(String key) { + public void removeMetric(String key) { metricsRegistry.removeMetric(key); + JmxCacheBuster.clearJmxCache(); } - /** - * Remove a named counter. - * - * @param key - */ - public void removeCounter(String key) { - metricsRegistry.removeMetric(key); - } - - protected DynamicMetricsRegistry getMetricsRegistry() { + public DynamicMetricsRegistry getMetricsRegistry() { return metricsRegistry; } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionAggregateMetricsSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionAggregateMetricsSourceImpl.java new file mode 100644 index 0000000..a9e4531 --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionAggregateMetricsSourceImpl.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.metrics.BaseMetricsSourceImpl; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; + +import java.util.TreeSet; + +public class RegionAggregateMetricsSourceImpl extends BaseMetricsSourceImpl + implements RegionAggregateMetricsSource { + private enum SingleTonStorage { + INSTANCE; + RegionAggregateMetricsSourceImpl instance = new RegionAggregateMetricsSourceImpl(); + } + + private final Log LOG = LogFactory.getLog(this.getClass()); + + public static RegionAggregateMetricsSourceImpl getInstance() { + return SingleTonStorage.INSTANCE.instance; + } + + private final + TreeSet + regionSources = + new TreeSet(); + + public RegionAggregateMetricsSourceImpl() { + this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); + } + + + public RegionAggregateMetricsSourceImpl(String metricsName, + String metricsDescription, + String metricsContext, + String metricsJmxContext) { + super(metricsName, metricsDescription, metricsContext, metricsJmxContext); + } + + @Override + public void register(RegionMetricSource regionSource) { + regionSources.add((RegionMetricSourceImpl) regionSource); + } + + @Override + public void deregister(RegionMetricSource regionSource) { + regionSources.remove(regionSource); + } + + @Override + public void getMetrics(MetricsCollector collector, boolean all) { + + + MetricsRecordBuilder mrb = collector.addRecord(metricsName) + .setContext(metricsContext); + + if (regionSources != null) { + for (RegionMetricSourceImpl regionMetricSource : regionSources) { + regionMetricSource.snapshot(mrb, all); + } + } + + metricsRegistry.snapshot(mrb, all); + + } +} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricSourceImpl.java new file mode 100644 index 0000000..a3b1985 --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricSourceImpl.java @@ -0,0 +1,205 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.impl.JmxCacheBuster; +import org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry; +import org.apache.hadoop.metrics2.lib.Interns; +import org.apache.hadoop.metrics2.lib.MutableCounterLong; + +public class RegionMetricSourceImpl implements RegionMetricSource { + + private final HRegionMetricsWrapper regionWrapper; + private boolean closed = false; + private RegionAggregateMetricsSourceImpl agg; + private DynamicMetricsRegistry registry; + private static final Log LOG = LogFactory.getLog(RegionMetricSourceImpl.class); + + private String regionNamePrefix; + private String tableNamePrefix; + + private String tablePutKey; + private String tableDeleteKey; + private String tableGetKey; + private String tableIncrementKey; + private String tableAppendKey; + private MutableCounterLong tablePut; + private MutableCounterLong tableDelete; + private MutableCounterLong tableGet; + private MutableCounterLong tableIncrement; + private MutableCounterLong tableAppend; + + + private String regionPutKey; + private String regionDeleteKey; + private String regionGetKey; + private String regionIncrementKey; + private String regionAppendKey; + private MutableCounterLong regionPut; + private MutableCounterLong regionDelete; + private MutableCounterLong regionGet; + private MutableCounterLong regionIncrement; + private MutableCounterLong regionAppend; + + + public RegionMetricSourceImpl(HRegionMetricsWrapper regionWrapper, + RegionAggregateMetricsSourceImpl aggregate) { + this.regionWrapper = regionWrapper; + agg = aggregate; + agg.register(this); + + LOG.debug("Creating new RegionMetricSourceImpl for table " + + regionWrapper.getTableName() + + " " + + regionWrapper.getRegionName()); + + registry = agg.getMetricsRegistry(); + + tableNamePrefix = "table." + regionWrapper.getTableName() + "."; + + regionNamePrefix = tableNamePrefix + "region." + regionWrapper.getRegionName() + "."; + String suffix = "Count"; + + + tablePutKey = tableNamePrefix + RegionServerGeneralMetricsSource.PUT_KEY + suffix; + tablePut = registry.getLongCounter(tablePutKey, 0l); + + tableDeleteKey = tableNamePrefix + RegionServerGeneralMetricsSource.DELETE_KEY + suffix; + tableDelete = registry.getLongCounter(tableDeleteKey, 0l); + + tableGetKey = tableNamePrefix + RegionServerGeneralMetricsSource.GET_KEY + suffix; + tableGet = registry.getLongCounter(tableGetKey, 0l); + + tableIncrementKey = tableNamePrefix + RegionServerGeneralMetricsSource.INCREMENT_KEY + suffix; + tableIncrement = registry.getLongCounter(tableIncrementKey, 0l); + + tableAppendKey = tableNamePrefix + RegionServerGeneralMetricsSource.APPEND_KEY + suffix; + tableAppend = registry.getLongCounter(tableAppendKey, 0l); + + + regionPutKey = regionNamePrefix + RegionServerGeneralMetricsSource.PUT_KEY + suffix; + regionPut = registry.getLongCounter(regionPutKey, 0l); + + regionDeleteKey = regionNamePrefix + RegionServerGeneralMetricsSource.DELETE_KEY + suffix; + regionDelete = registry.getLongCounter(regionDeleteKey, 0l); + + regionGetKey = regionNamePrefix + RegionServerGeneralMetricsSource.GET_KEY + suffix; + regionGet = registry.getLongCounter(regionGetKey, 0l); + + regionIncrementKey = regionNamePrefix + RegionServerGeneralMetricsSource.INCREMENT_KEY + suffix; + regionIncrement = registry.getLongCounter(regionIncrementKey, 0l); + + regionAppendKey = regionNamePrefix + RegionServerGeneralMetricsSource.APPEND_KEY + suffix; + regionAppend = registry.getLongCounter(regionAppendKey, 0l); + + + } + + @Override + public void close() { + closed = true; + agg.deregister(this); + + + LOG.trace("Removing Table Metrics: " + regionWrapper.getTableName()); + registry.removeMetric(tablePutKey); + registry.removeMetric(tableDeleteKey); + registry.removeMetric(tableGetKey); + registry.removeMetric(tableIncrementKey); + + registry.removeMetric(tableAppendKey); + + + LOG.trace("Removing region Metrics: " + regionWrapper.getRegionName()); + registry.removeMetric(regionPutKey); + registry.removeMetric(regionDeleteKey); + registry.removeMetric(regionGetKey); + registry.removeMetric(regionIncrementKey); + + registry.removeMetric(regionAppendKey); + + JmxCacheBuster.clearJmxCache(); + } + + @Override + public void updatePut() { + tablePut.incr(); + regionPut.incr(); + } + + @Override + public void updateDelete() { + tableDelete.incr(); + regionDelete.incr(); + } + + @Override + public void updateGet() { + tableGet.incr(); + regionGet.incr(); + } + + @Override + public void updateIncrement() { + tableIncrement.incr(); + regionIncrement.incr(); + } + + @Override + public void updateAppend() { + tableAppend.incr(); + regionAppend.incr(); + } + + @Override + public HRegionMetricsWrapper getRegionWrapper() { + return regionWrapper; + } + + @Override + public int compareTo(RegionMetricSource regionMetricSource) { + return this.regionWrapper + .getRegionName() + .compareTo(regionMetricSource.getRegionWrapper().getRegionName()); + } + + void snapshot(MetricsRecordBuilder mrb, boolean ignored) { + if (closed) return; + + mrb.addGauge( + Interns.info(regionNamePrefix + RegionServerGeneralMetricsSource.STORE_COUNT, + RegionServerGeneralMetricsSource.STORE_COUNT_DESC), + this.regionWrapper.getNumStores()); + mrb.addGauge(Interns.info(regionNamePrefix + RegionServerGeneralMetricsSource.STOREFILE_COUNT, + RegionServerGeneralMetricsSource.STOREFILE_COUNT_DESC), + this.regionWrapper.getNumStoreFiles()); + mrb.addGauge(Interns.info(regionNamePrefix + RegionServerGeneralMetricsSource.STOREFILE_COUNT, + RegionServerGeneralMetricsSource.STOREFILE_COUNT_DESC), + this.regionWrapper.getNumStoreFiles()); + mrb.addGauge(Interns.info(regionNamePrefix + RegionServerGeneralMetricsSource.MEMSTORE_SIZE, + RegionServerGeneralMetricsSource.MEMSTORE_SIZE_DESC), + this.regionWrapper.getMemstoreSize()); + mrb.addGauge(Interns.info(regionNamePrefix + RegionServerGeneralMetricsSource.STOREFILE_SIZE, + RegionServerGeneralMetricsSource.STOREFILE_SIZE_DESC), + this.regionWrapper.getStoreFileSize()); + } +} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerGeneralMetricsSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerGeneralMetricsSourceImpl.java new file mode 100644 index 0000000..94de9bc --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerGeneralMetricsSourceImpl.java @@ -0,0 +1,160 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.hadoop.hbase.metrics.BaseMetricsSourceImpl; +import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.lib.Interns; + +/** + * Hadoop1 implementation of RegionServerGeneralMetricsSource. + */ +public class RegionServerGeneralMetricsSourceImpl + extends BaseMetricsSourceImpl implements RegionServerGeneralMetricsSource { + + final HRegionServerMetricsWrapper rsWrap; + private final MetricHistogram putHisto; + private final MetricHistogram deleteHisto; + private final MetricHistogram getHisto; + private final MetricHistogram incrementHisto; + private final MetricHistogram appendHisto; + + public RegionServerGeneralMetricsSourceImpl(HRegionServerMetricsWrapper rsWrap) { + this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT, rsWrap); + } + + public RegionServerGeneralMetricsSourceImpl(String metricsName, + String metricsDescription, + String metricsContext, + String metricsJmxContext, + HRegionServerMetricsWrapper rsWrap) { + super(metricsName, metricsDescription, metricsContext, metricsJmxContext); + this.rsWrap = rsWrap; + + putHisto = getMetricsRegistry().getHistogram(PUT_KEY); + deleteHisto = getMetricsRegistry().getHistogram(DELETE_KEY); + getHisto = getMetricsRegistry().getHistogram(GET_KEY); + incrementHisto = getMetricsRegistry().getHistogram(INCREMENT_KEY); + appendHisto = getMetricsRegistry().getHistogram(APPEND_KEY); + } + + @Override + public void init() { + super.init(); + } + + @Override + public void updatePut(long t) { + putHisto.add(t); + } + + @Override + public void updateDelete(long t) { + deleteHisto.add(t); + } + + @Override + public void updateGet(long t) { + getHisto.add(t); + } + + @Override + public void updateIncrement(long t) { + incrementHisto.add(t); + } + + @Override + public void updateAppend(long t) { + appendHisto.add(t); + } + + /** + * Method to export all the metrics. + * + * @param metricsCollector Collector to accept metrics + * @param all push all or only changed? + */ + @Override + public void getMetrics(MetricsCollector metricsCollector, boolean all) { + + MetricsRecordBuilder mrb = metricsCollector.addRecord(metricsName) + .setContext(metricsContext); + + // rsWrap can be null because this function is called inside of init. + if (rsWrap != null) { + mrb.addGauge(Interns.info(REGION_COUNT, REGION_COUNT_DESC), rsWrap.getNumOnlineRegions()) + .addGauge(Interns.info(STORE_COUNT, STORE_COUNT_DESC), rsWrap.getNumStores()) + .addGauge(Interns.info(STOREFILE_COUNT, STOREFILE_COUNT_DESC), rsWrap.getNumStoreFiles()) + .addGauge(Interns.info(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC), rsWrap.getMemstoreSize()) + .addGauge(Interns.info(STOREFILE_SIZE, STOREFILE_SIZE_DESC), rsWrap.getStoreFileSize()) + .addGauge(Interns.info(RS_START_TIME_NAME, RS_START_TIME_DESC), + rsWrap.getRegionServerStartTime()) + .addCounter(Interns.info(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC), + rsWrap.getTotalRequestCount()) + .addCounter(Interns.info(READ_REQUEST_COUNT, READ_REQUEST_COUNT_DESC), + rsWrap.getReadRequestsCount()) + .addCounter(Interns.info(WRITE_REQUEST_COUNT, WRITE_REQUEST_COUNT_DESC), + rsWrap.getWriteRequestsCount()) + .addCounter(Interns.info(CHECK_MUTATE_FAILED_COUNT, CHECK_MUTATE_FAILED_COUNT_DESC), + rsWrap.getCheckAndMutateChecksFailed()) + .addCounter(Interns.info(CHECK_MUTATE_PASSED_COUNT, CHECK_MUTATE_PASSED_COUNT_DESC), + rsWrap.getCheckAndMutateChecksPassed()) + .addGauge(Interns.info(STOREFILE_INDEX_SIZE, STOREFILE_INDEX_SIZE_DESC), + rsWrap.getStoreFileIndexSize()) + .addGauge(Interns.info(STATIC_INDEX_SIZE, STATIC_INDEX_SIZE_DESC), + rsWrap.getTotalStaticIndexSize()) + .addGauge(Interns.info(STATIC_BLOOM_SIZE, STATIC_BLOOM_SIZE_DESC), + rsWrap.getTotalStaticBloomSize()) + .addGauge(Interns.info(NUMBER_OF_PUTS_WITHOUT_WAL, NUMBER_OF_PUTS_WITHOUT_WAL_DESC), + rsWrap.getNumPutsWithoutWAL()) + .addGauge(Interns.info(DATA_SIZE_WITHOUT_WAL, DATA_SIZE_WITHOUT_WAL_DESC), + rsWrap.getDataInMemoryWithoutWAL()) + .addGauge(Interns.info(PERCENT_FILES_LOCAL, PERCENT_FILES_LOCAL_DESC), + rsWrap.getPercentFileLocal()) + .addGauge(Interns.info(COMPACTION_QUEUE_LENGTH, COMPACTION_QUEUE_LENGTH_DESC), + rsWrap.getCompactionQueueSize()) + .addGauge(Interns.info(FLUSH_QUEUE_LENGTH, FLUSH_QUEUE_LENGTH_DESC), + rsWrap.getFlushQueueSize()) + .addGauge(Interns.info(BLOCK_CACHE_FREE_SIZE, BLOCK_CACHE_FREE_DESC), + rsWrap.getBlockCacheFreeSize()) + .addGauge(Interns.info(BLOCK_CACHE_COUNT, BLOCK_CACHE_COUNT_DESC), + rsWrap.getBlockCacheCount()) + .addGauge(Interns.info(BLOCK_CACHE_SIZE, BLOCK_CACHE_SIZE_DESC), + rsWrap.getBlockCacheSize()) + .addCounter(Interns.info(BLOCK_CACHE_HIT_COUNT, BLOCK_CACHE_HIT_COUNT_DESC), + rsWrap.getBlockCacheHitCount()) + .addCounter(Interns.info(BLOCK_CACHE_MISS_COUNT, BLOCK_COUNT_MISS_COUNT_DESC), + rsWrap.getBlockCacheMissCount()) + .addCounter(Interns.info(BLOCK_CACHE_EVICTION_COUNT, BLOCK_CACHE_EVICTION_COUNT_DESC), + rsWrap.getBlockCacheEvictedCount()) + .addGauge(Interns.info(BLOCK_CACHE_HIT_PERCENT, BLOCK_CACHE_HIT_PERCENT_DESC), + rsWrap.getBlockCacheHitPercent()) + .addGauge(Interns.info(BLOCK_CACHE_EXPRESS_HIT_PERCENT, + BLOCK_CACHE_EXPRESS_HIT_PERCENT_DESC), rsWrap.getBlockCacheHitCachingPercent()) + .tag(Interns.info(ZOOKEEPER_QUORUM_NAME, ZOOKEEPER_QUORUM_DESC), + rsWrap.getZookeeperQuorum()) + .tag(Interns.info(SERVER_NAME_NAME, SERVER_NAME_DESC), rsWrap.getServerName()) + .tag(Interns.info(CLUSTER_ID_NAME, CLUSTER_ID_DESC), rsWrap.getClusterId()); + } + + metricsRegistry.snapshot(mrb, all); + } +} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceFactoryImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceFactoryImpl.java new file mode 100644 index 0000000..0c07d81 --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceFactoryImpl.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +/** + * Factory to create RegionServerGeneralMetricsSource when given a HRegionServerMetricsWrapper + */ +public class RegionServerMetricsSourceFactoryImpl implements RegionServerMetricsSourceFactory { + private static enum FactoryStorage { + INSTANCE; + private RegionServerGeneralMetricsSource sourceGeneral; + private RegionAggregateMetricsSourceImpl aggImpl; + } + + private synchronized RegionAggregateMetricsSourceImpl getAggregate() { + if (FactoryStorage.INSTANCE.aggImpl == null) { + FactoryStorage.INSTANCE.aggImpl = new RegionAggregateMetricsSourceImpl(); + } + return FactoryStorage.INSTANCE.aggImpl; + } + + + @Override + public synchronized RegionServerGeneralMetricsSource createGeneral(HRegionServerMetricsWrapper beanWrapper) { + if (FactoryStorage.INSTANCE.sourceGeneral == null) { + FactoryStorage.INSTANCE.sourceGeneral = new RegionServerGeneralMetricsSourceImpl(beanWrapper); + } + return FactoryStorage.INSTANCE.sourceGeneral; + } + + @Override + public RegionMetricSource createRegion(HRegionMetricsWrapper wrapper) { + return new RegionMetricSourceImpl(wrapper, getAggregate()); + } +} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java new file mode 100644 index 0000000..8512afd --- /dev/null +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/impl/JmxCacheBuster.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.metrics2.impl; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; + + +public class JmxCacheBuster { + private static final Log LOG = LogFactory.getLog(JmxCacheBuster.class); + + public static void clearJmxCache() { + LOG.trace("Clearing JMX mbean cache."); + + // This is pretty extreme but it's the best way that + // I could find to get metrics to be removed. + try { + if (DefaultMetricsSystem.instance() != null ) { + DefaultMetricsSystem.instance().stop(); + DefaultMetricsSystem.instance().start(); + } + + } catch (Exception exception ) { + LOG.debug("error clearing the jmx it appears the metrics system hasn't been started", exception); + } + } +} diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java index a423893..080bd4d 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/DynamicMetricsRegistry.java @@ -528,6 +528,7 @@ public class DynamicMetricsRegistry { return returnExistingWithCast(metric, metricClass, name); } + @SuppressWarnings("unchecked") private T returnExistingWithCast(MutableMetric metric, Class metricClass, String name) { if (!metricClass.isAssignableFrom(metric.getClass())) { diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java index 28e92c1..766cf96 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricMutableQuantiles.java @@ -22,8 +22,8 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.metrics.MetricHistogram; -import org.apache.hadoop.metrics.MetricsExecutor; +import org.apache.hadoop.metrics2.MetricHistogram; +import org.apache.hadoop.metrics2.MetricsExecutor; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.util.MetricQuantile; diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java index 3135758..d47912c 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MetricsExecutorImpl.java @@ -18,7 +18,7 @@ package org.apache.hadoop.metrics2.lib; -import org.apache.hadoop.metrics.MetricsExecutor; +import org.apache.hadoop.metrics2.MetricsExecutor; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java index 4fb0be9..3b012e9 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/metrics2/lib/MutableHistogram.java @@ -22,7 +22,7 @@ import com.yammer.metrics.stats.ExponentiallyDecayingSample; import com.yammer.metrics.stats.Sample; import com.yammer.metrics.stats.Snapshot; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.metrics.MetricHistogram; +import org.apache.hadoop.metrics2.MetricHistogram; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; diff --git a/hbase-hadoop2-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetricsSourceFactory b/hbase-hadoop2-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetricsSourceFactory new file mode 100644 index 0000000..650c6c3 --- /dev/null +++ b/hbase-hadoop2-compat/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetricsSourceFactory @@ -0,0 +1 @@ +org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetricsSourceFactoryImpl diff --git a/hbase-hadoop2-compat/src/main/resources/META-INF/services/org.apache.hadoop.metrics2.MetricsExecutor b/hbase-hadoop2-compat/src/main/resources/META-INF/services/org.apache.hadoop.metrics2.MetricsExecutor new file mode 100644 index 0000000..dc12052 --- /dev/null +++ b/hbase-hadoop2-compat/src/main/resources/META-INF/services/org.apache.hadoop.metrics2.MetricsExecutor @@ -0,0 +1 @@ +org.apache.hadoop.metrics2.lib.MetricsExecutorImpl diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseMetricsSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseMetricsSourceImpl.java index f334702..6f5a98e 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseMetricsSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/metrics/TestBaseMetricsSourceImpl.java @@ -75,16 +75,10 @@ public class TestBaseMetricsSourceImpl { } @Test - public void testRemoveGauge() throws Exception { + public void testRemoveMetric() throws Exception { bmsi.setGauge("testrmgauge", 100); - bmsi.removeGauge("testrmgauge"); + bmsi.removeMetric("testrmgauge"); assertNull(bmsi.metricsRegistry.get("testrmgauge")); } - @Test - public void testRemoveCounter() throws Exception { - bmsi.incCounters("testrmcounter", 100); - bmsi.removeCounter("testrmcounter"); - assertNull(bmsi.metricsRegistry.get("testrmcounter")); - } } diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceImplTest.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceImplTest.java new file mode 100644 index 0000000..387647a --- /dev/null +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetricsSourceImplTest.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.junit.Test; + +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * Test for RegionServerGeneralMetricsSourceImpl + */ +public class RegionServerMetricsSourceImplTest { + + @Test + public void testGetInstance() throws Exception { + RegionServerMetricsSourceFactory regionServerMetricsSourceFactory = + CompatibilitySingletonFactory.getInstance(RegionServerMetricsSourceFactory.class); + RegionServerGeneralMetricsSource regionServerGeneralMetricsSource = + regionServerMetricsSourceFactory.createGeneral(null); + assertTrue(regionServerGeneralMetricsSource instanceof RegionServerGeneralMetricsSourceImpl); + assertSame(regionServerMetricsSourceFactory, + CompatibilitySingletonFactory.getInstance(RegionServerMetricsSourceFactory.class)); + } + + + @Test(expected = RuntimeException.class) + public void testNoGetRegionServerMetricsSourceImpl() throws Exception { + // This should throw an exception because RegionServerGeneralMetricsSourceImpl should only + // be created by a factory. + CompatibilitySingletonFactory.getInstance(RegionServerGeneralMetricsSourceImpl.class); + } +} diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java index b8b06ab..bcac935 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/test/MetricsAssertHelperImpl.java @@ -156,7 +156,7 @@ public class MetricsAssertHelperImpl implements MetricsAssertHelper { @Override public void assertGauge(String name, double expected, BaseMetricsSource source) { double found = getGaugeDouble(name, source); - assertEquals("Metrics Should be equal", (double) Double.valueOf(expected), found); + assertEquals("Metrics Should be equal", (double) Double.valueOf(expected), found, 0.01); } @Override diff --git a/hbase-hadoop2-compat/src/test/resources/hadoop-metrics2.properties b/hbase-hadoop2-compat/src/test/resources/hadoop-metrics2.properties new file mode 100644 index 0000000..e69de29 diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon index ee66fdd..f664017 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon @@ -24,7 +24,6 @@ String format = "html"; <%import> java.util.*; org.apache.hadoop.hbase.regionserver.HRegionServer; -org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics; org.apache.hadoop.hbase.util.Bytes; org.apache.hadoop.hbase.HRegionInfo; org.apache.hadoop.hbase.ServerName; @@ -38,7 +37,6 @@ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionLoad; <%java return; %> <%java> - RegionServerMetrics metrics = regionServer.getMetrics(); ServerInfo serverInfo = ProtobufUtil.getServerInfo(regionServer); ServerName serverName = ProtobufUtil.toServerName(serverInfo.getServerName()); List onlineRegions = ProtobufUtil.getOnlineRegions(regionServer); @@ -98,7 +96,7 @@ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionLoad;

Server Metrics

- <& ServerMetricsTmpl; metrics = metrics; &> + <& ServerMetricsTmpl; mWrap = regionServer.getMetrics().getRsWrapper(); &> <& ../common/TaskMonitorTmpl; filter = filter &> diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon index ac0fe6f..9977939 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RegionListTmpl.jamon @@ -23,7 +23,6 @@ <%import> java.util.*; org.apache.hadoop.hbase.regionserver.HRegionServer; - org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics; org.apache.hadoop.hbase.util.Bytes; org.apache.hadoop.hbase.HRegionInfo; org.apache.hadoop.hbase.ServerName; diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon index 0478c15..b686906 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon @@ -17,12 +17,12 @@ See the License for the specific language governing permissions and limitations under the License. <%args> - RegionServerMetrics metrics; +HRegionServerMetricsWrapper mWrap; <%import> java.util.*; org.apache.hadoop.hbase.regionserver.HRegionServer; -org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics; +org.apache.hadoop.hbase.regionserver.metrics.HRegionServerMetricsWrapper; org.apache.hadoop.hbase.util.Bytes; org.apache.hadoop.hbase.HRegionInfo; org.apache.hadoop.hbase.ServerName; @@ -46,32 +46,32 @@ java.lang.management.ManagementFactory;
- <& baseStats; metrics = metrics; &> + <& baseStats; mWrap = mWrap &>
- <& memoryStats; metrics = metrics; &> + <& memoryStats; mWrap = mWrap &>
- <& requestStats; metrics = metrics; &> + <& requestStats; mWrap = mWrap &>
- <& storeStats; metrics = metrics; &> + <& storeStats; mWrap = mWrap &>
- <& queueStats; metrics = metrics; &> + <& queueStats; mWrap = mWrap &>
- <& blockCacheStats; metrics = metrics; &> + <& blockCacheStats; mWrap = mWrap &>
- <& latencyStats; metrics = metrics; &> + <& latencyStats; mWrap = mWrap &>
<%def baseStats> <%args> - RegionServerMetrics metrics; + HRegionServerMetricsWrapper mWrap; @@ -82,17 +82,17 @@ java.lang.management.ManagementFactory; - - - - + + + +
Slow HLog Append Count
<% metrics.requests.getPreviousIntervalValue() %><% metrics.regions.get() %><% metrics.hdfsBlocksLocalityIndex.get() %><% metrics.slowHLogAppendCount.get() %><% mWrap.getRequestsPerSecond() %><% mWrap.getNumOnlineRegions() %><% mWrap.getPercentFileLocal() %><% 0 %>
<%def memoryStats> <%args> - RegionServerMetrics metrics; +HRegionServerMetricsWrapper mWrap; @@ -104,19 +104,19 @@ java.lang.management.ManagementFactory; - +
- <% ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed() / (1024*1024) %>MB + <% ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed() %> - <% ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax() / (1024*1024) %>MB + <% ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()%> <% metrics.memstoreSizeMB.get()%>MB<% mWrap.getMemstoreSize() %>
<%def storeStats> <%args> - RegionServerMetrics metrics; +HRegionServerMetricsWrapper mWrap; @@ -128,11 +128,11 @@ java.lang.management.ManagementFactory; - - - - - + + + + +
Bloom Size
<% metrics.stores.get() %><% metrics.storefiles.get() %><% metrics.rootIndexSizeKB.get() %>KB<% metrics.totalStaticIndexSizeKB.get() %>KB<% metrics.totalStaticBloomSizeKB.get() %>KB<% mWrap.getNumStores() %><% mWrap.getNumStoreFiles() %><% mWrap.getStoreFileIndexSize() %><% mWrap.getTotalStaticIndexSize() %><% mWrap.getTotalStaticBloomSize() %>
@@ -140,8 +140,8 @@ java.lang.management.ManagementFactory; <%def requestStats> <%args> - RegionServerMetrics metrics; - +HRegionServerMetricsWrapper mWrap; + @@ -149,17 +149,17 @@ java.lang.management.ManagementFactory; - - - + + +
Request Per SecondWrite Request Count
<% metrics.requests.getPreviousIntervalValue() %><% metrics.readRequestsCount.get() %><% metrics.writeRequestsCount.get() %>KB<% 0 %><% mWrap.getReadRequestsCount() %><% mWrap.getWriteRequestsCount() %>KB
<%def queueStats> <%args> - RegionServerMetrics metrics; - +HRegionServerMetricsWrapper mWrap; + @@ -167,8 +167,8 @@ java.lang.management.ManagementFactory; - - + +
Compaction queue size
<% metrics.compactionQueueSize.get() %><% metrics.flushQueueSize.get() %>KB<% mWrap.getCompactionQueueSize() %><% mWrap.getFlushQueueSize() %>
@@ -176,8 +176,8 @@ java.lang.management.ManagementFactory; <%def blockCacheStats> <%args> - RegionServerMetrics metrics; - +HRegionServerMetricsWrapper mWrap; + @@ -190,21 +190,21 @@ java.lang.management.ManagementFactory; - - - - - - - + + + + + + +
Cache Size
<% metrics.blockCacheSize.get() / (1024*1024) %>MB<% metrics.blockCacheFree.get() / (1024 * 1024) %>MB<% metrics.blockCacheCount.get()%><% metrics.blockCacheHitCount.get()%><% metrics.blockCacheMissCount.get()%><% metrics.blockCacheHitRatio.get()%>%<% metrics.blockCacheEvictedCount.get()%><% mWrap.getBlockCacheSize()%><% mWrap.getBlockCacheFreeSize()%><% mWrap.getBlockCacheCount() %><% mWrap.getBlockCacheHitCount() %><% mWrap.getBlockCacheMissCount() %><% mWrap.getBlockCacheHitPercent() %>%<% mWrap.getBlockCacheEvictedCount() %>
<%def latencyStats> <%args> - RegionServerMetrics metrics; - +HRegionServerMetricsWrapper mWrap; + @@ -217,30 +217,28 @@ java.lang.management.ManagementFactory; - - <& histogramRow; op ="FS Read"; histo = metrics.fsReadLatencyHistogram &> - <& histogramRow; op ="FS PRead"; histo = metrics.fsPreadLatencyHistogram &> - <& histogramRow; op ="FS Write"; histo = metrics.fsWriteLatencyHistogram &> - + <& histogramRow; op ="FS Read" &> + <& histogramRow; op ="FS PRead" &> + <& histogramRow; op ="FS Write" &>
Operation99.9th
<%def histogramRow> <%args> String op; - MetricsHistogram histo; + <%java> - Snapshot s = histo.getSnapshot(); + <% op %> - <% histo.getCount()%> - <% String.format("%10.2f", histo.getMean()) %> - <% String.format("%10.2f", s.getMedian()) %> - <% String.format("%10.2f", s.get75thPercentile()) %> - <% String.format("%10.2f", s.get95thPercentile()) %> - <% String.format("%10.2f", s.get99thPercentile()) %> - <% String.format("%10.2f", s.get999thPercentile())%> + <% 0.0 %> + <% String.format("%10.2f", 0.0) %> + <% String.format("%10.2f", 0.0) %> + <% String.format("%10.2f", 0.0) %> + <% String.format("%10.2f", 0.0) %> + <% String.format("%10.2f", 0.0) %> + <% String.format("%10.2f", 0.0)%> diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java index b871865..ad0105e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java @@ -29,15 +29,13 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured; import org.apache.hadoop.io.RawComparator; /** * Common functionality needed by all versions of {@link HFile} readers. */ @InterfaceAudience.Private -public abstract class AbstractHFileReader extends SchemaConfigured - implements HFile.Reader { +public abstract class AbstractHFileReader implements HFile.Reader { /** Filesystem-level block reader for this HFile format version. */ protected HFileBlock.FSReader fsBlockReader; @@ -118,7 +116,6 @@ public abstract class AbstractHFileReader extends SchemaConfigured final long fileSize, final boolean closeIStream, final CacheConfig cacheConf, final HFileSystem hfs) { - super(null, path); this.trailer = trailer; this.compressAlgo = trailer.getCompressionCodec(); this.cacheConf = cacheConf; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java index 635b407..fe57a13 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue.KeyComparator; import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.io.RawComparator; @@ -43,8 +42,7 @@ import org.apache.hadoop.io.Writable; * Common functionality needed by all versions of {@link HFile} writers. */ @InterfaceAudience.Private -public abstract class AbstractHFileWriter extends SchemaConfigured - implements HFile.Writer { +public abstract class AbstractHFileWriter implements HFile.Writer { /** Key previously appended. Becomes the last key in the file. */ protected byte[] lastKeyBuffer = null; @@ -115,7 +113,6 @@ public abstract class AbstractHFileWriter extends SchemaConfigured Compression.Algorithm compressAlgo, HFileDataBlockEncoder dataBlockEncoder, KeyComparator comparator) { - super(null, path); this.outputStream = outputStream; this.path = path; this.name = path != null ? path.getName() : outputStream.toString(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java index c6b12eb..b205106 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java @@ -23,7 +23,6 @@ import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.io.HeapSize; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; /** * Cacheable is an interface that allows for an object to be cached. If using an @@ -57,14 +56,4 @@ public interface Cacheable extends HeapSize { */ public CacheableDeserializer getDeserializer(); - /** - * @return the block type of this cached HFile block - */ - public BlockType getBlockType(); - - /** - * @return the metrics object identified by table and column family - */ - public SchemaMetrics getSchemaMetrics(); - } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 598e3ba..e63417f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -48,8 +48,6 @@ import org.apache.hadoop.hbase.KeyValue.KeyComparator; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.HbaseMapWritable; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.SchemaAware; import org.apache.hadoop.hbase.util.BloomFilterWriter; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; @@ -283,8 +281,6 @@ public class HFile { /** @return the path to this {@link HFile} */ Path getPath(); - String getColumnFamilyName(); - void appendMetaBlock(String bloomFilterMetaKey, Writable metaWriter); /** @@ -430,7 +426,6 @@ public class HFile { */ public static final WriterFactory getWriterFactory(Configuration conf, CacheConfig cacheConf) { - SchemaMetrics.configureGlobally(conf); int version = getFormatVersion(conf); switch (version) { case 1: @@ -452,8 +447,7 @@ public class HFile { } /** An interface used by clients to open and iterate an {@link HFile}. */ - public interface Reader extends Closeable, CachingBlockReader, - SchemaAware { + public interface Reader extends Closeable, CachingBlockReader { /** * Returns this reader's "name". Usually the last component of the path. * Needs to be constant as the file is being moved to support caching on @@ -461,8 +455,6 @@ public class HFile { */ String getName(); - String getColumnFamilyName(); - RawComparator getComparator(); HFileScanner getScanner(boolean cacheBlocks, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java index c93c8d5..d3a9b0e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm; import org.apache.hadoop.hbase.regionserver.MemStore; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.ClassSize; @@ -84,7 +83,7 @@ import com.google.common.base.Preconditions; * except that the data section is always uncompressed in the cache. */ @InterfaceAudience.Private -public class HFileBlock extends SchemaConfigured implements Cacheable { +public class HFileBlock implements Cacheable { /** Minor versions starting with this number have hbase checksums */ static final int MINOR_VERSION_WITH_CHECKSUM = 1; @@ -538,8 +537,6 @@ public class HFileBlock extends SchemaConfigured implements Cacheable { @Override public long heapSize() { long size = ClassSize.align( - // Base class size, including object overhead. - SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE + // Block type and byte buffer references 2 * ClassSize.REFERENCE + // On-disk size, uncompressed size, and next block's on-disk size diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java index 7fbc06d..c7cf874 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.HFile.CachingBlockReader; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.CompoundBloomFilterWriter; @@ -719,8 +718,7 @@ public class HFileBlockIndex { * index. However, in most practical cases we will only have leaf-level * blocks and the root index, or just the root index. */ - public static class BlockIndexWriter extends SchemaConfigured - implements InlineBlockWriter { + public static class BlockIndexWriter implements InlineBlockWriter { /** * While the index is being written, this represents the current block * index referencing all leaf blocks, with one exception. If the file is @@ -954,7 +952,6 @@ public class HFileBlockIndex { if (blockCache != null) { HFileBlock blockForCaching = blockWriter.getBlockForCaching(); - passSchemaMetricsTo(blockForCaching); blockCache.cacheBlock(new BlockCacheKey(nameForCaching, beginOffset, DataBlockEncoding.NONE, blockForCaching.getBlockType()), blockForCaching); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java index 013218a..603f1c3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java @@ -243,7 +243,6 @@ public class HFileDataBlockEncoderImpl implements HFileDataBlockEncoder { includesMemstoreTS, block.getMinorVersion(), block.getBytesPerChecksum(), block.getChecksumType(), block.getOnDiskDataSizeWithHeader()); - block.passSchemaMetricsTo(encodedBlock); return encodedBlock; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java index e1882b3..be0fb17 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java @@ -43,7 +43,6 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; import org.apache.hadoop.hbase.regionserver.TimeRangeTracker; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.util.BloomFilter; import org.apache.hadoop.hbase.util.BloomFilterFactory; import org.apache.hadoop.hbase.util.ByteBloomFilter; @@ -174,7 +173,6 @@ public class HFilePrettyPrinter { conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR)); conf.set("fs.default.name", conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR)); - SchemaMetrics.configureGlobally(conf); try { if (!parseOptions(args)) return 1; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java index 56339da..436d0c4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java @@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; import org.apache.hadoop.hbase.io.hfile.HFile.Writer; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.RawComparator; @@ -235,8 +234,6 @@ public class HFileReaderV1 extends AbstractHFileReader { cacheConf.shouldCacheBlockOnRead(effectiveCategory)); if (cachedBlock != null) { cacheHits.incrementAndGet(); - getSchemaMetrics().updateOnCacheHit(effectiveCategory, - SchemaMetrics.NO_COMPACTION); return cachedBlock.getBufferWithoutHeader(); } // Cache Miss, please load. @@ -245,13 +242,10 @@ public class HFileReaderV1 extends AbstractHFileReader { HFileBlock hfileBlock = fsBlockReader.readBlockData(offset, nextOffset - offset, metaBlockIndexReader.getRootBlockDataSize(block), true); - passSchemaMetricsTo(hfileBlock); hfileBlock.expectType(BlockType.META); final long delta = System.nanoTime() - startTimeNs; HFile.offerReadLatency(delta, true); - getSchemaMetrics().updateOnCacheMiss(effectiveCategory, - SchemaMetrics.NO_COMPACTION, delta); // Cache the block if (cacheBlock && cacheConf.shouldCacheBlockOnRead(effectiveCategory)) { @@ -300,8 +294,6 @@ public class HFileReaderV1 extends AbstractHFileReader { cacheConf.shouldCacheDataOnRead()); if (cachedBlock != null) { cacheHits.incrementAndGet(); - getSchemaMetrics().updateOnCacheHit( - cachedBlock.getBlockType().getCategory(), isCompaction); return cachedBlock.getBufferWithoutHeader(); } // Carry on, please load. @@ -323,13 +315,10 @@ public class HFileReaderV1 extends AbstractHFileReader { HFileBlock hfileBlock = fsBlockReader.readBlockData(offset, nextOffset - offset, dataBlockIndexReader.getRootBlockDataSize(block), pread); - passSchemaMetricsTo(hfileBlock); hfileBlock.expectType(BlockType.DATA); final long delta = System.nanoTime() - startTimeNs; HFile.offerReadLatency(delta, pread); - getSchemaMetrics().updateOnCacheMiss(BlockCategory.DATA, isCompaction, - delta); // Cache the block if (cacheBlock && cacheConf.shouldCacheBlockOnRead( diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java index e252f38..71e4d09 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java @@ -226,7 +226,6 @@ public class HFileReaderV2 extends AbstractHFileReader { // Return a distinct 'shallow copy' of the block, // so pos does not get messed by the scanner cacheHits.incrementAndGet(); - getSchemaMetrics().updateOnCacheHit(BlockCategory.META, false); return cachedBlock.getBufferWithoutHeader(); } // Cache Miss, please load. @@ -234,11 +233,9 @@ public class HFileReaderV2 extends AbstractHFileReader { HFileBlock metaBlock = fsBlockReader.readBlockData(metaBlockOffset, blockSize, -1, true); - passSchemaMetricsTo(metaBlock); final long delta = System.nanoTime() - startTimeNs; HFile.offerReadLatency(delta, true); - getSchemaMetrics().updateOnCacheMiss(BlockCategory.META, false, delta); // Cache the block if (cacheBlock) { @@ -302,7 +299,6 @@ public class HFileReaderV2 extends AbstractHFileReader { cachedBlock.getBlockType().getCategory(); cacheHits.incrementAndGet(); - getSchemaMetrics().updateOnCacheHit(blockCategory, isCompaction); if (cachedBlock.getBlockType() == BlockType.DATA) { HFile.dataBlockReadCnt.incrementAndGet(); @@ -331,12 +327,10 @@ public class HFileReaderV2 extends AbstractHFileReader { hfileBlock = dataBlockEncoder.diskToCacheFormat(hfileBlock, isCompaction); validateBlockType(hfileBlock, expectedBlockType); - passSchemaMetricsTo(hfileBlock); BlockCategory blockCategory = hfileBlock.getBlockType().getCategory(); final long delta = System.nanoTime() - startTimeNs; HFile.offerReadLatency(delta, pread); - getSchemaMetrics().updateOnCacheMiss(blockCategory, isCompaction, delta); // Cache the block if necessary if (cacheBlock && cacheConf.shouldCacheBlockOnRead( diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java index 3861c00..a4956a6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm; import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo; import org.apache.hadoop.hbase.io.hfile.HFile.Writer; import org.apache.hadoop.hbase.regionserver.MemStore; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.BloomFilterWriter; import org.apache.hadoop.hbase.util.Bytes; @@ -108,7 +107,6 @@ public class HFileWriterV1 extends AbstractHFileWriter { final KeyComparator comparator) throws IOException { super(cacheConf, ostream == null ? createOutputStream(conf, fs, path) : ostream, path, blockSize, compress, blockEncoder, comparator); - SchemaMetrics.configureGlobally(conf); } /** @@ -157,7 +155,6 @@ public class HFileWriterV1 extends AbstractHFileWriter { HFileBlock.HEADER_SIZE_NO_CHECKSUM); // onDiskDataSizeWithHeader block = blockEncoder.diskToCacheFormat(block, false); - passSchemaMetricsTo(block); cacheConf.getBlockCache().cacheBlock( new BlockCacheKey(name, blockBegin, DataBlockEncoding.NONE, block.getBlockType()), block); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java index 92c3a62..63dfc9f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java @@ -36,7 +36,6 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KeyComparator; import org.apache.hadoop.hbase.io.hfile.HFile.Writer; import org.apache.hadoop.hbase.io.hfile.HFileBlock.BlockWritable; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.BloomFilterWriter; import org.apache.hadoop.hbase.util.Bytes; @@ -113,7 +112,6 @@ public class HFileWriterV2 extends AbstractHFileWriter { super(cacheConf, ostream == null ? createOutputStream(conf, fs, path) : ostream, path, blockSize, compressAlgo, blockEncoder, comparator); - SchemaMetrics.configureGlobally(conf); this.checksumType = checksumType; this.bytesPerChecksum = bytesPerChecksum; finishInit(conf); @@ -140,16 +138,6 @@ public class HFileWriterV2 extends AbstractHFileWriter { // Meta data block index writer metaBlockIndexWriter = new HFileBlockIndex.BlockIndexWriter(); LOG.debug("Initialized with " + cacheConf); - - if (isSchemaConfigured()) { - schemaConfigurationChanged(); - } - } - - @Override - protected void schemaConfigurationChanged() { - passSchemaMetricsTo(dataBlockIndexWriter); - passSchemaMetricsTo(metaBlockIndexWriter); } /** @@ -226,7 +214,6 @@ public class HFileWriterV2 extends AbstractHFileWriter { final boolean isCompaction = false; HFileBlock cacheFormatBlock = blockEncoder.diskToCacheFormat( fsBlockWriter.getBlockForCaching(), isCompaction); - passSchemaMetricsTo(cacheFormatBlock); cacheConf.getBlockCache().cacheBlock( new BlockCacheKey(name, offset, blockEncoder.getEncodingInCache(), cacheFormatBlock.getBlockType()), cacheFormatBlock); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index 3373b8c..4dda0c0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -44,7 +44,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.FSUtils; @@ -324,12 +323,6 @@ public class LruBlockCache implements BlockCache, HeapSize { if (evict) { heapsize *= -1; } - Cacheable cachedBlock = cb.getBuffer(); - SchemaMetrics schemaMetrics = cachedBlock.getSchemaMetrics(); - if (schemaMetrics != null) { - schemaMetrics.updateOnCachePutOrEvict( - cachedBlock.getBlockType().getCategory(), heapsize, evict); - } return size.addAndGet(heapsize); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 77f5776..5ecbfe5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -81,7 +81,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants.OperationStatusCode; import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HServerInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.NotServingRegionException; @@ -116,11 +115,8 @@ import org.apache.hadoop.hbase.ipc.HBaseServer; import org.apache.hadoop.hbase.ipc.RpcCallContext; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; -import org.apache.hadoop.hbase.regionserver.metrics.OperationMetrics; -import org.apache.hadoop.hbase.regionserver.metrics.RegionMetricsStorage; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; +import org.apache.hadoop.hbase.regionserver.metrics.RegionMetrics; import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.hbase.regionserver.wal.HLogFactory; import org.apache.hadoop.hbase.regionserver.wal.HLogKey; @@ -235,7 +231,7 @@ public class HRegion implements HeapSize { // , Writable{ // private int [] storeSize = null; // private byte [] name = null; - final AtomicLong memstoreSize = new AtomicLong(0); + public final AtomicLong memstoreSize = new AtomicLong(0); // Debug possible data loss due to WAL off final AtomicLong numPutsWithoutWAL = new AtomicLong(0); @@ -361,7 +357,7 @@ public class HRegion implements HeapSize { // , Writable{ public final static String REGIONINFO_FILE = ".regioninfo"; private HTableDescriptor htableDescriptor = null; private RegionSplitPolicy splitPolicy; - private final OperationMetrics opMetrics; + private final RegionMetrics metrics; /** * Should only be used for testing purposes @@ -385,7 +381,7 @@ public class HRegion implements HeapSize { // , Writable{ this.coprocessorHost = null; this.scannerReadPoints = new ConcurrentHashMap(); - this.opMetrics = new OperationMetrics(); + this.metrics = new RegionMetrics(new HRegionMetricsWrapperImpl(this)); } /** @@ -448,7 +444,7 @@ public class HRegion implements HeapSize { // , Writable{ this.regiondir = getRegionDir(this.tableDir, encodedNameStr); this.scannerReadPoints = new ConcurrentHashMap(); - this.opMetrics = new OperationMetrics(conf, this.regionInfo); + this.metrics = new RegionMetrics(new HRegionMetricsWrapperImpl(this)); /* * timestamp.slop provides a server-side constraint on the timestamp. This @@ -1022,7 +1018,7 @@ public class HRegion implements HeapSize { // , Writable{ status.setStatus("Running coprocessor post-close hooks"); this.coprocessorHost.postClose(abort); } - this.opMetrics.closeMetrics(); + this.metrics.close(); status.markComplete("Closed"); LOG.info("Closed " + this); return result; @@ -1722,7 +1718,6 @@ public class HRegion implements HeapSize { // , Writable{ protected RegionScanner getScanner(Scan scan, List additionalScanners) throws IOException { startRegionOperation(); - this.readRequestsCount.increment(); try { // Verify families are all valid prepareScanner(scan); @@ -2321,26 +2316,20 @@ public class HRegion implements HeapSize { // , Writable{ } } - // do after lock - final long netTimeMs = EnvironmentEdgeManager.currentTimeMillis() - startTimeMs; - // See if the column families were consistent through the whole thing. // if they were then keep them. If they were not then pass a null. // null will be treated as unknown. // Total time taken might be involving Puts and Deletes. // Split the time for puts and deletes based on the total number of Puts and Deletes. - long timeTakenForPuts = 0; + if (noOfPuts > 0) { // There were some Puts in the batch. double noOfMutations = noOfPuts + noOfDeletes; - timeTakenForPuts = (long) (netTimeMs * (noOfPuts / noOfMutations)); - final Set keptCfs = putsCfSetConsistent ? putsCfSet : null; - this.opMetrics.updateMultiPutMetrics(keptCfs, timeTakenForPuts); + this.metrics.updatePut(); } if (noOfDeletes > 0) { // There were some Deletes in the batch. - final Set keptCfs = deletesCfSetConsistent ? deletesCfSet : null; - this.opMetrics.updateMultiDeleteMetrics(keptCfs, netTimeMs - timeTakenForPuts); + this.metrics.updateDelete(); } if (!success) { for (int i = firstIndex; i < lastIndexExclusive; i++) { @@ -3169,7 +3158,7 @@ public class HRegion implements HeapSize { // , Writable{ /** * See if row is currently locked. - * @param lockid + * @param lockId * @return boolean */ boolean isRowLocked(final Integer lockId) { @@ -4238,7 +4227,6 @@ public class HRegion implements HeapSize { // , Writable{ */ private List get(Get get, boolean withCoprocessor) throws IOException { - long now = EnvironmentEdgeManager.currentTimeMillis(); List results = new ArrayList(); @@ -4254,7 +4242,7 @@ public class HRegion implements HeapSize { // , Writable{ RegionScanner scanner = null; try { scanner = getScanner(scan); - scanner.next(results, SchemaMetrics.METRIC_GETSIZE); + scanner.next(results); } finally { if (scanner != null) scanner.close(); @@ -4266,8 +4254,8 @@ public class HRegion implements HeapSize { // , Writable{ } // do after lock - final long after = EnvironmentEdgeManager.currentTimeMillis(); - this.opMetrics.updateGetMetrics(get.familySet(), after - now); + + this.metrics.updateGet(); return results; } @@ -4340,15 +4328,11 @@ public class HRegion implements HeapSize { // , Writable{ processor.postProcess(this, walEdit); } catch (IOException e) { long endNanoTime = System.nanoTime(); - RegionMetricsStorage.incrTimeVaryingMetric(metricsName + ".error.nano", - endNanoTime - startNanoTime); throw e; } finally { closeRegionOperation(); } final long endNanoTime = System.nanoTime(); - RegionMetricsStorage.incrTimeVaryingMetric(metricsName + ".nano", - endNanoTime - startNanoTime); return; } @@ -4455,8 +4439,6 @@ public class HRegion implements HeapSize { // , Writable{ } catch (IOException e) { long endNanoTime = System.nanoTime(); - RegionMetricsStorage.incrTimeVaryingMetric(metricsName + ".error.nano", - endNanoTime - startNanoTime); throw e; } finally { closeRegionOperation(); @@ -4467,20 +4449,6 @@ public class HRegion implements HeapSize { // , Writable{ } // Populate all metrics long endNanoTime = System.nanoTime(); - RegionMetricsStorage.incrTimeVaryingMetric(metricsName + ".nano", - endNanoTime - startNanoTime); - - RegionMetricsStorage.incrTimeVaryingMetric(metricsName + ".acquirelock.nano", - lockedNanoTime - startNanoTime); - - RegionMetricsStorage.incrTimeVaryingMetric(metricsName + ".process.nano", - processDoneNanoTime - lockedNanoTime); - - RegionMetricsStorage.incrTimeVaryingMetric(metricsName + ".occupylock.nano", - unlockedNanoTime - lockedNanoTime); - - RegionMetricsStorage.incrTimeVaryingMetric(metricsName + ".sync.nano", - endNanoTime - unlockedNanoTime); } private void doProcessRowWithTimeout(final RowProcessor processor, @@ -4557,7 +4525,7 @@ public class HRegion implements HeapSize { // , Writable{ WALEdit walEdits = null; List allKVs = new ArrayList(append.size()); Map> tempMemstore = new HashMap>(); - long before = EnvironmentEdgeManager.currentTimeMillis(); + long size = 0; long txid = 0; @@ -4674,8 +4642,7 @@ public class HRegion implements HeapSize { // , Writable{ closeRegionOperation(); } - long after = EnvironmentEdgeManager.currentTimeMillis(); - this.opMetrics.updateAppendMetrics(append.getFamilyMap().keySet(), after - before); + this.metrics.updateAppend(); if (flush) { @@ -4710,7 +4677,7 @@ public class HRegion implements HeapSize { // , Writable{ WALEdit walEdits = null; List allKVs = new ArrayList(increment.numColumns()); Map> tempMemstore = new HashMap>(); - long before = EnvironmentEdgeManager.currentTimeMillis(); + long size = 0; long txid = 0; @@ -4800,8 +4767,7 @@ public class HRegion implements HeapSize { // , Writable{ } } finally { closeRegionOperation(); - long after = EnvironmentEdgeManager.currentTimeMillis(); - this.opMetrics.updateIncrementMetrics(increment.getFamilyMap().keySet(), after - before); + this.metrics.updateIncrement(); } if (flush) { @@ -4826,9 +4792,6 @@ public class HRegion implements HeapSize { // , Writable{ public long incrementColumnValue(byte [] row, byte [] family, byte [] qualifier, long amount, boolean writeToWAL) throws IOException { - // to be used for metrics - long before = EnvironmentEdgeManager.currentTimeMillis(); - checkRow(row, "increment"); boolean flush = false; boolean wrongLength = false; @@ -4901,9 +4864,6 @@ public class HRegion implements HeapSize { // , Writable{ } // do after lock - long after = EnvironmentEdgeManager.currentTimeMillis(); - this.opMetrics.updateIncrementColumnValueMetrics(family, after - before); - if (flush) { // Request a cache flush. Do it outside update lock. requestFlush(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionMetricsWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionMetricsWrapperImpl.java new file mode 100644 index 0000000..e7b80db --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionMetricsWrapperImpl.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.apache.hadoop.hbase.regionserver.metrics.HRegionMetricsWrapper; +import org.apache.hadoop.metrics2.MetricsExecutor; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public class HRegionMetricsWrapperImpl implements HRegionMetricsWrapper { + + public static final int PERIOD = 45; + + private final HRegion region; + private ScheduledExecutorService executor; + private Runnable runnable; + private long numStoreFiles; + private long memstoreSize; + private long storeFileSize; + + public HRegionMetricsWrapperImpl(HRegion region) { + this.region = region; + this.executor = CompatibilitySingletonFactory.getInstance(MetricsExecutor.class).getExecutor(); + this.runnable = new HRegionMetricsWrapperImpl.HRegionMerticsWrappableRunnable(); + this.executor.scheduleWithFixedDelay(this.runnable, PERIOD, PERIOD, TimeUnit.SECONDS); + } + + @Override + public String getTableName() { + return this.region.getTableDesc().getNameAsString(); + } + + @Override + public String getRegionName() { + return this.region.getRegionInfo().getEncodedName(); + } + + @Override + public long getNumStores() { + return this.region.stores.size(); + } + + @Override + public long getNumStoreFiles() { + return numStoreFiles; + } + + @Override + public long getMemstoreSize() { + return memstoreSize; + } + + @Override + public long getStoreFileSize() { + return storeFileSize; + } + + public class HRegionMerticsWrappableRunnable implements Runnable { + + @Override + public void run() { + long tempNumStoreFiles = 0; + long tempMemstoreSize = 0; + long tempStoreFileSize = 0; + + for (Store store : region.stores.values()) { + tempNumStoreFiles += store.getStorefilesCount(); + tempMemstoreSize += store.getMemStoreSize(); + tempStoreFileSize += store.getStorefilesSize(); + } + + numStoreFiles = tempNumStoreFiles; + memstoreSize = tempMemstoreSize; + storeFileSize = tempStoreFileSize; + } + } + +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index cfe6bf7..4c401b3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -48,13 +48,11 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantReadWriteLock; import javax.management.ObjectName; import com.google.protobuf.Message; -import org.apache.commons.lang.mutable.MutableDouble; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -67,7 +65,6 @@ import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.FailedSanityCheckException; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; @@ -106,9 +103,7 @@ import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.ByteArrayComparable; import org.apache.hadoop.hbase.fs.HFileSystem; -import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.CacheConfig; -import org.apache.hadoop.hbase.io.hfile.CacheStats; import org.apache.hadoop.hbase.ipc.CoprocessorProtocol; import org.apache.hadoop.hbase.ipc.HBaseRPC; import org.apache.hadoop.hbase.ipc.HBaseRPCErrorHandler; @@ -190,11 +185,8 @@ import org.apache.hadoop.hbase.regionserver.handler.CloseRootHandler; import org.apache.hadoop.hbase.regionserver.handler.OpenMetaHandler; import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler; import org.apache.hadoop.hbase.regionserver.handler.OpenRootHandler; -import org.apache.hadoop.hbase.regionserver.metrics.RegionMetricsStorage; -import org.apache.hadoop.hbase.regionserver.metrics.RegionServerDynamicMetrics; import org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.StoreMetricType; +import org.apache.hadoop.hbase.regionserver.metrics.HRegionServerMetricsWrapper; import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.hbase.regionserver.wal.HLogUtil; import org.apache.hadoop.hbase.regionserver.wal.HLogFactory; @@ -224,11 +216,11 @@ import org.apache.hadoop.net.DNS; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; import org.apache.zookeeper.KeeperException; +import org.cliffc.high_scale_lib.Counter; import org.codehaus.jackson.map.ObjectMapper; import com.google.common.base.Function; import com.google.protobuf.ByteString; -import com.google.protobuf.Message; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -299,7 +291,7 @@ public class HRegionServer implements ClientProtocol, // Request counter. // Do we need this? Can't we just sum region counters? St.Ack 20110412 - protected AtomicInteger requestCount = new AtomicInteger(); + protected Counter requestCount = new Counter(); // If false, the file system has become unavailable protected volatile boolean fsOk; @@ -368,8 +360,6 @@ public class HRegionServer implements ClientProtocol, private RegionServerMetrics metrics; - private RegionServerDynamicMetrics dynamicMetrics; - /* * Check for compactions requests. */ @@ -403,7 +393,7 @@ public class HRegionServer implements ClientProtocol, private final RegionServerAccounting regionServerAccounting; // Cache configuration and block cache reference - private final CacheConfig cacheConfig; + final CacheConfig cacheConfig; // reference to the Thrift Server. volatile private HRegionThriftServer thriftServer; @@ -446,6 +436,11 @@ public class HRegionServer implements ClientProtocol, */ private final QosFunction qosFunction; + /** + * Time stamps for when a hregionserver was started + */ + private long regionServerStartTime; + /** * Starts a HRegionServer at the default location @@ -550,6 +545,19 @@ public class HRegionServer implements ClientProtocol, } } + + /** + * @return timestamp in millis when HRegionServer was started. + */ + public long getRegionServerStartTime() { + return regionServerStartTime; + } + + public String getClusterId() { + // TODO: cache in this class field? + return this.conf.get(HConstants.CLUSTER_ID); + } + @Retention(RetentionPolicy.RUNTIME) protected @interface QosPriority { int priority() default 0; @@ -834,6 +842,8 @@ public class HRegionServer implements ClientProtocol, * The HRegionServer sticks in this loop until closed. */ public void run() { + regionServerStartTime = System.currentTimeMillis(); + try { // Do pre-registration initializations; zookeeper, lease threads, etc. preRegistrationInitialization(); @@ -854,7 +864,6 @@ public class HRegionServer implements ClientProtocol, break; } } - registerMBean(); // We registered with the Master. Go into run mode. long lastMsg = 0; @@ -889,7 +898,6 @@ public class HRegionServer implements ClientProtocol, } long now = System.currentTimeMillis(); if ((now - lastMsg) >= msgInterval) { - doMetrics(); tryRegionServerReport(lastMsg, now); lastMsg = System.currentTimeMillis(); } @@ -1018,8 +1026,6 @@ public class HRegionServer implements ClientProtocol, void tryRegionServerReport(long reportStartTime, long reportEndTime) throws IOException { HBaseProtos.ServerLoad sl = buildServerLoad(reportStartTime, reportEndTime); - // Why we do this? - this.requestCount.set(0); try { RegionServerReportRequest.Builder request = RegionServerReportRequest.newBuilder(); ServerName sn = ServerName.parseVersionedServerName( @@ -1040,13 +1046,14 @@ public class HRegionServer implements ClientProtocol, } HBaseProtos.ServerLoad buildServerLoad(long reportStartTime, long reportEndTime) { + HRegionServerMetricsWrapper wrapper = this.metrics.getRsWrapper(); Collection regions = getOnlineRegionsLocalContext(); MemoryUsage memory = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); HBaseProtos.ServerLoad.Builder serverLoad = HBaseProtos.ServerLoad.newBuilder(); - serverLoad.setNumberOfRequests((int)metrics.getRequests()); - serverLoad.setTotalNumberOfRequests(requestCount.get()); + serverLoad.setNumberOfRequests((int)wrapper.getRequestsPerSecond()); + serverLoad.setTotalNumberOfRequests((int)requestCount.get()); serverLoad.setUsedHeapMB((int)(memory.getUsed() / 1024 / 1024)); serverLoad.setMaxHeapMB((int) (memory.getMax() / 1024 / 1024)); Set coprocessors = this.hlog.getCoprocessorHost().getCoprocessors(); @@ -1197,8 +1204,7 @@ public class HRegionServer implements ClientProtocol, this.tableDescriptors = new FSTableDescriptors(this.fs, this.rootDir, true); this.hlog = setupWALAndReplication(); // Init in here rather than in constructor after thread name has been set - this.metrics = new RegionServerMetrics(); - this.dynamicMetrics = RegionServerDynamicMetrics.newInstance(); + this.metrics = new RegionServerMetrics(new HRegionServerMetricsWrapperImpl(this)); startServiceThreads(); LOG.info("Serving as " + this.serverNameFromMasterPOV + ", RPC listening on " + this.isa + @@ -1433,170 +1439,8 @@ public class HRegionServer implements ClientProtocol, return hlogRoller; } - /* - * @param interval Interval since last time metrics were called. - */ - protected void doMetrics() { - try { - metrics(); - } catch (Throwable e) { - LOG.warn("Failed metrics", e); - } - } - - protected void metrics() { - this.metrics.regions.set(this.onlineRegions.size()); - this.metrics.incrementRequests(this.requestCount.get()); - this.metrics.requests.intervalHeartBeat(); - // Is this too expensive every three seconds getting a lock on onlineRegions - // and then per store carried? Can I make metrics be sloppier and avoid - // the synchronizations? - int stores = 0; - int storefiles = 0; - long memstoreSize = 0; - int readRequestsCount = 0; - int writeRequestsCount = 0; - long checkAndMutateChecksFailed = 0; - long checkAndMutateChecksPassed = 0; - long storefileIndexSize = 0; - HDFSBlocksDistribution hdfsBlocksDistribution = - new HDFSBlocksDistribution(); - long totalStaticIndexSize = 0; - long totalStaticBloomSize = 0; - long numPutsWithoutWAL = 0; - long dataInMemoryWithoutWAL = 0; - - // Note that this is a map of Doubles instead of Longs. This is because we - // do effective integer division, which would perhaps truncate more than it - // should because we do it only on one part of our sum at a time. Rather - // than dividing at the end, where it is difficult to know the proper - // factor, everything is exact then truncated. - final Map tempVals = - new HashMap(); - - for (Map.Entry e : this.onlineRegions.entrySet()) { - HRegion r = e.getValue(); - memstoreSize += r.memstoreSize.get(); - numPutsWithoutWAL += r.numPutsWithoutWAL.get(); - dataInMemoryWithoutWAL += r.dataInMemoryWithoutWAL.get(); - readRequestsCount += r.readRequestsCount.get(); - writeRequestsCount += r.writeRequestsCount.get(); - checkAndMutateChecksFailed += r.checkAndMutateChecksFailed.get(); - checkAndMutateChecksPassed += r.checkAndMutateChecksPassed.get(); - synchronized (r.stores) { - stores += r.stores.size(); - for (Map.Entry ee : r.stores.entrySet()) { - final Store store = ee.getValue(); - final SchemaMetrics schemaMetrics = store.getSchemaMetrics(); - - { - long tmpStorefiles = store.getStorefilesCount(); - schemaMetrics.accumulateStoreMetric(tempVals, - StoreMetricType.STORE_FILE_COUNT, tmpStorefiles); - storefiles += tmpStorefiles; - } - - - { - long tmpStorefileIndexSize = store.getStorefilesIndexSize(); - schemaMetrics.accumulateStoreMetric(tempVals, - StoreMetricType.STORE_FILE_INDEX_SIZE, - (long) (tmpStorefileIndexSize / (1024.0 * 1024))); - storefileIndexSize += tmpStorefileIndexSize; - } - - { - long tmpStorefilesSize = store.getStorefilesSize(); - schemaMetrics.accumulateStoreMetric(tempVals, - StoreMetricType.STORE_FILE_SIZE_MB, - (long) (tmpStorefilesSize / (1024.0 * 1024))); - } - - { - long tmpStaticBloomSize = store.getTotalStaticBloomSize(); - schemaMetrics.accumulateStoreMetric(tempVals, - StoreMetricType.STATIC_BLOOM_SIZE_KB, - (long) (tmpStaticBloomSize / 1024.0)); - totalStaticBloomSize += tmpStaticBloomSize; - } - - { - long tmpStaticIndexSize = store.getTotalStaticIndexSize(); - schemaMetrics.accumulateStoreMetric(tempVals, - StoreMetricType.STATIC_INDEX_SIZE_KB, - (long) (tmpStaticIndexSize / 1024.0)); - totalStaticIndexSize += tmpStaticIndexSize; - } - - schemaMetrics.accumulateStoreMetric(tempVals, - StoreMetricType.MEMSTORE_SIZE_MB, - (long) (store.getMemStoreSize() / (1024.0 * 1024))); - } - } - - hdfsBlocksDistribution.add(r.getHDFSBlocksDistribution()); - } - - for (Entry e : tempVals.entrySet()) { - RegionMetricsStorage.setNumericMetric(e.getKey(), e.getValue().longValue()); - } - this.metrics.stores.set(stores); - this.metrics.storefiles.set(storefiles); - this.metrics.memstoreSizeMB.set((int) (memstoreSize / (1024 * 1024))); - this.metrics.mbInMemoryWithoutWAL.set((int) (dataInMemoryWithoutWAL / (1024 * 1024))); - this.metrics.numPutsWithoutWAL.set(numPutsWithoutWAL); - this.metrics.storefileIndexSizeMB.set( - (int) (storefileIndexSize / (1024 * 1024))); - this.metrics.rootIndexSizeKB.set( - (int) (storefileIndexSize / 1024)); - this.metrics.totalStaticIndexSizeKB.set( - (int) (totalStaticIndexSize / 1024)); - this.metrics.totalStaticBloomSizeKB.set( - (int) (totalStaticBloomSize / 1024)); - this.metrics.readRequestsCount.set(readRequestsCount); - this.metrics.writeRequestsCount.set(writeRequestsCount); - this.metrics.checkAndMutateChecksFailed.set(checkAndMutateChecksFailed); - this.metrics.checkAndMutateChecksPassed.set(checkAndMutateChecksPassed); - this.metrics.compactionQueueSize.set(compactSplitThread - .getCompactionQueueSize()); - this.metrics.flushQueueSize.set(cacheFlusher - .getFlushQueueSize()); - - BlockCache blockCache = cacheConfig.getBlockCache(); - if (blockCache != null) { - this.metrics.blockCacheCount.set(blockCache.size()); - this.metrics.blockCacheFree.set(blockCache.getFreeSize()); - this.metrics.blockCacheSize.set(blockCache.getCurrentSize()); - CacheStats cacheStats = blockCache.getStats(); - this.metrics.blockCacheHitCount.set(cacheStats.getHitCount()); - this.metrics.blockCacheMissCount.set(cacheStats.getMissCount()); - this.metrics.blockCacheEvictedCount.set(blockCache.getEvictedCount()); - double ratio = blockCache.getStats().getHitRatio(); - int percent = (int) (ratio * 100); - this.metrics.blockCacheHitRatio.set(percent); - ratio = blockCache.getStats().getHitCachingRatio(); - percent = (int) (ratio * 100); - this.metrics.blockCacheHitCachingRatio.set(percent); - // past N period block cache hit / hit caching ratios - cacheStats.rollMetricsPeriod(); - ratio = cacheStats.getHitRatioPastNPeriods(); - percent = (int) (ratio * 100); - this.metrics.blockCacheHitRatioPastNPeriods.set(percent); - ratio = cacheStats.getHitCachingRatioPastNPeriods(); - percent = (int) (ratio * 100); - this.metrics.blockCacheHitCachingRatioPastNPeriods.set(percent); - } - float localityIndex = hdfsBlocksDistribution.getBlockLocalityIndex( - getServerName().getHostname()); - int percent = (int) (localityIndex * 100); - this.metrics.hdfsBlocksLocalityIndex.set(percent); - - } - /** - * @return Region server metrics instance. - */ public RegionServerMetrics getMetrics() { return this.metrics; } @@ -1826,9 +1670,6 @@ public class HRegionServer implements ClientProtocol, // java.util.HashSet's toString() method to print the coprocessor names. LOG.fatal("RegionServer abort: loaded coprocessors are: " + CoprocessorHost.getLoadedCoprocessors()); - if (this.metrics != null) { - LOG.info("Dump of metrics: " + this.metrics); - } // Do our best to report our abort to the master, but this may not work try { if (cause != null) { @@ -2169,7 +2010,7 @@ public class HRegionServer implements ClientProtocol, } /** - * For tests and web ui. + * For tests, web ui and metrics. * This method will only work if HRegionServer is in the same JVM as client; * HRegion cannot be serialized to cross an rpc. * @see #getOnlineRegions() @@ -2204,7 +2045,7 @@ public class HRegionServer implements ClientProtocol, } /** @return the request count */ - public AtomicInteger getRequestCount() { + public Counter getRequestCount() { return this.requestCount; } @@ -2483,16 +2324,6 @@ public class HRegionServer implements ClientProtocol, } /** - * Register bean with platform management server - */ - void registerMBean() { - MXBeanImpl mxBeanInfo = MXBeanImpl.init(this); - mxBean = MBeanUtil.registerMBean("RegionServer", "RegionServer", - mxBeanInfo); - LOG.info("Registered RegionServer MXBean"); - } - - /** * Instantiated as a row lock lease. If the lease times out, the row lock is * released */ @@ -2670,14 +2501,7 @@ public class HRegionServer implements ClientProtocol, if (destination != null){ addToMovedRegions(encodedRegionName, destination); } - - //Clear all of the dynamic metrics as they are now probably useless. - //This is a clear because dynamic metrics could include metrics per cf and - //per hfile. Figuring out which cfs, hfiles, and regions are still relevant to - //this region server would be an onerous task. Instead just clear everything - //and on the next tick of the metrics everything that is still relevant will be - //re-added. - this.dynamicMetrics.clear(); + return toReturn != null; } @@ -2870,8 +2694,9 @@ public class HRegionServer implements ClientProtocol, @Override public GetResponse get(final RpcController controller, final GetRequest request) throws ServiceException { + long before = EnvironmentEdgeManager.currentTimeMillis(); try { - requestCount.incrementAndGet(); + requestCount.increment(); HRegion region = getRegion(request.getRegion()); GetResponse.Builder builder = GetResponse.newBuilder(); ClientProtos.Get get = request.getGet(); @@ -2911,6 +2736,8 @@ public class HRegionServer implements ClientProtocol, return builder.build(); } catch (IOException ie) { throw new ServiceException(ie); + } finally { + metrics.updateGet(EnvironmentEdgeManager.currentTimeMillis() - before); } } @@ -2925,7 +2752,7 @@ public class HRegionServer implements ClientProtocol, public MutateResponse mutate(final RpcController controller, final MutateRequest request) throws ServiceException { try { - requestCount.incrementAndGet(); + requestCount.increment(); HRegion region = getRegion(request.getRegion()); MutateResponse.Builder builder = MutateResponse.newBuilder(); Mutate mutate = request.getMutate(); @@ -3058,7 +2885,7 @@ public class HRegionServer implements ClientProtocol, } throw e; } - requestCount.incrementAndGet(); + requestCount.increment(); try { int ttl = 0; @@ -3134,7 +2961,7 @@ public class HRegionServer implements ClientProtocol, for (int i = 0; i < rows && currentScanResultSize < maxResultSize; i++) { // Collect values to be returned here - boolean moreRows = scanner.next(values, SchemaMetrics.METRIC_NEXTSIZE); + boolean moreRows = scanner.next(values); if (!values.isEmpty()) { for (KeyValue kv : values) { currentScanResultSize += kv.heapSize(); @@ -3227,7 +3054,7 @@ public class HRegionServer implements ClientProtocol, throw new DoNotRetryIOException( "lockRow supports only one row now, not " + request.getRowCount() + " rows"); } - requestCount.incrementAndGet(); + requestCount.increment(); HRegion region = getRegion(request.getRegion()); byte[] row = request.getRow(0).toByteArray(); try { @@ -3258,7 +3085,7 @@ public class HRegionServer implements ClientProtocol, public UnlockRowResponse unlockRow(final RpcController controller, final UnlockRowRequest request) throws ServiceException { try { - requestCount.incrementAndGet(); + requestCount.increment(); HRegion region = getRegion(request.getRegion()); if (!request.hasLockId()) { throw new DoNotRetryIOException( @@ -3293,7 +3120,7 @@ public class HRegionServer implements ClientProtocol, public BulkLoadHFileResponse bulkLoadHFile(final RpcController controller, final BulkLoadHFileRequest request) throws ServiceException { try { - requestCount.incrementAndGet(); + requestCount.increment(); HRegion region = getRegion(request.getRegion()); List> familyPaths = new ArrayList>(); for (FamilyPath familyPath: request.getFamilyPathList()) { @@ -3340,7 +3167,7 @@ public class HRegionServer implements ClientProtocol, public ExecCoprocessorResponse execCoprocessor(final RpcController controller, final ExecCoprocessorRequest request) throws ServiceException { try { - requestCount.incrementAndGet(); + requestCount.increment(); HRegion region = getRegion(request.getRegion()); ExecCoprocessorResponse.Builder builder = ExecCoprocessorResponse.newBuilder(); @@ -3358,7 +3185,7 @@ public class HRegionServer implements ClientProtocol, public CoprocessorServiceResponse execService(final RpcController controller, final CoprocessorServiceRequest request) throws ServiceException { try { - requestCount.incrementAndGet(); + requestCount.increment(); HRegion region = getRegion(request.getRegion()); // ignore the passed in controller (from the serialized call) ServerRpcController execController = new ServerRpcController(); @@ -3407,7 +3234,7 @@ public class HRegionServer implements ClientProtocol, ActionResult.Builder resultBuilder = null; List mutates = new ArrayList(); for (ClientProtos.MultiAction actionUnion : request.getActionList()) { - requestCount.incrementAndGet(); + requestCount.increment(); try { Object result = null; if (actionUnion.hasGet()) { @@ -3490,7 +3317,7 @@ public class HRegionServer implements ClientProtocol, final GetRegionInfoRequest request) throws ServiceException { try { checkOpen(); - requestCount.incrementAndGet(); + requestCount.increment(); HRegion region = getRegion(request.getRegion()); HRegionInfo info = region.getRegionInfo(); GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder(); @@ -3510,7 +3337,7 @@ public class HRegionServer implements ClientProtocol, final GetStoreFileRequest request) throws ServiceException { try { HRegion region = getRegion(request.getRegion()); - requestCount.incrementAndGet(); + requestCount.increment(); Set columnFamilies = null; if (request.getFamilyCount() == 0) { columnFamilies = region.getStores().keySet(); @@ -3537,7 +3364,7 @@ public class HRegionServer implements ClientProtocol, final GetOnlineRegionRequest request) throws ServiceException { try { checkOpen(); - requestCount.incrementAndGet(); + requestCount.increment(); List list = new ArrayList(onlineRegions.size()); for (HRegion region: this.onlineRegions.values()) { list.add(region.getRegionInfo()); @@ -3572,7 +3399,7 @@ public class HRegionServer implements ClientProtocol, } catch (IOException ie) { throw new ServiceException(ie); } - requestCount.incrementAndGet(); + requestCount.increment(); OpenRegionResponse.Builder builder = OpenRegionResponse.newBuilder(); Map htds = new HashMap( request.getRegionList().size()); @@ -3660,7 +3487,7 @@ public class HRegionServer implements ClientProtocol, try { checkOpen(); - requestCount.incrementAndGet(); + requestCount.increment(); HRegion region = getRegion(request.getRegion()); CloseRegionResponse.Builder builder = CloseRegionResponse.newBuilder(); @@ -3691,7 +3518,7 @@ public class HRegionServer implements ClientProtocol, final FlushRegionRequest request) throws ServiceException { try { checkOpen(); - requestCount.incrementAndGet(); + requestCount.increment(); HRegion region = getRegion(request.getRegion()); LOG.info("Flushing " + region.getRegionNameAsString()); boolean shouldFlush = true; @@ -3722,7 +3549,7 @@ public class HRegionServer implements ClientProtocol, final SplitRegionRequest request) throws ServiceException { try { checkOpen(); - requestCount.incrementAndGet(); + requestCount.increment(); HRegion region = getRegion(request.getRegion()); LOG.info("Splitting " + region.getRegionNameAsString()); region.flushcache(); @@ -3751,7 +3578,7 @@ public class HRegionServer implements ClientProtocol, final CompactRegionRequest request) throws ServiceException { try { checkOpen(); - requestCount.incrementAndGet(); + requestCount.increment(); HRegion region = getRegion(request.getRegion()); LOG.info("Compacting " + region.getRegionNameAsString()); boolean major = false; @@ -3786,7 +3613,7 @@ public class HRegionServer implements ClientProtocol, try { if (replicationSinkHandler != null) { checkOpen(); - requestCount.incrementAndGet(); + requestCount.increment(); HLog.Entry[] entries = ProtobufUtil.toHLogEntries(request.getEntryList()); if (entries != null && entries.length > 0) { replicationSinkHandler.replicateLogEntries(entries); @@ -3809,7 +3636,7 @@ public class HRegionServer implements ClientProtocol, public RollWALWriterResponse rollWALWriter(final RpcController controller, final RollWALWriterRequest request) throws ServiceException { try { - requestCount.incrementAndGet(); + requestCount.increment(); HLog wal = this.getWAL(); byte[][] regionsToFlush = wal.rollWriter(true); RollWALWriterResponse.Builder builder = RollWALWriterResponse.newBuilder(); @@ -3834,7 +3661,7 @@ public class HRegionServer implements ClientProtocol, @Override public StopServerResponse stopServer(final RpcController controller, final StopServerRequest request) throws ServiceException { - requestCount.incrementAndGet(); + requestCount.increment(); String reason = request.getReason(); stop(reason); return StopServerResponse.newBuilder().build(); @@ -3851,7 +3678,7 @@ public class HRegionServer implements ClientProtocol, public GetServerInfoResponse getServerInfo(final RpcController controller, final GetServerInfoRequest request) throws ServiceException { ServerName serverName = getServerName(); - requestCount.incrementAndGet(); + requestCount.increment(); return ResponseConverter.buildGetServerInfoResponse(serverName, webuiport); } @@ -3891,6 +3718,7 @@ public class HRegionServer implements ClientProtocol, */ protected Result append(final HRegion region, final Mutate mutate) throws IOException { + long before = EnvironmentEdgeManager.currentTimeMillis(); Append append = ProtobufUtil.toAppend(mutate); Result r = null; if (region.getCoprocessorHost() != null) { @@ -3903,6 +3731,7 @@ public class HRegionServer implements ClientProtocol, region.getCoprocessorHost().postAppend(append, r); } } + metrics.updateAppend(EnvironmentEdgeManager.currentTimeMillis() - before); return r; } @@ -3916,6 +3745,7 @@ public class HRegionServer implements ClientProtocol, */ protected Result increment(final HRegion region, final Mutate mutate) throws IOException { + long before = EnvironmentEdgeManager.currentTimeMillis(); Increment increment = ProtobufUtil.toIncrement(mutate); Result r = null; if (region.getCoprocessorHost() != null) { @@ -3928,6 +3758,7 @@ public class HRegionServer implements ClientProtocol, r = region.getCoprocessorHost().postIncrement(increment, r); } } + metrics.updateIncrement(EnvironmentEdgeManager.currentTimeMillis() - before); return r; } @@ -3942,7 +3773,8 @@ public class HRegionServer implements ClientProtocol, final HRegion region, final List mutates) { @SuppressWarnings("unchecked") Pair[] mutationsWithLocks = new Pair[mutates.size()]; - + long before = EnvironmentEdgeManager.currentTimeMillis(); + boolean batchContainsPuts = false, batchContainsDelete = false; try { ActionResult.Builder resultBuilder = ActionResult.newBuilder(); NameBytesPair value = ProtobufUtil.toParameter(new Result()); @@ -3954,15 +3786,18 @@ public class HRegionServer implements ClientProtocol, Mutation mutation = null; if (m.getMutateType() == MutateType.PUT) { mutation = ProtobufUtil.toPut(m); + batchContainsPuts = true; } else { mutation = ProtobufUtil.toDelete(m); + batchContainsDelete = true; } Integer lock = getLockFromId(mutation.getLockId()); mutationsWithLocks[i++] = new Pair(mutation, lock); builder.addResult(result); } - requestCount.addAndGet(mutates.size()); + + requestCount.add(mutates.size()); if (!region.getRegionInfo().isMetaTable()) { cacheFlusher.reclaimMemStoreMemory(); } @@ -3998,6 +3833,13 @@ public class HRegionServer implements ClientProtocol, builder.setResult(i, result); } } + long after = EnvironmentEdgeManager.currentTimeMillis(); + if (batchContainsPuts) { + metrics.updatePut(after - before); + } + if (batchContainsDelete) { + metrics.updateDelete(after - before); + } } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerMetricsWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerMetricsWrapperImpl.java new file mode 100644 index 0000000..afd92d3 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServerMetricsWrapperImpl.java @@ -0,0 +1,337 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.CompatibilitySingletonFactory; +import org.apache.hadoop.hbase.HDFSBlocksDistribution; +import org.apache.hadoop.hbase.io.hfile.BlockCache; +import org.apache.hadoop.hbase.io.hfile.CacheStats; +import org.apache.hadoop.hbase.regionserver.metrics.HRegionServerMetricsWrapper; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.metrics2.MetricsExecutor; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * Impl for exposing HRegionServer Information through JMX + */ +@InterfaceAudience.Private class HRegionServerMetricsWrapperImpl + implements HRegionServerMetricsWrapper { + + public static final Log LOG = LogFactory.getLog(HRegionServerMetricsWrapperImpl.class); + + public static final int PERIOD = 15; + + private final HRegionServer regionServer; + private final BlockCache blockCache; + + private volatile long numStores = 0; + private volatile long numStoreFiles = 0; + private volatile long memstoreSize = 0; + private volatile long storeFileSize = 0; + private volatile double requestsPerSecond = 0.0; + private volatile long readRequestsCount = 0; + private volatile long writeRequestsCount = 0; + private volatile long checkAndMutateChecksFailed = 0; + private volatile long checkAndMutateChecksPassed = 0; + private volatile long storefileIndexSize = 0; + private volatile long totalStaticIndexSize = 0; + private volatile long totalStaticBloomSize = 0; + private volatile long numPutsWithoutWAL = 0; + private volatile long dataInMemoryWithoutWAL = 0; + private volatile int percentFileLocal = 0; + + private CacheStats cacheStats; + private ScheduledExecutorService executor; + private Runnable runnable; + + public HRegionServerMetricsWrapperImpl(final HRegionServer regionServer) { + this.regionServer = regionServer; + this.blockCache = this.regionServer.cacheConfig.getBlockCache(); + this.cacheStats = blockCache.getStats(); + this.executor = CompatibilitySingletonFactory.getInstance(MetricsExecutor.class).getExecutor(); + this.runnable = new RegionServerMetricsWrapperRunnable(); + this.executor.scheduleWithFixedDelay(this.runnable, PERIOD, PERIOD, TimeUnit.SECONDS); + } + + @Override + public String getClusterId() { + return regionServer.getClusterId(); + } + + @Override + public long getRegionServerStartTime() { + return regionServer.getRegionServerStartTime(); + } + + @Override + public String getZookeeperQuorum() { + return regionServer.getZooKeeperWatcher().getQuorum(); + } + + @Override + public String getCoprocessors() { + return StringUtils.join(regionServer.getCoprocessors(), ", "); + } + + @Override + public String getServerName() { + return regionServer.getServerName().getServerName(); + } + + @Override + public long getNumOnlineRegions() { + return regionServer.getOnlineRegionsLocalContext().size(); + } + + @Override + public long getTotalRequestCount() { + return regionServer.requestCount.get(); + } + + @Override + public int getCompactionQueueSize() { + return this.regionServer.compactSplitThread.getCompactionQueueSize(); + } + + @Override + public int getFlushQueueSize() { + return this.regionServer.cacheFlusher.getFlushQueueSize(); + } + + @Override + public long getBlockCacheCount() { + return this.blockCache.size(); + } + + @Override + public long getBlockCacheSize() { + return this.blockCache.getCurrentSize(); + } + + @Override + public long getBlockCacheFreeSize() { + return this.blockCache.getFreeSize(); + } + + @Override + public long getBlockCacheHitCount() { + return this.cacheStats.getHitCount(); + } + + @Override + public long getBlockCacheMissCount() { + return this.cacheStats.getMissCount(); + } + + @Override + public long getBlockCacheEvictedCount() { + return this.cacheStats.getEvictedCount(); + } + + @Override + public int getBlockCacheHitPercent() { + return (int) (this.cacheStats.getHitRatio() * 100); + } + + @Override + public int getBlockCacheHitCachingPercent() { + return (int) (blockCache.getStats().getHitCachingRatio() * 100); + } + + @Override public void forceRecompute() { + this.runnable.run(); + } + + @Override + public long getNumStores() { + return numStores; + } + + @Override + public long getNumStoreFiles() { + return numStoreFiles; + } + + @Override + public long getMemstoreSize() { + return memstoreSize; + } + + @Override + public long getStoreFileSize() { + return storeFileSize; + } + + @Override public double getRequestsPerSecond() { + return requestsPerSecond; + } + + @Override + public long getReadRequestsCount() { + return readRequestsCount; + } + + @Override + public long getWriteRequestsCount() { + return writeRequestsCount; + } + + @Override + public long getCheckAndMutateChecksFailed() { + return checkAndMutateChecksFailed; + } + + @Override + public long getCheckAndMutateChecksPassed() { + return checkAndMutateChecksPassed; + } + + @Override + public long getStoreFileIndexSize() { + return storefileIndexSize; + } + + @Override + public long getTotalStaticIndexSize() { + return totalStaticIndexSize; + } + + @Override + public long getTotalStaticBloomSize() { + return totalStaticBloomSize; + } + + @Override + public long getNumPutsWithoutWAL() { + return numPutsWithoutWAL; + } + + @Override + public long getDataInMemoryWithoutWAL() { + return dataInMemoryWithoutWAL; + } + + @Override + public int getPercentFileLocal() { + + return percentFileLocal; + } + + + /** + * This is the runnable that will be executed on the executor ever PERIOD number of seconds + * It will take metrics/numbers from all of the regions and use them to compute point in + * time metrics. + */ + public class RegionServerMetricsWrapperRunnable implements Runnable { + + private long lastRan = 0; + private long lastRequestCount = 0; + + @Override + synchronized public void run() { + + cacheStats = blockCache.getStats(); + + HDFSBlocksDistribution hdfsBlocksDistribution = + new HDFSBlocksDistribution(); + + long tempNumStores = 0; + long tempNumStoreFiles = 0; + long tempMemstoreSize = 0; + long tempStoreFileSize = 0; + long tempReadRequestsCount = 0; + long tempWriteRequestsCount = 0; + long tempCheckAndMutateChecksFailed = 0; + long tempCheckAndMutateChecksPassed = 0; + long tempStorefileIndexSize = 0; + long tempTotalStaticIndexSize = 0; + long tempTotalStaticBloomSize = 0; + long tempNumPutsWithoutWAL = 0; + long tempDataInMemoryWithoutWAL = 0; + int tempPercentFileLocal = 0; + + + for (HRegion r : regionServer.getOnlineRegionsLocalContext()) { + tempNumPutsWithoutWAL += r.numPutsWithoutWAL.get(); + tempDataInMemoryWithoutWAL += r.dataInMemoryWithoutWAL.get(); + tempReadRequestsCount += r.readRequestsCount.get(); + tempWriteRequestsCount += r.writeRequestsCount.get(); + tempCheckAndMutateChecksFailed += r.checkAndMutateChecksFailed.get(); + tempCheckAndMutateChecksPassed += r.checkAndMutateChecksPassed.get(); + tempNumStores += r.stores.size(); + for (Store store : r.stores.values()) { + tempNumStoreFiles += store.getStorefilesCount(); + tempMemstoreSize += store.getMemStoreSize(); + tempStoreFileSize += store.getStorefilesSize(); + tempStorefileIndexSize += store.getStorefilesIndexSize(); + tempTotalStaticBloomSize += store.getTotalStaticBloomSize(); + tempTotalStaticIndexSize += store.getTotalStaticIndexSize(); + } + + hdfsBlocksDistribution.add(r.getHDFSBlocksDistribution()); + } + + float localityIndex = hdfsBlocksDistribution.getBlockLocalityIndex( + regionServer.getServerName().getHostname()); + tempPercentFileLocal = (int) (localityIndex * 100); + + + //Compute the number of requests per second + long currentTime = EnvironmentEdgeManager.currentTimeMillis(); + + // assume that it took PERIOD seconds to start the executor. + // this is a guess but it's a pretty good on. + if (lastRan == 0) { + lastRan = currentTime - (PERIOD*1000); + } + + + //If we've time traveled keep the last requests per second. + if ((currentTime - lastRan) > 10) { + long currentRequestCount = getTotalRequestCount(); + requestsPerSecond = (currentRequestCount - lastRequestCount) / ((currentTime - lastRan) / 1000.0); + lastRequestCount = currentRequestCount; + } + lastRan = currentTime; + + //Copy over computed values so that no thread sees half computed values. + numStores = tempNumStores; + numStoreFiles = tempNumStoreFiles; + memstoreSize = tempMemstoreSize; + storeFileSize = tempStoreFileSize; + readRequestsCount = tempReadRequestsCount; + writeRequestsCount = tempWriteRequestsCount; + checkAndMutateChecksFailed = tempCheckAndMutateChecksFailed; + checkAndMutateChecksPassed = tempCheckAndMutateChecksPassed; + storefileIndexSize = tempStorefileIndexSize; + totalStaticIndexSize = tempTotalStaticIndexSize; + totalStaticBloomSize = tempTotalStaticBloomSize; + numPutsWithoutWAL = tempNumPutsWithoutWAL; + dataInMemoryWithoutWAL = tempDataInMemoryWithoutWAL; + percentFileLocal = tempPercentFileLocal; + } + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 32d83f9..85b693d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -66,8 +66,6 @@ import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.regionserver.compactions.CompactSelection; import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.ClassSize; @@ -106,7 +104,7 @@ import com.google.common.collect.Lists; * not be called directly but by an HRegion manager. */ @InterfaceAudience.Private -public class HStore extends SchemaConfigured implements Store { +public class HStore implements Store { static final Log LOG = LogFactory.getLog(HStore.class); protected final MemStore memstore; @@ -174,9 +172,7 @@ public class HStore extends SchemaConfigured implements Store { protected HStore(Path basedir, HRegion region, HColumnDescriptor family, FileSystem fs, Configuration confParam) throws IOException { - super(new CompoundConfiguration().add(confParam).add( - family.getValues()), region.getRegionInfo().getTableNameAsString(), - Bytes.toString(family.getName())); + HRegionInfo info = region.getRegionInfo(); this.fs = fs; // Assemble the store's home directory. @@ -260,6 +256,15 @@ public class HStore extends SchemaConfigured implements Store { return ttl; } + public String getColumnFamilyName() { + return this.family.getNameAsString(); + } + + @Override + public String getTableName() { + return this.region.getTableDesc().getNameAsString(); + } + /** * Create this store's homedir * @param fs @@ -414,7 +419,6 @@ public class HStore extends SchemaConfigured implements Store { public StoreFile call() throws IOException { StoreFile storeFile = new StoreFile(fs, p, conf, cacheConf, family.getBloomFilterType(), dataBlockEncoder); - passSchemaMetricsTo(storeFile); storeFile.createReader(); return storeFile; } @@ -573,7 +577,6 @@ public class HStore extends SchemaConfigured implements Store { StoreFile sf = new StoreFile(fs, dstPath, this.conf, this.cacheConf, this.family.getBloomFilterType(), this.dataBlockEncoder); - passSchemaMetricsTo(sf); StoreFile.Reader r = sf.createReader(); this.storeSize += r.length(); @@ -817,19 +820,11 @@ public class HStore extends SchemaConfigured implements Store { status.setStatus("Flushing " + this + ": reopening flushed file"); StoreFile sf = new StoreFile(this.fs, dstPath, this.conf, this.cacheConf, this.family.getBloomFilterType(), this.dataBlockEncoder); - passSchemaMetricsTo(sf); StoreFile.Reader r = sf.createReader(); this.storeSize += r.length(); this.totalUncompressedBytes += r.getTotalUncompressedBytes(); - // This increments the metrics associated with total flushed bytes for this - // family. The overall flush count is stored in the static metrics and - // retrieved from HRegion.recentFlushes, which is set within - // HRegion.internalFlushcache, which indirectly calls this to actually do - // the flushing through the StoreFlusherImpl class - getSchemaMetrics().updatePersistentStoreMetric( - SchemaMetrics.StoreMetricType.FLUSH_SIZE, flushedSize.longValue()); if (LOG.isInfoEnabled()) { LOG.info("Added " + sf + ", entries=" + r.getEntries() + ", sequenceid=" + logCacheFlushId + @@ -875,11 +870,6 @@ public class HStore extends SchemaConfigured implements Store { .withBytesPerChecksum(bytesPerChecksum) .withCompression(compression) .build(); - // The store file writer's path does not include the CF name, so we need - // to configure the HFile writer directly. - SchemaConfigured sc = (SchemaConfigured) w.writer; - SchemaConfigured.resetSchemaMetricsConf(sc); - passSchemaMetricsTo(sc); return w; } @@ -1409,8 +1399,8 @@ public class HStore extends SchemaConfigured implements Store { (forcemajor || isMajorCompaction(compactSelection.getFilesToCompact())) && (compactSelection.getFilesToCompact().size() < this.maxFilesToCompact ); - LOG.debug(this.getHRegionInfo().getEncodedName() + " - " + - this.getColumnFamilyName() + ": Initiating " + + LOG.debug(this.getHRegionInfo().getEncodedName() + " - " + + this.getColumnFamilyName() + ": Initiating " + (majorcompaction ? "major" : "minor") + "compaction"); if (!majorcompaction && @@ -1523,7 +1513,6 @@ public class HStore extends SchemaConfigured implements Store { storeFile = new StoreFile(this.fs, path, this.conf, this.cacheConf, this.family.getBloomFilterType(), NoOpDataBlockEncoder.INSTANCE); - passSchemaMetricsTo(storeFile); storeFile.createReader(); } catch (IOException e) { LOG.error("Failed to open store file : " + path @@ -1575,7 +1564,6 @@ public class HStore extends SchemaConfigured implements Store { } result = new StoreFile(this.fs, destPath, this.conf, this.cacheConf, this.family.getBloomFilterType(), this.dataBlockEncoder); - passSchemaMetricsTo(result); result.createReader(); } try { @@ -1936,7 +1924,7 @@ public class HStore extends SchemaConfigured implements Store { @Override public String toString() { - return getColumnFamilyName(); + return this.getColumnFamilyName(); } @Override @@ -2125,9 +2113,8 @@ public class HStore extends SchemaConfigured implements Store { } public static final long FIXED_OVERHEAD = - ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE + - + (17 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG) - + (5 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN); + ClassSize.align((19 * ClassSize.REFERENCE) + (6 * Bytes.SIZEOF_LONG) + + (5 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN); public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD + ClassSize.OBJECT + ClassSize.REENTRANT_LOCK diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MXBean.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MXBean.java deleted file mode 100644 index b0a92c5..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MXBean.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.regionserver; - -import org.apache.hadoop.classification.InterfaceStability.Evolving; - -/** - * This is the JMX management interface for HBase Region Server information - */ -@Evolving -public interface MXBean { - - /** - * Return RegionServer's ServerName - * @return ServerName - */ - public String getServerName(); - - /** - * Get loaded co-processors - * @return Loaded Co-processors - */ - public String[] getCoprocessors(); - - /** - * Get Zookeeper Quorum - * @return Comma-separated list of Zookeeper Quorum servers - */ - public String getZookeeperQuorum(); -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MXBeanImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MXBeanImpl.java deleted file mode 100644 index 78f3b6f..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MXBeanImpl.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -/** - * Impl for exposing Region Server Information through JMX - */ -public class MXBeanImpl implements MXBean { - - private final HRegionServer regionServer; - - private static MXBeanImpl instance = null; - public synchronized static MXBeanImpl init(final HRegionServer rs){ - if (instance == null) { - instance = new MXBeanImpl(rs); - } - return instance; - } - - protected MXBeanImpl(final HRegionServer rs) { - this.regionServer = rs; - } - - @Override - public String[] getCoprocessors() { - return regionServer.getCoprocessors(); - } - - @Override - public String getZookeeperQuorum() { - return regionServer.getZooKeeper().getQuorum(); - } - - @Override - public String getServerName() { - return regionServer.getServerName().getServerName(); - } - -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java index 3a6251b..933f783 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java @@ -409,7 +409,6 @@ class MemStoreFlusher extends HasThread implements FlushRequester { server.compactSplitThread.requestCompaction(region, getName()); } - server.getMetrics().addFlush(region.getRecentFlushInfo()); } catch (DroppedSnapshotException ex) { // Cache flush can fail in a few places. If it fails in a critical // section, we get a DroppedSnapshotException and a replay of hlog diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java index f945ffd..0b5a316 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitRequest.java @@ -66,7 +66,6 @@ class SplitRequest implements Runnable { if (!st.prepare()) return; try { st.execute(this.server, this.server); - this.server.getMetrics().incrementSplitSuccessCount(System.currentTimeMillis() - startTime); } catch (Exception e) { if (this.server.isStopping() || this.server.isStopped()) { LOG.info( @@ -81,7 +80,6 @@ class SplitRequest implements Runnable { if (st.rollback(this.server, this.server)) { LOG.info("Successful rollback of failed split of " + parent.getRegionNameAsString()); - this.server.getMetrics().incrementSplitFailureCount(); } else { this.server.abort("Abort; we got an error after point-of-no-return"); } @@ -102,7 +100,6 @@ class SplitRequest implements Runnable { } catch (IOException ex) { LOG.error("Split failed " + this, RemoteExceptionHandler .checkIOException(ex)); - this.server.getMetrics().incrementSplitFailureCount(); server.checkFileSystem(); } finally { if (this.parent.getCoprocessorHost() != null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java index d391a16..1841eeb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.SchemaAware; import com.google.common.collect.ImmutableList; @@ -42,7 +41,7 @@ import com.google.common.collect.ImmutableList; */ @InterfaceAudience.Private @InterfaceStability.Evolving -public interface Store extends SchemaAware, HeapSize { +public interface Store extends HeapSize { /* The default priority for user-specified compaction requests. * The user gets top priority unless we have blocking compactions. (Pri <= 0) @@ -287,4 +286,8 @@ public interface Store extends SchemaAware, HeapSize { * @return the parent region hosting this store */ public HRegion getHRegion(); + + public String getColumnFamilyName(); + + public String getTableName(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java index af1225a..b1418fc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java @@ -58,8 +58,6 @@ import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.HFileWriterV1; import org.apache.hadoop.hbase.io.hfile.HFileWriterV2; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured; import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder; import org.apache.hadoop.hbase.util.ChecksumType; @@ -80,7 +78,7 @@ import com.google.common.collect.Ordering; /** * A Store data file. Stores usually have one or more of these files. They * are produced by flushing the memstore to disk. To - * create, instantiate a writer using {@link StoreFile#WriterBuilder} + * create, instantiate a writer using {@link StoreFile.WriterBuilder} * and append data. Be sure to add any metadata before calling close on the * Writer (Use the appendMetadata convenience methods). On close, a StoreFile * is sitting in the Filesystem. To refer to it, create a StoreFile instance @@ -91,7 +89,7 @@ import com.google.common.collect.Ordering; * writer and a reader is that we write once but read a lot more. */ @InterfaceAudience.LimitedPrivate("Coprocessor") -public class StoreFile extends SchemaConfigured { +public class StoreFile { static final Log LOG = LogFactory.getLog(StoreFile.class.getName()); public static enum BloomType { @@ -277,7 +275,6 @@ public class StoreFile extends SchemaConfigured { this.modificationTimeStamp = 0; } - SchemaMetrics.configureGlobally(conf); } /** @@ -545,11 +542,6 @@ public class StoreFile extends SchemaConfigured { dataBlockEncoder.getEncodingInCache()); } - if (isSchemaConfigured()) { - SchemaConfigured.resetSchemaMetricsConf(reader); - passSchemaMetricsTo(reader); - } - computeHDFSBlockDistribution(); // Load up indices and fileinfo. This also loads Bloom filter type. @@ -1287,7 +1279,7 @@ public class StoreFile extends SchemaConfigured { /** * Reader for a StoreFile. */ - public static class Reader extends SchemaConfigured { + public static class Reader { static final Log LOG = LogFactory.getLog(Reader.class.getName()); protected BloomFilter generalBloomFilter = null; @@ -1301,7 +1293,6 @@ public class StoreFile extends SchemaConfigured { public Reader(FileSystem fs, Path path, CacheConfig cacheConf, DataBlockEncoding preferredEncodingInCache) throws IOException { - super(path); reader = HFile.createReaderWithEncoding(fs, path, cacheConf, preferredEncodingInCache); bloomFilterType = BloomType.NONE; @@ -1310,7 +1301,6 @@ public class StoreFile extends SchemaConfigured { public Reader(FileSystem fs, Path path, HFileLink hfileLink, long size, CacheConfig cacheConf, DataBlockEncoding preferredEncodingInCache, boolean closeIStream) throws IOException { - super(path); FSDataInputStream in = hfileLink.open(fs); FSDataInputStream inNoChecksum = in; @@ -1584,7 +1574,6 @@ public class StoreFile extends SchemaConfigured { && bloomFilter.contains(key, 0, key.length, bloom); } - getSchemaMetrics().updateBloomMetrics(exists); return exists; } } catch (IOException e) { @@ -1728,10 +1717,6 @@ public class StoreFile extends SchemaConfigured { return reader.indexSize(); } - public String getColumnFamilyName() { - return reader.getColumnFamilyName(); - } - public BloomType getBloomFilterType() { return this.bloomFilterType; } @@ -1774,11 +1759,6 @@ public class StoreFile extends SchemaConfigured { public long getMaxTimestamp() { return timeRangeTracker.maximumTimestamp; } - - @Override - public void schemaConfigurationChanged() { - passSchemaMetricsTo((SchemaConfigured) reader); - } } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index b595c06..4ccdbcc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -33,8 +33,6 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.regionserver.HStore.ScanInfo; -import org.apache.hadoop.hbase.regionserver.metrics.RegionMetricsStorage; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -110,7 +108,6 @@ public class StoreScanner extends NonLazyKeyValueScanner throws IOException { this(store, scan.getCacheBlocks(), scan, columns, scanInfo.getTtl(), scanInfo.getMinVersions()); - initializeMetricNames(); if (columns != null && scan.isRaw()) { throw new DoNotRetryIOException( "Cannot specify any column for a raw scan"); @@ -163,7 +160,6 @@ public class StoreScanner extends NonLazyKeyValueScanner long smallestReadPoint, long earliestPutTs) throws IOException { this(store, false, scan, null, scanInfo.getTtl(), scanInfo.getMinVersions()); - initializeMetricNames(); matcher = new ScanQueryMatcher(scan, scanInfo, null, scanType, smallestReadPoint, earliestPutTs, oldestUnexpiredTS); @@ -194,7 +190,6 @@ public class StoreScanner extends NonLazyKeyValueScanner throws IOException { this(null, scan.getCacheBlocks(), scan, columns, scanInfo.getTtl(), scanInfo.getMinVersions()); - this.initializeMetricNames(); this.matcher = new ScanQueryMatcher(scan, scanInfo, columns, scanType, Long.MAX_VALUE, earliestPutTs, oldestUnexpiredTS); @@ -206,23 +201,6 @@ public class StoreScanner extends NonLazyKeyValueScanner } /** - * Method used internally to initialize metric names throughout the - * constructors. - * - * To be called after the store variable has been initialized! - */ - private void initializeMetricNames() { - String tableName = SchemaMetrics.UNKNOWN; - String family = SchemaMetrics.UNKNOWN; - if (store != null) { - tableName = store.getTableName(); - family = Bytes.toString(store.getFamily().getName()); - } - this.metricNamePrefix = - SchemaMetrics.generateSchemaMetricsPrefix(tableName, family); - } - - /** * Get a filtered list of scanners. Assumes we are not in a compaction. * @return list of scanners to seek */ @@ -458,8 +436,7 @@ public class StoreScanner extends NonLazyKeyValueScanner } } finally { if (cumulativeMetric > 0 && metric != null) { - RegionMetricsStorage.incrNumericMetric(this.metricNamePrefix + metric, - cumulativeMetric); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java index bee9668..80e4d5e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java @@ -253,7 +253,6 @@ public class CompactionRequest implements Comparable, LOG.info(((completed) ? "completed" : "aborted") + " compaction: " + this + "; duration=" + StringUtils.formatTimeDiff(now, start)); if (completed) { - server.getMetrics().addCompaction(now - start, this.totalSize); // degenerate case: blocked regions require recursive enqueues if (s.getCompactPriority() <= 0) { server.compactSplitThread diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java deleted file mode 100644 index bff18c3..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/OperationMetrics.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hbase.regionserver.metrics; - -import java.util.Set; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.client.Append; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Increment; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.util.Bytes; - -/** - * This class provides a simplified interface to expose time varying metrics - * about GET/DELETE/PUT/ICV operations on a region and on Column Families. All - * metrics are stored in {@link RegionMetricsStorage} and exposed to hadoop - * metrics through {@link RegionServerDynamicMetrics}. - */ -@InterfaceAudience.Private -public class OperationMetrics { - - private static final String DELETE_KEY = "delete_"; - private static final String PUT_KEY = "put_"; - private static final String GET_KEY = "get_"; - private static final String ICV_KEY = "incrementColumnValue_"; - private static final String INCREMENT_KEY = "increment_"; - private static final String MULTIPUT_KEY = "multiput_"; - private static final String MULTIDELETE_KEY = "multidelete_"; - private static final String APPEND_KEY = "append_"; - - /** Conf key controlling whether we should expose metrics.*/ - private static final String CONF_KEY = - "hbase.metrics.exposeOperationTimes"; - - private String tableName = null; - private String regionName = null; - private String regionMetrixPrefix = null; - private Configuration conf = null; - - - /** - * Create a new OperationMetrics - * @param conf The Configuration of the HRegion reporting operations coming in. - * @param regionInfo The region info - */ - public OperationMetrics(Configuration conf, HRegionInfo regionInfo) { - // Configure SchemaMetrics before trying to create a RegionOperationMetrics instance as - // RegionOperationMetrics relies on SchemaMetrics to do naming. - if (conf != null) { - SchemaMetrics.configureGlobally(conf); - - this.conf = conf; - if (regionInfo != null) { - this.tableName = regionInfo.getTableNameAsString(); - this.regionName = regionInfo.getEncodedName(); - } else { - this.tableName = SchemaMetrics.UNKNOWN; - this.regionName = SchemaMetrics.UNKNOWN; - } - this.regionMetrixPrefix = - SchemaMetrics.generateRegionMetricsPrefix(this.tableName, this.regionName); - } - } - - /** - * This is used in creating a testing HRegion where the regionInfo is unknown - * @param conf - */ - public OperationMetrics() { - this(null, null); - } - - - /** - * Update the stats associated with {@link HTable#put(java.util.List)}. - * - * @param columnFamilies Set of CF's this multiput is associated with - * @param value the time - */ - public void updateMultiPutMetrics(Set columnFamilies, long value) { - doUpdateTimeVarying(columnFamilies, MULTIPUT_KEY, value); - } - - /** - * Update the stats associated with {@link HTable#delete(java.util.List)}. - * - * @param columnFamilies Set of CF's this multidelete is associated with - * @param value the time - */ - public void updateMultiDeleteMetrics(Set columnFamilies, long value) { - doUpdateTimeVarying(columnFamilies, MULTIDELETE_KEY, value); - } - - /** - * Update the metrics associated with a {@link Get} - * - * @param columnFamilies - * Set of Column Families in this get. - * @param value - * the time - */ - public void updateGetMetrics(Set columnFamilies, long value) { - doUpdateTimeVarying(columnFamilies, GET_KEY, value); - } - - /** - * Update metrics associated with an {@link Increment} - * @param columnFamilies - * @param value - */ - public void updateIncrementMetrics(Set columnFamilies, long value) { - doUpdateTimeVarying(columnFamilies, INCREMENT_KEY, value); - } - - - /** - * Update the metrics associated with an {@link Append} - * @param columnFamilies - * @param value - */ - public void updateAppendMetrics(Set columnFamilies, long value) { - doUpdateTimeVarying(columnFamilies, APPEND_KEY, value); - } - - - /** - * Update the metrics associated with - * {@link HTable#incrementColumnValue(byte[], byte[], byte[], long)} - * - * @param columnFamily - * The single column family associated with an ICV - * @param value - * the time - */ - public void updateIncrementColumnValueMetrics(byte[] columnFamily, long value) { - String cfMetricPrefix = - SchemaMetrics.generateSchemaMetricsPrefix(this.tableName, Bytes.toString(columnFamily)); - doSafeIncTimeVarying(cfMetricPrefix, ICV_KEY, value); - doSafeIncTimeVarying(this.regionMetrixPrefix, ICV_KEY, value); - } - - /** - * update metrics associated with a {@link Put} - * - * @param columnFamilies - * Set of column families involved. - * @param value - * the time. - */ - public void updatePutMetrics(Set columnFamilies, long value) { - doUpdateTimeVarying(columnFamilies, PUT_KEY, value); - } - - /** - * update metrics associated with a {@link Delete} - * - * @param columnFamilies - * @param value - * the time. - */ - public void updateDeleteMetrics(Set columnFamilies, long value) { - doUpdateTimeVarying(columnFamilies, DELETE_KEY, value); - } - - /** - * This deletes all old metrics this instance has ever created or updated. - */ - public void closeMetrics() { - RegionMetricsStorage.clear(); - } - - /** - * Method to send updates for cf and region metrics. This is the normal method - * used if the naming of stats and CF's are in line with put/delete/multiput. - * - * @param columnFamilies - * the set of column families involved. - * @param key - * the metric name. - * @param value - * the time. - */ - private void doUpdateTimeVarying(Set columnFamilies, String key, long value) { - String cfPrefix = null; - if (columnFamilies != null) { - cfPrefix = SchemaMetrics.generateSchemaMetricsPrefix(tableName, columnFamilies); - } else { - cfPrefix = SchemaMetrics.generateSchemaMetricsPrefix(tableName, SchemaMetrics.UNKNOWN); - } - - doSafeIncTimeVarying(cfPrefix, key, value); - doSafeIncTimeVarying(this.regionMetrixPrefix, key, value); - } - - private void doSafeIncTimeVarying(String prefix, String key, long value) { - if (conf.getBoolean(CONF_KEY, true)) { - if (prefix != null && !prefix.isEmpty() && key != null && !key.isEmpty()) { - String m = prefix + key; - RegionMetricsStorage.incrTimeVaryingMetric(m, value); - } - } - } - -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetrics.java new file mode 100644 index 0000000..83d2f00 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetrics.java @@ -0,0 +1,57 @@ +/* + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.CompatibilityFactory; + +@InterfaceAudience.Private +public class RegionMetrics { + + private RegionMetricSource regionMetricSource; + + public RegionMetrics(HRegionMetricsWrapper wrapper) { + regionMetricSource = CompatibilityFactory.getInstance(RegionServerMetricsSourceFactory.class) + .createRegion(wrapper); + } + + public void close() { + regionMetricSource.close(); + } + + public void updatePut() { + regionMetricSource.updatePut(); + } + + public void updateDelete() { + regionMetricSource.updateDelete(); + } + + public void updateGet() { + regionMetricSource.updateGet(); + } + + public void updateAppend() { + regionMetricSource.updateAppend(); + } + + public void updateIncrement() { + regionMetricSource.updateIncrement(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java deleted file mode 100644 index 5d4beff..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionMetricsStorage.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hbase.regionserver.metrics; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hbase.util.Pair; - -/** - * This class if for maintaining the maps used to power metrics for hfiles, - * regions, and regionservers. It has methods to mutate and get state of metrics - * numbers. These numbers are exposed to Hadoop metrics through - * RegionServerDynamicMetrics. - */ -@InterfaceAudience.Private -public class RegionMetricsStorage { - - // for simple numeric metrics (# of blocks read from block cache) - private static final ConcurrentMap numericMetrics = - new ConcurrentHashMap(); - - // for simple numeric metrics (current block cache size) - // These ones are not reset to zero when queried, unlike the previous. - private static final ConcurrentMap numericPersistentMetrics = - new ConcurrentHashMap(); - - /** - * Used for metrics where we want track a metrics (such as latency) over a - * number of operations. - */ - private static final ConcurrentMap> timeVaryingMetrics = - new ConcurrentHashMap>(); - - public static Map getNumericMetrics() { - return numericMetrics; - } - - public static Map getNumericPersistentMetrics() { - return numericPersistentMetrics; - } - - public static Map> getTimeVaryingMetrics() { - return timeVaryingMetrics; - } - - public static void incrNumericMetric(String key, long amount) { - AtomicLong oldVal = numericMetrics.get(key); - if (oldVal == null) { - oldVal = numericMetrics.putIfAbsent(key, new AtomicLong(amount)); - if (oldVal == null) - return; - } - oldVal.addAndGet(amount); - } - - public static void incrTimeVaryingMetric(String key, long amount) { - Pair oldVal = timeVaryingMetrics.get(key); - if (oldVal == null) { - oldVal = - timeVaryingMetrics.putIfAbsent(key, - new Pair( - new AtomicLong(amount), - new AtomicInteger(1))); - if (oldVal == null) - return; - } - oldVal.getFirst().addAndGet(amount); // total time - oldVal.getSecond().incrementAndGet(); // increment ops by 1 - } - - public static void incrNumericPersistentMetric(String key, long amount) { - AtomicLong oldVal = numericPersistentMetrics.get(key); - if (oldVal == null) { - oldVal = numericPersistentMetrics.putIfAbsent(key, new AtomicLong(amount)); - if (oldVal == null) - return; - } - oldVal.addAndGet(amount); - } - - public static void setNumericMetric(String key, long amount) { - numericMetrics.put(key, new AtomicLong(amount)); - } - - public static long getNumericMetric(String key) { - AtomicLong m = numericMetrics.get(key); - if (m == null) - return 0; - return m.get(); - } - - public static Pair getTimeVaryingMetric(String key) { - Pair pair = timeVaryingMetrics.get(key); - if (pair == null) { - return new Pair(0L, 0); - } - - return new Pair(pair.getFirst().get(), pair.getSecond().get()); - } - - public static long getNumericPersistentMetric(String key) { - AtomicLong m = numericPersistentMetrics.get(key); - if (m == null) - return 0; - return m.get(); - } - - /** - * Clear all copies of the metrics this stores. - */ - public static void clear() { - timeVaryingMetrics.clear(); - numericMetrics.clear(); - numericPersistentMetrics.clear(); - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java deleted file mode 100644 index bb06a10..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicMetrics.java +++ /dev/null @@ -1,230 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.regionserver.metrics; - -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.metrics.MetricsContext; -import org.apache.hadoop.metrics.MetricsRecord; -import org.apache.hadoop.metrics.MetricsUtil; -import org.apache.hadoop.metrics.Updater; -import org.apache.hadoop.metrics.util.MetricsBase; -import org.apache.hadoop.metrics.util.MetricsLongValue; -import org.apache.hadoop.metrics.util.MetricsRegistry; -import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate; - -/** - * - * This class is for maintaining the various RPC statistics - * and publishing them through the metrics interfaces. - * This also registers the JMX MBean for RPC. - *

- * This class has a number of metrics variables that are publicly accessible; - * these variables (objects) have methods to update their values; - * for example: rpcQueueTime.inc(time) - * - */ -@InterfaceAudience.Private -public class RegionServerDynamicMetrics implements Updater { - private static final String UNABLE_TO_CLEAR = "Unable to clear RegionServerDynamicMetrics"; - - private MetricsRecord metricsRecord; - private MetricsContext context; - private final RegionServerDynamicStatistics rsDynamicStatistics; - private Method updateMbeanInfoIfMetricsListChanged = null; - private static final Log LOG = - LogFactory.getLog(RegionServerDynamicStatistics.class); - - private boolean reflectionInitialized = false; - private boolean needsUpdateMessage = false; - private Field recordMetricMapField; - private Field registryMetricMapField; - - /** - * The metrics variables are public: - * - they can be set directly by calling their set/inc methods - * -they can also be read directly - e.g. JMX does this. - */ - public final MetricsRegistry registry = new MetricsRegistry(); - - private RegionServerDynamicMetrics() { - this.context = MetricsUtil.getContext("hbase-dynamic"); - this.metricsRecord = MetricsUtil.createRecord( - this.context, - "RegionServerDynamicStatistics"); - context.registerUpdater(this); - this.rsDynamicStatistics = new RegionServerDynamicStatistics(this.registry); - try { - updateMbeanInfoIfMetricsListChanged = - this.rsDynamicStatistics.getClass().getSuperclass() - .getDeclaredMethod("updateMbeanInfoIfMetricsListChanged", - new Class[]{}); - updateMbeanInfoIfMetricsListChanged.setAccessible(true); - } catch (Exception e) { - LOG.error(e); - } - } - - public static RegionServerDynamicMetrics newInstance() { - RegionServerDynamicMetrics metrics = - new RegionServerDynamicMetrics(); - return metrics; - } - - public synchronized void setNumericMetric(String name, long amt) { - MetricsLongValue m = (MetricsLongValue)registry.get(name); - if (m == null) { - m = new MetricsLongValue(name, this.registry); - this.needsUpdateMessage = true; - } - m.set(amt); - } - - public synchronized void incrTimeVaryingMetric( - String name, - long amt, - int numOps) { - MetricsTimeVaryingRate m = (MetricsTimeVaryingRate)registry.get(name); - if (m == null) { - m = new MetricsTimeVaryingRate(name, this.registry); - this.needsUpdateMessage = true; - } - if (numOps > 0) { - m.inc(numOps, amt); - } - } - - /** - * Clear all metrics this exposes. - * Uses reflection to clear them from hadoop metrics side as well. - */ - @SuppressWarnings("rawtypes") - public void clear() { - this.needsUpdateMessage = true; - // If this is the first clear use reflection to get the two maps that hold copies of our - // metrics on the hadoop metrics side. We have to use reflection because there is not - // remove metrics on the hadoop side. If we can't get them then clearing old metrics - // is not possible and bailing out early is our best option. - if (!this.reflectionInitialized) { - this.reflectionInitialized = true; - try { - this.recordMetricMapField = this.metricsRecord.getClass().getDeclaredField("metricTable"); - this.recordMetricMapField.setAccessible(true); - } catch (SecurityException e) { - LOG.debug(UNABLE_TO_CLEAR); - return; - } catch (NoSuchFieldException e) { - LOG.debug(UNABLE_TO_CLEAR); - return; - } - - try { - this.registryMetricMapField = this.registry.getClass().getDeclaredField("metricsList"); - this.registryMetricMapField.setAccessible(true); - } catch (SecurityException e) { - LOG.debug(UNABLE_TO_CLEAR); - return; - } catch (NoSuchFieldException e) { - LOG.debug(UNABLE_TO_CLEAR); - return; - } - } - - - //If we found both fields then try and clear the maps. - if (this.recordMetricMapField != null && this.registryMetricMapField != null) { - try { - Map recordMap = (Map) this.recordMetricMapField.get(this.metricsRecord); - recordMap.clear(); - Map registryMap = (Map) this.registryMetricMapField.get(this.registry); - registryMap.clear(); - } catch (IllegalArgumentException e) { - LOG.debug(UNABLE_TO_CLEAR); - } catch (IllegalAccessException e) { - LOG.debug(UNABLE_TO_CLEAR); - } - } else { - LOG.debug(UNABLE_TO_CLEAR); - } - } - - /** - * Push the metrics to the monitoring subsystem on doUpdate() call. - * @param context ctx - */ - public void doUpdates(MetricsContext context) { - /* get dynamically created numeric metrics, and push the metrics */ - for (Entry entry : RegionMetricsStorage.getNumericMetrics().entrySet()) { - this.setNumericMetric(entry.getKey(), entry.getValue().getAndSet(0)); - } - /* get dynamically created numeric metrics, and push the metrics. - * These ones aren't to be reset; they are cumulative. */ - for (Entry entry : RegionMetricsStorage.getNumericPersistentMetrics().entrySet()) { - this.setNumericMetric(entry.getKey(), entry.getValue().get()); - } - /* get dynamically created time varying metrics, and push the metrics */ - for (Entry> entry : - RegionMetricsStorage.getTimeVaryingMetrics().entrySet()) { - Pair value = entry.getValue(); - this.incrTimeVaryingMetric(entry.getKey(), - value.getFirst().getAndSet(0), - value.getSecond().getAndSet(0)); - } - - // If there are new metrics sending this message to jmx tells it to update everything. - // This is not ideal we should just move to metrics2 that has full support for dynamic metrics. - if (needsUpdateMessage) { - try { - if (updateMbeanInfoIfMetricsListChanged != null) { - updateMbeanInfoIfMetricsListChanged.invoke(this.rsDynamicStatistics, - new Object[]{}); - } - } catch (Exception e) { - LOG.error(e); - } - needsUpdateMessage = false; - } - - - synchronized (registry) { - // Iterate through the registry to propagate the different rpc metrics. - for (String metricName : registry.getKeyList() ) { - MetricsBase value = registry.get(metricName); - value.pushMetric(metricsRecord); - } - } - metricsRecord.update(); - } - - public void shutdown() { - if (rsDynamicStatistics != null) - rsDynamicStatistics.shutdown(); - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicStatistics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicStatistics.java deleted file mode 100644 index b4df6a7..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerDynamicStatistics.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.regionserver.metrics; - -import javax.management.ObjectName; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hbase.metrics.MetricsMBeanBase; -import org.apache.hadoop.metrics.util.MBeanUtil; -import org.apache.hadoop.metrics.util.MetricsRegistry; - -/** - * Exports dynamic region server metric recorded in - * {@link RegionServerDynamicMetrics} as an MBean - * for JMX monitoring. - */ -@InterfaceAudience.Private -public class RegionServerDynamicStatistics extends MetricsMBeanBase { - private final ObjectName mbeanName; - - public RegionServerDynamicStatistics(MetricsRegistry registry) { - super(registry, "RegionServerDynamicStatistics"); - mbeanName = MBeanUtil.registerMBean("RegionServer", "RegionServerDynamicStatistics", this); - } - - public void shutdown() { - if (mbeanName != null) - MBeanUtil.unregisterMBean(mbeanName); - } - -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java index a6910f5..989e328 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java @@ -1,5 +1,4 @@ /** - * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,36 +17,11 @@ */ package org.apache.hadoop.hbase.regionserver.metrics; -import java.io.IOException; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryUsage; -import java.util.List; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.metrics.ExactCounterMetric; -import org.apache.hadoop.hbase.metrics.HBaseInfo; -import org.apache.hadoop.hbase.metrics.MetricsRate; -import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram; -import org.apache.hadoop.hbase.metrics.PersistentMetricsTimeVaryingRate; -import com.yammer.metrics.stats.Snapshot; -import org.apache.hadoop.hbase.regionserver.wal.HLogMetrics; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.hadoop.hbase.util.Strings; -import org.apache.hadoop.metrics.ContextFactory; -import org.apache.hadoop.metrics.MetricsContext; -import org.apache.hadoop.metrics.MetricsRecord; -import org.apache.hadoop.metrics.MetricsUtil; -import org.apache.hadoop.metrics.Updater; -import org.apache.hadoop.metrics.jvm.JvmMetrics; -import org.apache.hadoop.metrics.util.MetricsIntValue; -import org.apache.hadoop.metrics.util.MetricsLongValue; -import org.apache.hadoop.metrics.util.MetricsRegistry; -import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate; -import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong; -import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.CompatibilitySingletonFactory; /** * This class is for maintaining the various regionserver statistics @@ -56,557 +30,46 @@ import org.apache.hadoop.util.StringUtils; * This class has a number of metrics variables that are publicly accessible; * these variables (objects) have methods to update their values. */ +@InterfaceStability.Evolving @InterfaceAudience.Private -public class RegionServerMetrics implements Updater { - @SuppressWarnings({"FieldCanBeLocal"}) +public class RegionServerMetrics { private final Log LOG = LogFactory.getLog(this.getClass()); - private final MetricsRecord metricsRecord; - private long lastUpdate = System.currentTimeMillis(); - private long lastExtUpdate = System.currentTimeMillis(); - private long extendedPeriod = 0; - private static final int MB = 1024*1024; - private MetricsRegistry registry = new MetricsRegistry(); - private final RegionServerStatistics statistics; - - public final MetricsTimeVaryingRate atomicIncrementTime = - new MetricsTimeVaryingRate("atomicIncrementTime", registry); - - /** - * Count of regions carried by this regionserver - */ - public final MetricsIntValue regions = - new MetricsIntValue("regions", registry); - - /** - * Block cache size. - */ - public final MetricsLongValue blockCacheSize = - new MetricsLongValue("blockCacheSize", registry); - - /** - * Block cache free size. - */ - public final MetricsLongValue blockCacheFree = - new MetricsLongValue("blockCacheFree", registry); - - /** - * Block cache item count. - */ - public final MetricsLongValue blockCacheCount = - new MetricsLongValue("blockCacheCount", registry); - - /** - * Block cache hit count. - */ - public final MetricsLongValue blockCacheHitCount = - new MetricsLongValue("blockCacheHitCount", registry); - - /** - * Block cache miss count. - */ - public final MetricsLongValue blockCacheMissCount = - new MetricsLongValue("blockCacheMissCount", registry); - - /** - * Block cache evict count. - */ - public final MetricsLongValue blockCacheEvictedCount = - new MetricsLongValue("blockCacheEvictedCount", registry); - - /** - * Block hit ratio. - */ - public final MetricsIntValue blockCacheHitRatio = - new MetricsIntValue("blockCacheHitRatio", registry); - - /** - * Block hit caching ratio. This only includes the requests to the block - * cache where caching was turned on. See HBASE-2253. - */ - public final MetricsIntValue blockCacheHitCachingRatio = - new MetricsIntValue("blockCacheHitCachingRatio", registry); - - /** Block hit ratio for past N periods. */ - public final MetricsIntValue blockCacheHitRatioPastNPeriods = new MetricsIntValue("blockCacheHitRatioPastNPeriods", registry); - - /** Block hit caching ratio for past N periods */ - public final MetricsIntValue blockCacheHitCachingRatioPastNPeriods = new MetricsIntValue("blockCacheHitCachingRatioPastNPeriods", registry); - - /* - * Count of requests to the regionservers since last call to metrics update - */ - public final MetricsRate requests = new MetricsRate("requests", registry); - - /** - * Count of stores open on the regionserver. - */ - public final MetricsIntValue stores = new MetricsIntValue("stores", registry); - - /** - * Count of storefiles open on the regionserver. - */ - public final MetricsIntValue storefiles = - new MetricsIntValue("storefiles", registry); - - /** - * Count of read requests - */ - public final MetricsLongValue readRequestsCount = - new MetricsLongValue("readRequestsCount", registry); - - /** - * Count of write requests - */ - public final MetricsLongValue writeRequestsCount = - new MetricsLongValue("writeRequestsCount", registry); - - /** - * Count of checkAndMutates the failed the check - */ - public final MetricsLongValue checkAndMutateChecksFailed = - new MetricsLongValue("checkAndMutateChecksFailed", registry); - - /** - * Count of checkAndMutates that passed the check - */ - public final MetricsLongValue checkAndMutateChecksPassed = - new MetricsLongValue("checkAndMutateChecksPassed", registry); - /** - */ - public final MetricsIntValue storefileIndexSizeMB = - new MetricsIntValue("storefileIndexSizeMB", registry); - - /** The total size of block index root levels in this regionserver in KB. */ - public final MetricsIntValue rootIndexSizeKB = - new MetricsIntValue("rootIndexSizeKB", registry); - - /** Total size of all block indexes (not necessarily loaded in memory) */ - public final MetricsIntValue totalStaticIndexSizeKB = - new MetricsIntValue("totalStaticIndexSizeKB", registry); - - /** Total size of all Bloom filters (not necessarily loaded in memory) */ - public final MetricsIntValue totalStaticBloomSizeKB = - new MetricsIntValue("totalStaticBloomSizeKB", registry); - - /** - * HDFS blocks locality index - */ - public final MetricsIntValue hdfsBlocksLocalityIndex = - new MetricsIntValue("hdfsBlocksLocalityIndex", registry); - - /** - * Sum of all the memstore sizes in this regionserver in MB - */ - public final MetricsIntValue memstoreSizeMB = - new MetricsIntValue("memstoreSizeMB", registry); - - /** - * Number of put with WAL disabled in this regionserver in MB - */ - public final MetricsLongValue numPutsWithoutWAL = - new MetricsLongValue("numPutsWithoutWAL", registry); - - /** - * Possible data loss sizes (due to put with WAL disabled) in this regionserver in MB - */ - public final MetricsIntValue mbInMemoryWithoutWAL = - new MetricsIntValue("mbInMemoryWithoutWAL", registry); - - /** - * Size of the compaction queue. - */ - public final MetricsIntValue compactionQueueSize = - new MetricsIntValue("compactionQueueSize", registry); - - /** - * Size of the flush queue. - */ - public final MetricsIntValue flushQueueSize = - new MetricsIntValue("flushQueueSize", registry); - - /** - * filesystem sequential read latency distribution - */ - public final MetricsHistogram fsReadLatencyHistogram = - new MetricsHistogram("fsReadLatencyHistogram", registry); - - /** - * filesystem pread latency distribution - */ - public final MetricsHistogram fsPreadLatencyHistogram = - new MetricsHistogram("fsPreadLatencyHistogram", registry); - - /** - * Metrics on the distribution of filesystem write latencies (improved version of fsWriteLatency) - */ - public final MetricsHistogram fsWriteLatencyHistogram = - new MetricsHistogram("fsWriteLatencyHistogram", registry); - - - /** - * filesystem read latency - */ - public final MetricsTimeVaryingRate fsReadLatency = - new MetricsTimeVaryingRate("fsReadLatency", registry); - - /** - * filesystem positional read latency - */ - public final MetricsTimeVaryingRate fsPreadLatency = - new MetricsTimeVaryingRate("fsPreadLatency", registry); - - /** - * filesystem write latency - */ - public final MetricsTimeVaryingRate fsWriteLatency = - new MetricsTimeVaryingRate("fsWriteLatency", registry); - - /** - * size (in bytes) of data in HLog append calls - */ - public final MetricsTimeVaryingRate fsWriteSize = - new MetricsTimeVaryingRate("fsWriteSize", registry); - - /** - * filesystem sync latency - */ - public final MetricsTimeVaryingRate fsSyncLatency = - new MetricsTimeVaryingRate("fsSyncLatency", registry); - - - /** - * time each scheduled compaction takes - */ - protected final MetricsHistogram compactionTime = - new MetricsHistogram("compactionTime", registry); - - protected final MetricsHistogram compactionSize = - new MetricsHistogram("compactionSize", registry); - - /** - * time each scheduled flush takes - */ - protected final MetricsHistogram flushTime = - new MetricsHistogram("flushTime", registry); - - protected final MetricsHistogram flushSize = - new MetricsHistogram("flushSize", registry); - - public final MetricsLongValue slowHLogAppendCount = - new MetricsLongValue("slowHLogAppendCount", registry); - - public final MetricsTimeVaryingRate slowHLogAppendTime = - new MetricsTimeVaryingRate("slowHLogAppendTime", registry); - - public final PersistentMetricsTimeVaryingRate regionSplitSuccessCount = - new PersistentMetricsTimeVaryingRate("regionSplitSuccessCount", registry); - - public final MetricsLongValue regionSplitFailureCount = - new MetricsLongValue("regionSplitFailureCount", registry); - - /** - * Number of times checksum verification failed. - */ - public final MetricsLongValue checksumFailuresCount = - new MetricsLongValue("checksumFailuresCount", registry); - - public RegionServerMetrics() { - MetricsContext context = MetricsUtil.getContext("hbase"); - metricsRecord = MetricsUtil.createRecord(context, "regionserver"); - String name = Thread.currentThread().getName(); - metricsRecord.setTag("RegionServer", name); - context.registerUpdater(this); - // Add jvmmetrics. - JvmMetrics.init("RegionServer", name); - // Add Hbase Info metrics - HBaseInfo.init(); - - // export for JMX - statistics = new RegionServerStatistics(this.registry, name); - - // get custom attributes - try { - Object m = ContextFactory.getFactory().getAttribute("hbase.extendedperiod"); - if (m instanceof String) { - this.extendedPeriod = Long.parseLong((String) m)*1000; - } - } catch (IOException ioe) { - LOG.info("Couldn't load ContextFactory for Metrics config info"); - } - - LOG.info("Initialized"); + private RegionServerGeneralMetricsSource source; + private HRegionServerMetricsWrapper rsWrapper; + + public RegionServerMetrics(HRegionServerMetricsWrapper rsWrapper) { + this.rsWrapper = rsWrapper; + source = + CompatibilitySingletonFactory.getInstance(RegionServerMetricsSourceFactory.class) + .createGeneral(rsWrapper); } - public void shutdown() { - if (statistics != null) - statistics.shutdown(); + // for unit-test usage + public RegionServerGeneralMetricsSource getMetricsSource() { + return source; } - /** - * Since this object is a registered updater, this method will be called - * periodically, e.g. every 5 seconds. - * @param caller the metrics context that this responsible for calling us - */ - public void doUpdates(MetricsContext caller) { - synchronized (this) { - this.lastUpdate = System.currentTimeMillis(); - - // has the extended period for long-living stats elapsed? - if (this.extendedPeriod > 0 && - this.lastUpdate - this.lastExtUpdate >= this.extendedPeriod) { - this.lastExtUpdate = this.lastUpdate; - this.compactionTime.clear(); - this.compactionSize.clear(); - this.flushTime.clear(); - this.flushSize.clear(); - this.resetAllMinMax(); - } - - this.stores.pushMetric(this.metricsRecord); - this.storefiles.pushMetric(this.metricsRecord); - this.storefileIndexSizeMB.pushMetric(this.metricsRecord); - this.rootIndexSizeKB.pushMetric(this.metricsRecord); - this.totalStaticIndexSizeKB.pushMetric(this.metricsRecord); - this.totalStaticBloomSizeKB.pushMetric(this.metricsRecord); - this.memstoreSizeMB.pushMetric(this.metricsRecord); - this.mbInMemoryWithoutWAL.pushMetric(this.metricsRecord); - this.numPutsWithoutWAL.pushMetric(this.metricsRecord); - this.readRequestsCount.pushMetric(this.metricsRecord); - this.writeRequestsCount.pushMetric(this.metricsRecord); - this.regions.pushMetric(this.metricsRecord); - this.requests.pushMetric(this.metricsRecord); - this.compactionQueueSize.pushMetric(this.metricsRecord); - this.flushQueueSize.pushMetric(this.metricsRecord); - this.blockCacheSize.pushMetric(this.metricsRecord); - this.blockCacheFree.pushMetric(this.metricsRecord); - this.blockCacheCount.pushMetric(this.metricsRecord); - this.blockCacheHitCount.pushMetric(this.metricsRecord); - this.blockCacheMissCount.pushMetric(this.metricsRecord); - this.blockCacheEvictedCount.pushMetric(this.metricsRecord); - this.blockCacheHitRatio.pushMetric(this.metricsRecord); - this.blockCacheHitCachingRatio.pushMetric(this.metricsRecord); - this.hdfsBlocksLocalityIndex.pushMetric(this.metricsRecord); - this.blockCacheHitRatioPastNPeriods.pushMetric(this.metricsRecord); - this.blockCacheHitCachingRatioPastNPeriods.pushMetric(this.metricsRecord); - - // Mix in HFile and HLog metrics - // Be careful. Here is code for MTVR from up in hadoop: - // public synchronized void inc(final int numOps, final long time) { - // currentData.numOperations += numOps; - // currentData.time += time; - // long timePerOps = time/numOps; - // minMax.update(timePerOps); - // } - // Means you can't pass a numOps of zero or get a ArithmeticException / by zero. - // HLog metrics - addHLogMetric(HLogMetrics.getWriteTime(), this.fsWriteLatency); - addHLogMetric(HLogMetrics.getWriteSize(), this.fsWriteSize); - addHLogMetric(HLogMetrics.getSyncTime(), this.fsSyncLatency); - addHLogMetric(HLogMetrics.getSlowAppendTime(), this.slowHLogAppendTime); - this.slowHLogAppendCount.set(HLogMetrics.getSlowAppendCount()); - // HFile metrics, sequential reads - int ops = HFile.getReadOps(); - if (ops != 0) this.fsReadLatency.inc(ops, HFile.getReadTimeMs()); - // HFile metrics, positional reads - ops = HFile.getPreadOps(); - if (ops != 0) this.fsPreadLatency.inc(ops, HFile.getPreadTimeMs()); - this.checksumFailuresCount.set(HFile.getChecksumFailuresCount()); - - /* NOTE: removed HFile write latency. 2 reasons: - * 1) Mixing HLog latencies are far higher priority since they're - * on-demand and HFile is used in background (compact/flush) - * 2) HFile metrics are being handled at a higher level - * by compaction & flush metrics. - */ - - for(Long latency : HFile.getReadLatenciesNanos()) { - this.fsReadLatencyHistogram.update(latency); - } - for(Long latency : HFile.getPreadLatenciesNanos()) { - this.fsPreadLatencyHistogram.update(latency); - } - for(Long latency : HFile.getWriteLatenciesNanos()) { - this.fsWriteLatencyHistogram.update(latency); - } - - - // push the result - this.fsPreadLatency.pushMetric(this.metricsRecord); - this.fsReadLatency.pushMetric(this.metricsRecord); - this.fsWriteLatency.pushMetric(this.metricsRecord); - this.fsWriteSize.pushMetric(this.metricsRecord); - - this.fsReadLatencyHistogram.pushMetric(this.metricsRecord); - this.fsWriteLatencyHistogram.pushMetric(this.metricsRecord); - this.fsPreadLatencyHistogram.pushMetric(this.metricsRecord); - - this.fsSyncLatency.pushMetric(this.metricsRecord); - this.compactionTime.pushMetric(this.metricsRecord); - this.compactionSize.pushMetric(this.metricsRecord); - this.flushTime.pushMetric(this.metricsRecord); - this.flushSize.pushMetric(this.metricsRecord); - this.slowHLogAppendCount.pushMetric(this.metricsRecord); - this.regionSplitSuccessCount.pushMetric(this.metricsRecord); - this.regionSplitFailureCount.pushMetric(this.metricsRecord); - this.checksumFailuresCount.pushMetric(this.metricsRecord); - } - this.metricsRecord.update(); + public HRegionServerMetricsWrapper getRsWrapper() { + return rsWrapper; } - private void addHLogMetric(HLogMetrics.Metric logMetric, - MetricsTimeVaryingRate hadoopMetric) { - if (logMetric.count > 0) - hadoopMetric.inc(logMetric.min); - if (logMetric.count > 1) - hadoopMetric.inc(logMetric.max); - if (logMetric.count > 2) { - int ops = logMetric.count - 2; - hadoopMetric.inc(ops, logMetric.total - logMetric.max - logMetric.min); - } + public void updatePut(long t){ + source.updatePut(t); } - public void resetAllMinMax() { - this.atomicIncrementTime.resetMinMax(); - this.fsReadLatency.resetMinMax(); - this.fsWriteLatency.resetMinMax(); - this.fsWriteSize.resetMinMax(); - this.fsSyncLatency.resetMinMax(); - this.slowHLogAppendTime.resetMinMax(); + public void updateDelete(long t){ + source.updateDelete(t); } - /** - * @return Count of requests. - */ - public float getRequests() { - return this.requests.getPreviousIntervalValue(); + public void updateGet(long t){ + source.updateGet(t); } - /** - * @param time time that compaction took - * @param size bytesize of storefiles in the compaction - */ - public synchronized void addCompaction(long time, long size) { - this.compactionTime.update(time); - this.compactionSize.update(size); + public void updateIncrement(long t){ + source.updateIncrement(t); } - /** - * @param flushes history in - */ - public synchronized void addFlush(final List> flushes) { - for (Pair f : flushes) { - this.flushTime.update(f.getFirst()); - this.flushSize.update(f.getSecond()); - } - } - - /** - * @param inc How much to add to requests. - */ - public void incrementRequests(final int inc) { - this.requests.inc(inc); - } - - public void incrementSplitSuccessCount(long time) { - this.regionSplitSuccessCount.inc(time); - } - - public void incrementSplitFailureCount() { - this.regionSplitFailureCount.set(this.regionSplitFailureCount.get() + 1); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb = Strings.appendKeyValue(sb, "requestsPerSecond", Integer - .valueOf((int) this.requests.getPreviousIntervalValue())); - sb = Strings.appendKeyValue(sb, "numberOfOnlineRegions", - Integer.valueOf(this.regions.get())); - sb = Strings.appendKeyValue(sb, "numberOfStores", - Integer.valueOf(this.stores.get())); - sb = Strings.appendKeyValue(sb, "numberOfStorefiles", - Integer.valueOf(this.storefiles.get())); - sb = Strings.appendKeyValue(sb, this.storefileIndexSizeMB.getName(), - Integer.valueOf(this.storefileIndexSizeMB.get())); - sb = Strings.appendKeyValue(sb, "rootIndexSizeKB", - Integer.valueOf(this.rootIndexSizeKB.get())); - sb = Strings.appendKeyValue(sb, "totalStaticIndexSizeKB", - Integer.valueOf(this.totalStaticIndexSizeKB.get())); - sb = Strings.appendKeyValue(sb, "totalStaticBloomSizeKB", - Integer.valueOf(this.totalStaticBloomSizeKB.get())); - sb = Strings.appendKeyValue(sb, this.memstoreSizeMB.getName(), - Integer.valueOf(this.memstoreSizeMB.get())); - sb = Strings.appendKeyValue(sb, "mbInMemoryWithoutWAL", - Integer.valueOf(this.mbInMemoryWithoutWAL.get())); - sb = Strings.appendKeyValue(sb, "numberOfPutsWithoutWAL", - Long.valueOf(this.numPutsWithoutWAL.get())); - sb = Strings.appendKeyValue(sb, "readRequestsCount", - Long.valueOf(this.readRequestsCount.get())); - sb = Strings.appendKeyValue(sb, "writeRequestsCount", - Long.valueOf(this.writeRequestsCount.get())); - sb = Strings.appendKeyValue(sb, "compactionQueueSize", - Integer.valueOf(this.compactionQueueSize.get())); - sb = Strings.appendKeyValue(sb, "flushQueueSize", - Integer.valueOf(this.flushQueueSize.get())); - // Duplicate from jvmmetrics because metrics are private there so - // inaccessible. - MemoryUsage memory = - ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); - sb = Strings.appendKeyValue(sb, "usedHeapMB", - Long.valueOf(memory.getUsed()/MB)); - sb = Strings.appendKeyValue(sb, "maxHeapMB", - Long.valueOf(memory.getMax()/MB)); - sb = Strings.appendKeyValue(sb, this.blockCacheSize.getName()+"MB", - StringUtils.limitDecimalTo2((float)this.blockCacheSize.get()/MB)); - sb = Strings.appendKeyValue(sb, this.blockCacheFree.getName()+"MB", - StringUtils.limitDecimalTo2((float)this.blockCacheFree.get()/MB)); - sb = Strings.appendKeyValue(sb, this.blockCacheCount.getName(), - Long.valueOf(this.blockCacheCount.get())); - sb = Strings.appendKeyValue(sb, this.blockCacheHitCount.getName(), - Long.valueOf(this.blockCacheHitCount.get())); - sb = Strings.appendKeyValue(sb, this.blockCacheMissCount.getName(), - Long.valueOf(this.blockCacheMissCount.get())); - sb = Strings.appendKeyValue(sb, this.blockCacheEvictedCount.getName(), - Long.valueOf(this.blockCacheEvictedCount.get())); - sb = Strings.appendKeyValue(sb, this.blockCacheHitRatio.getName(), - Long.valueOf(this.blockCacheHitRatio.get())+"%"); - sb = Strings.appendKeyValue(sb, this.blockCacheHitCachingRatio.getName(), - Long.valueOf(this.blockCacheHitCachingRatio.get())+"%"); - sb = Strings.appendKeyValue(sb, this.hdfsBlocksLocalityIndex.getName(), - Long.valueOf(this.hdfsBlocksLocalityIndex.get())); - sb = Strings.appendKeyValue(sb, "slowHLogAppendCount", - Long.valueOf(this.slowHLogAppendCount.get())); - sb = appendHistogram(sb, this.fsReadLatencyHistogram); - sb = appendHistogram(sb, this.fsPreadLatencyHistogram); - sb = appendHistogram(sb, this.fsWriteLatencyHistogram); - - return sb.toString(); - } - - private StringBuilder appendHistogram(StringBuilder sb, - MetricsHistogram histogram) { - sb = Strings.appendKeyValue(sb, - histogram.getName() + "Mean", - StringUtils.limitDecimalTo2(histogram.getMean())); - sb = Strings.appendKeyValue(sb, - histogram.getName() + "Count", - StringUtils.limitDecimalTo2(histogram.getCount())); - final Snapshot s = histogram.getSnapshot(); - sb = Strings.appendKeyValue(sb, - histogram.getName() + "Median", - StringUtils.limitDecimalTo2(s.getMedian())); - sb = Strings.appendKeyValue(sb, - histogram.getName() + "75th", - StringUtils.limitDecimalTo2(s.get75thPercentile())); - sb = Strings.appendKeyValue(sb, - histogram.getName() + "95th", - StringUtils.limitDecimalTo2(s.get95thPercentile())); - sb = Strings.appendKeyValue(sb, - histogram.getName() + "99th", - StringUtils.limitDecimalTo2(s.get99thPercentile())); - sb = Strings.appendKeyValue(sb, - histogram.getName() + "999th", - StringUtils.limitDecimalTo2(s.get999thPercentile())); - return sb; + public void updateAppend(long t){ + source.updateAppend(t); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerStatistics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerStatistics.java deleted file mode 100644 index 37c978e..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerStatistics.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver.metrics; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hbase.metrics.MetricsMBeanBase; -import org.apache.hadoop.metrics.util.MBeanUtil; -import org.apache.hadoop.metrics.util.MetricsRegistry; - -import javax.management.ObjectName; - -/** - * Exports metrics recorded by {@link RegionServerMetrics} as an MBean - * for JMX monitoring. - */ -@InterfaceAudience.Private -public class RegionServerStatistics extends MetricsMBeanBase { - - private final ObjectName mbeanName; - - public RegionServerStatistics(MetricsRegistry registry, String rsName) { - super(registry, "RegionServerStatistics"); - mbeanName = MBeanUtil.registerMBean("RegionServer", - "RegionServerStatistics", this); - } - - public void shutdown() { - if (mbeanName != null) - MBeanUtil.unregisterMBean(mbeanName); - } - -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java deleted file mode 100644 index 5252984..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.regionserver.metrics; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.io.HeapSize; -import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.SchemaAware; -import org.apache.hadoop.hbase.util.ClassSize; - -/** - * A base class for objects that are associated with a particular table and - * column family. Provides a way to obtain the schema metrics object. - *

- * Due to the variety of things that can be associated with a table/CF, there - * are many ways to initialize this base class, either in the constructor, or - * from another similar object. For example, an HFile reader configures HFile - * blocks it reads with its own table/CF name. - */ -@InterfaceAudience.Private -public class SchemaConfigured implements HeapSize, SchemaAware { - private static final Log LOG = LogFactory.getLog(SchemaConfigured.class); - - // These are not final because we set them at runtime e.g. for HFile blocks. - private String cfName; - private String tableName; - - /** - * Schema metrics. Can only be initialized when we know our column family - * name, table name, and have had a chance to take a look at the - * configuration (in {@link SchemaMetrics#configureGlobally(Configuration)) - * so we know whether we are using per-table metrics. Therefore, initialized - * lazily. We don't make this volatile because even if a thread sees a stale - * value of null, it will be re-initialized to the same value that other - * threads see. - */ - private SchemaMetrics schemaMetrics; - - static { - if (ClassSize.OBJECT <= 0 || ClassSize.REFERENCE <= 0) { - throw new AssertionError("Class sizes are not initialized"); - } - } - - /** - * Estimated heap size of this object. We don't count table name and column - * family name characters because these strings are shared among many - * objects. We need unaligned size to reuse this in subclasses. - */ - public static final int SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE = - ClassSize.OBJECT + 3 * ClassSize.REFERENCE; - - private static final int SCHEMA_CONFIGURED_ALIGNED_HEAP_SIZE = - ClassSize.align(SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE); - - /** A helper constructor that configures the "use table name" flag. */ - private SchemaConfigured(Configuration conf) { - SchemaMetrics.configureGlobally(conf); - // Even though we now know if table-level metrics are used, we can't - // initialize schemaMetrics yet, because CF and table name are only known - // to the calling constructor. - } - - /** - * Creates an instance corresponding to an unknown table and column family. - * Used in unit tests. - */ - public static SchemaConfigured createUnknown() { - return new SchemaConfigured(null, SchemaMetrics.UNKNOWN, - SchemaMetrics.UNKNOWN); - } - - /** - * Default constructor. Only use when column/family name are not known at - * construction (i.e. for HFile blocks). - */ - public SchemaConfigured() { - } - - /** - * Initialize table and column family name from an HFile path. If - * configuration is null, - * {@link SchemaMetrics#configureGlobally(Configuration)} should have been - * called already. - */ - public SchemaConfigured(Configuration conf, Path path) { - this(conf); - - if (path != null) { - String splits[] = path.toString().split("/"); - int numPathLevels = splits.length; - if (numPathLevels > 0 && splits[0].isEmpty()) { - // This path starts with an '/'. - --numPathLevels; - } - if (numPathLevels < HFile.MIN_NUM_HFILE_PATH_LEVELS) { - LOG.warn("Could not determine table and column family of the HFile " - + "path " + path + ". Expecting at least " - + HFile.MIN_NUM_HFILE_PATH_LEVELS + " path components."); - path = null; - } else { - cfName = splits[splits.length - 2]; - if (cfName.equals(HRegion.REGION_TEMP_SUBDIR)) { - // This is probably a compaction or flush output file. We will set - // the real CF name later. - cfName = null; - } else { - cfName = cfName.intern(); - } - tableName = splits[splits.length - 4].intern(); - return; - } - } - - // This might also happen if we were passed an incorrect path. - cfName = SchemaMetrics.UNKNOWN; - tableName = SchemaMetrics.UNKNOWN; - } - - /** - * Used when we know an HFile path to deduce table and CF name from, but do - * not have a configuration. - * @param path an HFile path - */ - public SchemaConfigured(Path path) { - this(null, path); - } - - /** - * Used when we know table and column family name. If configuration is null, - * {@link SchemaMetrics#configureGlobally(Configuration)} should have been - * called already. - */ - public SchemaConfigured(Configuration conf, String tableName, String cfName) - { - this(conf); - if (tableName != null) { - this.tableName = tableName.intern(); - } - if (cfName != null) { - this.cfName = cfName.intern(); - } - } - - public SchemaConfigured(SchemaAware that) { - tableName = that.getTableName().intern(); - cfName = that.getColumnFamilyName().intern(); - schemaMetrics = that.getSchemaMetrics(); - } - - @Override - public String getTableName() { - return tableName; - } - - @Override - public String getColumnFamilyName() { - return cfName; - } - - @Override - public SchemaMetrics getSchemaMetrics() { - if (schemaMetrics == null) { - if (tableName == null || cfName == null) { - throw new IllegalStateException("Schema metrics requested before " + - "table/CF name initialization: " + schemaConfAsJSON()); - } - schemaMetrics = SchemaMetrics.getInstance(tableName, cfName); - } - return schemaMetrics; - } - - /** - * Configures the given object (e.g. an HFile block) with the current table - * and column family name, and the associated collection of metrics. Please - * note that this method configures the other object, not this - * object. - */ - public void passSchemaMetricsTo(SchemaConfigured target) { - if (isNull()) { - resetSchemaMetricsConf(target); - return; - } - - if (!isSchemaConfigured()) { - // Cannot configure another object if we are not configured ourselves. - throw new IllegalStateException("Table name/CF not initialized: " + - schemaConfAsJSON()); - } - - if (conflictingWith(target)) { - // Make sure we don't try to change table or CF name. - throw new IllegalArgumentException("Trying to change table name to \"" + - tableName + "\", CF name to \"" + cfName + "\" from " + - target.schemaConfAsJSON()); - } - - target.tableName = tableName.intern(); - target.cfName = cfName.intern(); - target.schemaMetrics = schemaMetrics; - target.schemaConfigurationChanged(); - } - - /** - * Reset schema metrics configuration in this particular instance. Used when - * legitimately need to re-initialize the object with another table/CF. - * This is a static method because its use is discouraged and reserved for - * when it is really necessary (e.g. writing HFiles in a temp direcdtory - * on compaction). - */ - public static void resetSchemaMetricsConf(SchemaConfigured target) { - target.tableName = null; - target.cfName = null; - target.schemaMetrics = null; - target.schemaConfigurationChanged(); - } - - @Override - public long heapSize() { - return SCHEMA_CONFIGURED_ALIGNED_HEAP_SIZE; - } - - public String schemaConfAsJSON() { - return "{\"tableName\":\"" + tableName + "\",\"cfName\":\"" + cfName - + "\"}"; - } - - protected boolean isSchemaConfigured() { - return tableName != null && cfName != null; - } - - private boolean isNull() { - return tableName == null && cfName == null && schemaMetrics == null; - } - - /** - * Determines if the current object's table/CF settings are not in conflict - * with the other object's table and CF. If the other object's table/CF are - * undefined, they are not considered to be in conflict. Used to sanity-check - * configuring the other object with this object's table/CF. - */ - boolean conflictingWith(SchemaConfigured other) { - return (other.tableName != null && !tableName.equals(other.tableName)) || - (other.cfName != null && !cfName.equals(other.cfName)); - } - - /** - * A hook method called when schema configuration changes. Can be used to - * update schema-aware member fields. - */ - protected void schemaConfigurationChanged() { - } - -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java deleted file mode 100644 index 7a824ac..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaMetrics.java +++ /dev/null @@ -1,926 +0,0 @@ -/* - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hbase.regionserver.metrics; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.commons.lang.mutable.MutableDouble; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; - -/** - * A names in a given column family or a (table, column - * family) combination. The following "dimensions" are supported: - *

    - *
  • Table name (optional; enabled based on configuration)
  • - *
  • Per-column family vs. aggregated. The aggregated mode is only supported - * when table name is not included.
  • - *
  • Block category (data, index, bloom filter, etc.)
  • - *
  • Whether the request is part of a compaction
  • - *
  • Metric type (read time, block read count, cache hits/misses, etc.)
  • - *
- *

- * An instance of this class does not store any metric values. It just allows - * to determine the correct metric name for each combination of the above - * dimensions. - *

- * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
Metric keyPer-table metrics conf settingDescription
OnOff
tbl.T.cf.CF.M Include Skip A specific column family of a specific table
tbl.T.M Skip Skip All column families in the given table
cf.CF.M Skip Include A specific column family in all tables
M Include Include All column families in all tables
- */ -@InterfaceAudience.Private -public class SchemaMetrics { - - public interface SchemaAware { - public String getTableName(); - public String getColumnFamilyName(); - public SchemaMetrics getSchemaMetrics(); - } - - private static final Log LOG = LogFactory.getLog(SchemaMetrics.class); - - public static enum BlockMetricType { - // Metric configuration: compactionAware, timeVarying - READ_TIME("Read", true, true), - READ_COUNT("BlockReadCnt", true, false), - CACHE_HIT("BlockReadCacheHitCnt", true, false), - CACHE_MISS("BlockReadCacheMissCnt", true, false), - - CACHE_SIZE("blockCacheSize", false, false), - CACHED("blockCacheNumCached", false, false), - EVICTED("blockCacheNumEvicted", false, false); - - private final String metricStr; - private final boolean compactionAware; - private final boolean timeVarying; - - BlockMetricType(String metricStr, boolean compactionAware, - boolean timeVarying) { - this.metricStr = metricStr; - this.compactionAware = compactionAware; - this.timeVarying = timeVarying; - } - - @Override - public String toString() { - return metricStr; - } - - private static final String BLOCK_METRIC_TYPE_RE; - static { - StringBuilder sb = new StringBuilder(); - for (BlockMetricType bmt : values()) { - if (sb.length() > 0) - sb.append("|"); - sb.append(bmt); - } - BLOCK_METRIC_TYPE_RE = sb.toString(); - } - }; - - public static enum StoreMetricType { - STORE_FILE_COUNT("storeFileCount"), - STORE_FILE_INDEX_SIZE("storeFileIndexSizeMB"), - STORE_FILE_SIZE_MB("storeFileSizeMB"), - STATIC_BLOOM_SIZE_KB("staticBloomSizeKB"), - MEMSTORE_SIZE_MB("memstoreSizeMB"), - STATIC_INDEX_SIZE_KB("staticIndexSizeKB"), - FLUSH_SIZE("flushSize"); - - private final String metricStr; - - StoreMetricType(String metricStr) { - this.metricStr = metricStr; - } - - @Override - public String toString() { - return metricStr; - } - }; - - // Constants - /** - * A string used when column family or table name is unknown, and in some - * unit tests. This should not normally show up in metric names but if it - * does it is better than creating a silent discrepancy in total vs. - * per-CF/table metrics. - */ - public static final String UNKNOWN = "__unknown"; - - public static final String TABLE_PREFIX = "tbl."; - public static final String CF_PREFIX = "cf."; - public static final String BLOCK_TYPE_PREFIX = "bt."; - public static final String REGION_PREFIX = "region."; - - public static final String CF_UNKNOWN_PREFIX = CF_PREFIX + UNKNOWN + "."; - public static final String CF_BAD_FAMILY_PREFIX = CF_PREFIX + "__badfamily."; - - /** Use for readability when obtaining non-compaction counters */ - public static final boolean NO_COMPACTION = false; - - /** - * A special schema metric value that means "all tables aggregated" or - * "all column families aggregated" when used as a table name or a column - * family name. - */ - public static final String TOTAL_KEY = ""; - - /** - * Special handling for meta-block-specific metrics for - * backwards-compatibility. - */ - private static final String META_BLOCK_CATEGORY_STR = "Meta"; - - private static final int NUM_BLOCK_CATEGORIES = - BlockCategory.values().length; - - private static final int NUM_METRIC_TYPES = - BlockMetricType.values().length; - - static final boolean[] BOOL_VALUES = new boolean[] { false, true }; - - private static final int NUM_BLOCK_METRICS = - NUM_BLOCK_CATEGORIES * // blockCategory - BOOL_VALUES.length * // isCompaction - NUM_METRIC_TYPES; // metricType - - private static final int NUM_STORE_METRIC_TYPES = - StoreMetricType.values().length; - - /** Conf key controlling whether we include table name in metric names */ - private static final String SHOW_TABLE_NAME_CONF_KEY = - "hbase.metrics.showTableName"; - - /** We use this when too many column families are involved in a request. */ - private static final String MORE_CFS_OMITTED_STR = "__more"; - - /** - * Maximum length of a metric name prefix. Used when constructing metric - * names from a set of column families participating in a request. - */ - private static final int MAX_METRIC_PREFIX_LENGTH = - 256 - MORE_CFS_OMITTED_STR.length(); - - // Global variables - /** - * Maps a string key consisting of table name and column family name, with - * table name optionally replaced with {@link #TOTAL_KEY} if per-table - * metrics are disabled, to an instance of this class. - */ - private static final ConcurrentHashMap - tableAndFamilyToMetrics = new ConcurrentHashMap(); - - /** Metrics for all tables and column families. */ - // This has to be initialized after cfToMetrics. - public static final SchemaMetrics ALL_SCHEMA_METRICS = - getInstance(TOTAL_KEY, TOTAL_KEY); - - /** - * Whether to include table name in metric names. If this is null, it has not - * been initialized. This is a global instance, but we also have a copy of it - * per a {@link SchemaMetrics} object to avoid synchronization overhead. - */ - private static volatile Boolean useTableNameGlobally; - - /** Whether we logged a message about configuration inconsistency */ - private static volatile boolean loggedConfInconsistency; - - // Instance variables - private final String[] blockMetricNames = new String[NUM_BLOCK_METRICS]; - private final boolean[] blockMetricTimeVarying = - new boolean[NUM_BLOCK_METRICS]; - - private final String[] bloomMetricNames = new String[2]; - private final String[] storeMetricNames = new String[NUM_STORE_METRIC_TYPES]; - private final String[] storeMetricNamesMax = new String[NUM_STORE_METRIC_TYPES]; - - private SchemaMetrics(final String tableName, final String cfName) { - String metricPrefix = SchemaMetrics.generateSchemaMetricsPrefix( - tableName, cfName); - - for (BlockCategory blockCategory : BlockCategory.values()) { - for (boolean isCompaction : BOOL_VALUES) { - for (BlockMetricType metricType : BlockMetricType.values()) { - if (!metricType.compactionAware && isCompaction) { - continue; - } - - StringBuilder sb = new StringBuilder(metricPrefix); - if (blockCategory != BlockCategory.ALL_CATEGORIES - && blockCategory != BlockCategory.META) { - String categoryStr = blockCategory.toString(); - categoryStr = categoryStr.charAt(0) - + categoryStr.substring(1).toLowerCase(); - sb.append(BLOCK_TYPE_PREFIX + categoryStr + "."); - } - - if (metricType.compactionAware) { - sb.append(isCompaction ? "compaction" : "fs"); - } - - // A special-case for meta blocks for backwards-compatibility. - if (blockCategory == BlockCategory.META) { - sb.append(META_BLOCK_CATEGORY_STR); - } - - sb.append(metricType); - - int i = getBlockMetricIndex(blockCategory, isCompaction, metricType); - blockMetricNames[i] = sb.toString().intern(); - blockMetricTimeVarying[i] = metricType.timeVarying; - } - } - } - - for (boolean isInBloom : BOOL_VALUES) { - bloomMetricNames[isInBloom ? 1 : 0] = metricPrefix - + (isInBloom ? "keyMaybeInBloomCnt" : "keyNotInBloomCnt"); - } - - for (StoreMetricType storeMetric : StoreMetricType.values()) { - String coreName = metricPrefix + storeMetric.toString(); - storeMetricNames[storeMetric.ordinal()] = coreName; - storeMetricNamesMax[storeMetric.ordinal()] = coreName + ".max"; - } - } - - - - public static final String METRIC_GETSIZE = "getsize"; - public static final String METRIC_NEXTSIZE = "nextsize"; - - /** - * Returns a {@link SchemaMetrics} object for the given table and column - * family, instantiating it if necessary. - * - * @param tableName table name (null is interpreted as "unknown"). This is - * ignored - * @param cfName column family name (null is interpreted as "unknown") - */ - public static SchemaMetrics getInstance(String tableName, String cfName) { - if (tableName == null) { - tableName = UNKNOWN; - } - - if (cfName == null) { - cfName = UNKNOWN; - } - - tableName = getEffectiveTableName(tableName); - - final String instanceKey = tableName + "\t" + cfName; - SchemaMetrics schemaMetrics = tableAndFamilyToMetrics.get(instanceKey); - if (schemaMetrics != null) { - return schemaMetrics; - } - - schemaMetrics = new SchemaMetrics(tableName, cfName); - SchemaMetrics existingMetrics = - tableAndFamilyToMetrics.putIfAbsent(instanceKey, schemaMetrics); - return existingMetrics != null ? existingMetrics : schemaMetrics; - } - - private static final int getBlockMetricIndex(BlockCategory blockCategory, - boolean isCompaction, BlockMetricType metricType) { - int i = 0; - i = i * NUM_BLOCK_CATEGORIES + blockCategory.ordinal(); - i = i * BOOL_VALUES.length + (isCompaction ? 1 : 0); - i = i * NUM_METRIC_TYPES + metricType.ordinal(); - return i; - } - - public String getBlockMetricName(BlockCategory blockCategory, - boolean isCompaction, BlockMetricType metricType) { - if (isCompaction && !metricType.compactionAware) { - throw new IllegalArgumentException("isCompaction cannot be true for " - + metricType); - } - return blockMetricNames[getBlockMetricIndex(blockCategory, isCompaction, - metricType)]; - } - - public String getBloomMetricName(boolean isInBloom) { - return bloomMetricNames[isInBloom ? 1 : 0]; - } - - /** - * Increments the given metric, both per-CF and aggregate, for both the given - * category and all categories in aggregate (four counters total). - */ - private void incrNumericMetric(BlockCategory blockCategory, - boolean isCompaction, BlockMetricType metricType) { - if (blockCategory == null) { - blockCategory = BlockCategory.UNKNOWN; // So that we see this in stats. - } - RegionMetricsStorage.incrNumericMetric(getBlockMetricName(blockCategory, - isCompaction, metricType), 1); - - if (blockCategory != BlockCategory.ALL_CATEGORIES) { - incrNumericMetric(BlockCategory.ALL_CATEGORIES, isCompaction, - metricType); - } - } - - private void addToReadTime(BlockCategory blockCategory, - boolean isCompaction, long timeMs) { - RegionMetricsStorage.incrTimeVaryingMetric(getBlockMetricName(blockCategory, - isCompaction, BlockMetricType.READ_TIME), timeMs); - - // Also update the read time aggregated across all block categories - if (blockCategory != BlockCategory.ALL_CATEGORIES) { - addToReadTime(BlockCategory.ALL_CATEGORIES, isCompaction, timeMs); - } - } - - /** - * Used to accumulate store metrics across multiple regions in a region - * server. These metrics are not "persistent", i.e. we keep overriding them - * on every update instead of incrementing, so we need to accumulate them in - * a temporary map before pushing them to the global metric collection. - * @param tmpMap a temporary map for accumulating store metrics - * @param storeMetricType the store metric type to increment - * @param val the value to add to the metric - */ - public void accumulateStoreMetric(final Map tmpMap, - StoreMetricType storeMetricType, double val) { - final String key = getStoreMetricName(storeMetricType); - if (tmpMap.get(key) == null) { - tmpMap.put(key, new MutableDouble(val)); - } else { - tmpMap.get(key).add(val); - } - - if (this == ALL_SCHEMA_METRICS) { - // also compute the max value across all Stores on this server - final String maxKey = getStoreMetricNameMax(storeMetricType); - MutableDouble cur = tmpMap.get(maxKey); - if (cur == null) { - tmpMap.put(maxKey, new MutableDouble(val)); - } else if (cur.doubleValue() < val) { - cur.setValue(val); - } - } else { - ALL_SCHEMA_METRICS.accumulateStoreMetric(tmpMap, storeMetricType, val); - } - } - - public String getStoreMetricName(StoreMetricType storeMetricType) { - return storeMetricNames[storeMetricType.ordinal()]; - } - - public String getStoreMetricNameMax(StoreMetricType storeMetricType) { - return storeMetricNamesMax[storeMetricType.ordinal()]; - } - - /** - * Update a metric that does not get reset on every poll. - * @param storeMetricType the store metric to update - * @param value the value to update the metric to - */ - public void updatePersistentStoreMetric(StoreMetricType storeMetricType, - long value) { - RegionMetricsStorage.incrNumericPersistentMetric( - storeMetricNames[storeMetricType.ordinal()], value); - } - - /** - * Updates the number of hits and the total number of block reads on a block - * cache hit. - */ - public void updateOnCacheHit(BlockCategory blockCategory, - boolean isCompaction) { - blockCategory.expectSpecific(); - incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_HIT); - incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT); - if (this != ALL_SCHEMA_METRICS) { - ALL_SCHEMA_METRICS.updateOnCacheHit(blockCategory, isCompaction); - } - } - - /** - * Updates read time, the number of misses, and the total number of block - * reads on a block cache miss. - */ - public void updateOnCacheMiss(BlockCategory blockCategory, - boolean isCompaction, long timeMs) { - blockCategory.expectSpecific(); - addToReadTime(blockCategory, isCompaction, timeMs); - incrNumericMetric(blockCategory, isCompaction, BlockMetricType.CACHE_MISS); - incrNumericMetric(blockCategory, isCompaction, BlockMetricType.READ_COUNT); - if (this != ALL_SCHEMA_METRICS) { - ALL_SCHEMA_METRICS.updateOnCacheMiss(blockCategory, isCompaction, - timeMs); - } - } - - /** - * Adds the given delta to the cache size for the given block category and - * the aggregate metric for all block categories. Updates both the per-CF - * counter and the counter for all CFs (four metrics total). The cache size - * metric is "persistent", i.e. it does not get reset when metrics are - * collected. - */ - public void addToCacheSize(BlockCategory category, long cacheSizeDelta) { - if (category == null) { - category = BlockCategory.ALL_CATEGORIES; - } - RegionMetricsStorage.incrNumericPersistentMetric(getBlockMetricName(category, false, - BlockMetricType.CACHE_SIZE), cacheSizeDelta); - - if (category != BlockCategory.ALL_CATEGORIES) { - addToCacheSize(BlockCategory.ALL_CATEGORIES, cacheSizeDelta); - } - } - - public void updateOnCachePutOrEvict(BlockCategory blockCategory, - long cacheSizeDelta, boolean isEviction) { - addToCacheSize(blockCategory, cacheSizeDelta); - incrNumericMetric(blockCategory, false, - isEviction ? BlockMetricType.EVICTED : BlockMetricType.CACHED); - if (this != ALL_SCHEMA_METRICS) { - ALL_SCHEMA_METRICS.updateOnCachePutOrEvict(blockCategory, cacheSizeDelta, - isEviction); - } - } - - /** - * Increments both the per-CF and the aggregate counter of bloom - * positives/negatives as specified by the argument. - */ - public void updateBloomMetrics(boolean isInBloom) { - RegionMetricsStorage.incrNumericMetric(getBloomMetricName(isInBloom), 1); - if (this != ALL_SCHEMA_METRICS) { - ALL_SCHEMA_METRICS.updateBloomMetrics(isInBloom); - } - } - - /** - * Sets the flag whether to use table name in metric names according to the - * given configuration. This must be called at least once before - * instantiating HFile readers/writers. - */ - public static void configureGlobally(Configuration conf) { - if (conf != null) { - final boolean useTableNameNew = - conf.getBoolean(SHOW_TABLE_NAME_CONF_KEY, false); - setUseTableName(useTableNameNew); - } else { - setUseTableName(false); - } - } - - /** - * Determine the table name to be included in metric keys. If the global - * configuration says that we should not use table names in metrics, - * we always return {@link #TOTAL_KEY} even if nontrivial table name is - * provided. - * - * @param tableName a table name or {@link #TOTAL_KEY} when aggregating - * across all tables - * @return the table name to use in metric keys - */ - private static String getEffectiveTableName(String tableName) { - if (!tableName.equals(TOTAL_KEY)) { - // We are provided with a non-trivial table name (including "unknown"). - // We need to know whether table name should be included into metrics. - if (useTableNameGlobally == null) { - throw new IllegalStateException("The value of the " - + SHOW_TABLE_NAME_CONF_KEY + " conf option has not been specified " - + "in SchemaMetrics"); - } - final boolean useTableName = useTableNameGlobally; - if (!useTableName) { - // Don't include table name in metric keys. - tableName = TOTAL_KEY; - } - } - return tableName; - } - - /** - * Method to transform a combination of a table name and a column family name - * into a metric key prefix. Tables/column family names equal to - * {@link #TOTAL_KEY} are omitted from the prefix. - * - * @param tableName the table name or {@link #TOTAL_KEY} for all tables - * @param cfName the column family name or {@link #TOTAL_KEY} for all CFs - * @return the metric name prefix, ending with a dot. - */ - public static String generateSchemaMetricsPrefix(String tableName, - final String cfName) { - tableName = getEffectiveTableName(tableName); - String schemaMetricPrefix = - tableName.equals(TOTAL_KEY) ? "" : TABLE_PREFIX + tableName + "."; - schemaMetricPrefix += - cfName.equals(TOTAL_KEY) ? "" : CF_PREFIX + cfName + "."; - return schemaMetricPrefix; - } - - public static String generateSchemaMetricsPrefix(byte[] tableName, - byte[] cfName) { - return generateSchemaMetricsPrefix(Bytes.toString(tableName), - Bytes.toString(cfName)); - } - - /** - * Method to transform a set of column families in byte[] format with table - * name into a metric key prefix. - * - * @param tableName the table name or {@link #TOTAL_KEY} for all tables - * @param families the ordered set of column families - * @return the metric name prefix, ending with a dot, or an empty string in - * case of invalid arguments. This is OK since we always expect - * some CFs to be included. - */ - public static String generateSchemaMetricsPrefix(String tableName, - Set families) { - if (families == null || families.isEmpty() || - tableName == null || tableName.isEmpty()) { - return ""; - } - - if (families.size() == 1) { - return generateSchemaMetricsPrefix(tableName, - Bytes.toString(families.iterator().next())); - } - - tableName = getEffectiveTableName(tableName); - List sortedFamilies = new ArrayList(families); - Collections.sort(sortedFamilies, Bytes.BYTES_COMPARATOR); - - StringBuilder sb = new StringBuilder(); - - int numCFsLeft = families.size(); - for (byte[] family : sortedFamilies) { - if (sb.length() > MAX_METRIC_PREFIX_LENGTH) { - sb.append(MORE_CFS_OMITTED_STR); - break; - } - --numCFsLeft; - sb.append(Bytes.toString(family)); - if (numCFsLeft > 0) { - sb.append("~"); - } - } - - return SchemaMetrics.generateSchemaMetricsPrefix(tableName, sb.toString()); - } - - - /** - * Get the prefix for metrics generated about a single region. - * @param tableName the table name or {@link #TOTAL_KEY} for all tables - * @param regionName regionName - * @return the prefix for this table/region combination. - */ - static String generateRegionMetricsPrefix(String tableName, String regionName) { - tableName = getEffectiveTableName(tableName); - String schemaMetricPrefix = - tableName.equals(TOTAL_KEY) ? "" : TABLE_PREFIX + tableName + "."; - schemaMetricPrefix += - regionName.equals(TOTAL_KEY) ? "" : REGION_PREFIX + regionName + "."; - - return schemaMetricPrefix; - } - - /** - * Sets the flag of whether to use table name in metric names. This flag - * is specified in configuration and is not expected to change at runtime, - * so we log an error message when it does change. - */ - private static void setUseTableName(final boolean useTableNameNew) { - if (useTableNameGlobally == null) { - // This configuration option has not yet been set. - useTableNameGlobally = useTableNameNew; - } else if (useTableNameGlobally != useTableNameNew - && !loggedConfInconsistency) { - // The configuration is inconsistent and we have not reported it - // previously. Once we report it, just keep ignoring the new setting. - LOG.error("Inconsistent configuration. Previous configuration " - + "for using table name in metrics: " + useTableNameGlobally + ", " - + "new configuration: " + useTableNameNew); - loggedConfInconsistency = true; - } - } - - // Methods used in testing - - private static final String regexEscape(String s) { - return s.replace(".", "\\."); - } - - /** - * Assume that table names used in tests don't contain dots, except for the - * META table. - */ - private static final String WORD_AND_DOT_RE_STR = "([^.]+|" + - regexEscape(Bytes.toString(HConstants.META_TABLE_NAME)) + - ")\\."; - - /** "tbl.." */ - private static final String TABLE_NAME_RE_STR = - "\\b" + regexEscape(TABLE_PREFIX) + WORD_AND_DOT_RE_STR; - - /** "cf.." */ - private static final String CF_NAME_RE_STR = - "\\b" + regexEscape(CF_PREFIX) + WORD_AND_DOT_RE_STR; - private static final Pattern CF_NAME_RE = Pattern.compile(CF_NAME_RE_STR); - - /** "tbl..cf.." */ - private static final Pattern TABLE_AND_CF_NAME_RE = Pattern.compile( - TABLE_NAME_RE_STR + CF_NAME_RE_STR); - - private static final Pattern BLOCK_CATEGORY_RE = Pattern.compile( - "\\b" + regexEscape(BLOCK_TYPE_PREFIX) + "[^.]+\\." + - // Also remove the special-case block type marker for meta blocks - "|" + META_BLOCK_CATEGORY_STR + "(?=" + - BlockMetricType.BLOCK_METRIC_TYPE_RE + ")"); - - /** - * A suffix for the "number of operations" part of "time-varying metrics". We - * only use this for metric verification in unit testing. Time-varying - * metrics are handled by a different code path in production. - */ - private static String NUM_OPS_SUFFIX = "numops"; - - /** - * A custom suffix that we use for verifying the second component of - * a "time-varying metric". - */ - private static String TOTAL_SUFFIX = "_total"; - private static final Pattern TIME_VARYING_SUFFIX_RE = Pattern.compile( - "(" + NUM_OPS_SUFFIX + "|" + TOTAL_SUFFIX + ")$"); - - void printMetricNames() { - for (BlockCategory blockCategory : BlockCategory.values()) { - for (boolean isCompaction : BOOL_VALUES) { - for (BlockMetricType metricType : BlockMetricType.values()) { - int i = getBlockMetricIndex(blockCategory, isCompaction, metricType); - LOG.debug("blockCategory=" + blockCategory + ", " - + "metricType=" + metricType + ", isCompaction=" + isCompaction - + ", metricName=" + blockMetricNames[i]); - } - } - } - } - - private Collection getAllMetricNames() { - List allMetricNames = new ArrayList(); - for (int i = 0; i < blockMetricNames.length; ++i) { - final String blockMetricName = blockMetricNames[i]; - final boolean timeVarying = blockMetricTimeVarying[i]; - if (blockMetricName != null) { - if (timeVarying) { - allMetricNames.add(blockMetricName + NUM_OPS_SUFFIX); - allMetricNames.add(blockMetricName + TOTAL_SUFFIX); - } else { - allMetricNames.add(blockMetricName); - } - } - } - allMetricNames.addAll(Arrays.asList(bloomMetricNames)); - return allMetricNames; - } - - private static final boolean isTimeVaryingKey(String metricKey) { - return metricKey.endsWith(NUM_OPS_SUFFIX) - || metricKey.endsWith(TOTAL_SUFFIX); - } - - private static final String stripTimeVaryingSuffix(String metricKey) { - return TIME_VARYING_SUFFIX_RE.matcher(metricKey).replaceAll(""); - } - - public static Map getMetricsSnapshot() { - Map metricsSnapshot = new TreeMap(); - for (SchemaMetrics cfm : tableAndFamilyToMetrics.values()) { - for (String metricName : cfm.getAllMetricNames()) { - long metricValue; - if (isTimeVaryingKey(metricName)) { - Pair totalAndCount = - RegionMetricsStorage.getTimeVaryingMetric(stripTimeVaryingSuffix(metricName)); - metricValue = metricName.endsWith(TOTAL_SUFFIX) ? - totalAndCount.getFirst() : totalAndCount.getSecond(); - } else { - metricValue = RegionMetricsStorage.getNumericMetric(metricName); - } - - metricsSnapshot.put(metricName, metricValue); - } - } - return metricsSnapshot; - } - - public static long getLong(Map m, String k) { - Long l = m.get(k); - return l != null ? l : 0; - } - - private static void putLong(Map m, String k, long v) { - if (v != 0) { - m.put(k, v); - } else { - m.remove(k); - } - } - - /** - * @return the difference between two sets of metrics (second minus first). - * Only includes keys that have nonzero difference. - */ - public static Map diffMetrics(Map a, - Map b) { - Set allKeys = new TreeSet(a.keySet()); - allKeys.addAll(b.keySet()); - Map diff = new TreeMap(); - for (String k : allKeys) { - long aVal = getLong(a, k); - long bVal = getLong(b, k); - if (aVal != bVal) { - diff.put(k, bVal - aVal); - } - } - return diff; - } - - public static void validateMetricChanges(Map oldMetrics) { - final Map newMetrics = getMetricsSnapshot(); - final Map allCfDeltas = new TreeMap(); - final Map allBlockCategoryDeltas = - new TreeMap(); - final Map deltas = diffMetrics(oldMetrics, newMetrics); - final Pattern cfTableMetricRE = - useTableNameGlobally ? TABLE_AND_CF_NAME_RE : CF_NAME_RE; - final Set allKeys = new TreeSet(oldMetrics.keySet()); - allKeys.addAll(newMetrics.keySet()); - - for (SchemaMetrics cfm : tableAndFamilyToMetrics.values()) { - for (String metricName : cfm.getAllMetricNames()) { - if (metricName.startsWith(CF_PREFIX + CF_PREFIX)) { - throw new AssertionError("Column family prefix used twice: " + - metricName); - } - - final long oldValue = getLong(oldMetrics, metricName); - final long newValue = getLong(newMetrics, metricName); - final long delta = newValue - oldValue; - - // Re-calculate values of metrics with no column family (or CF/table) - // specified based on all metrics with CF (or CF/table) specified. - if (delta != 0) { - if (cfm != ALL_SCHEMA_METRICS) { - final String aggregateMetricName = - cfTableMetricRE.matcher(metricName).replaceAll(""); - if (!aggregateMetricName.equals(metricName)) { - LOG.debug("Counting " + delta + " units of " + metricName - + " towards " + aggregateMetricName); - - putLong(allCfDeltas, aggregateMetricName, - getLong(allCfDeltas, aggregateMetricName) + delta); - } - } else { - LOG.debug("Metric=" + metricName + ", delta=" + delta); - } - } - - Matcher matcher = BLOCK_CATEGORY_RE.matcher(metricName); - if (matcher.find()) { - // Only process per-block-category metrics - String metricNoBlockCategory = matcher.replaceAll(""); - - putLong(allBlockCategoryDeltas, metricNoBlockCategory, - getLong(allBlockCategoryDeltas, metricNoBlockCategory) + delta); - } - } - } - - StringBuilder errors = new StringBuilder(); - for (String key : ALL_SCHEMA_METRICS.getAllMetricNames()) { - long actual = getLong(deltas, key); - long expected = getLong(allCfDeltas, key); - if (actual != expected) { - if (errors.length() > 0) - errors.append("\n"); - errors.append("The all-CF metric " + key + " changed by " - + actual + " but the aggregation of per-CF/table metrics " - + "yields " + expected); - } - } - - // Verify metrics computed for all block types based on the aggregation - // of per-block-type metrics. - for (String key : allKeys) { - if (BLOCK_CATEGORY_RE.matcher(key).find() || - key.contains(ALL_SCHEMA_METRICS.getBloomMetricName(false)) || - key.contains(ALL_SCHEMA_METRICS.getBloomMetricName(true))){ - // Skip per-block-category metrics. Also skip bloom filters, because - // they are not aggregated per block type. - continue; - } - long actual = getLong(deltas, key); - long expected = getLong(allBlockCategoryDeltas, key); - if (actual != expected) { - if (errors.length() > 0) - errors.append("\n"); - errors.append("The all-block-category metric " + key - + " changed by " + actual + " but the aggregation of " - + "per-block-category metrics yields " + expected); - } - } - - if (errors.length() > 0) { - throw new AssertionError(errors.toString()); - } - } - - /** - * Creates an instance pretending both the table and column family are - * unknown. Used in unit tests. - */ - public static SchemaMetrics getUnknownInstanceForTest() { - return getInstance(UNKNOWN, UNKNOWN); - } - - /** - * Set the flag to use or not use table name in metric names. Used in unit - * tests, so the flag can be set arbitrarily. - */ - public static void setUseTableNameInTest(final boolean useTableNameNew) { - useTableNameGlobally = useTableNameNew; - } - - /** Formats the given map of metrics in a human-readable way. */ - public static String formatMetrics(Map metrics) { - StringBuilder sb = new StringBuilder(); - for (Map.Entry entry : metrics.entrySet()) { - if (sb.length() > 0) { - sb.append('\n'); - } - sb.append(entry.getKey() + " : " + entry.getValue()); - } - return sb.toString(); - } - -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationSourceMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationSourceMetrics.java index fe24d39..415297e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationSourceMetrics.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/metrics/ReplicationSourceMetrics.java @@ -146,13 +146,13 @@ public class ReplicationSourceMetrics { /** Removes all metrics about this Source. */ public void clear() { - rms.removeGauge(sizeOfLogQueKey); + rms.removeMetric(sizeOfLogQueKey); rms.decGauge(SOURCE_SIZE_OF_LOG_QUEUE, lastQueueSize); lastQueueSize = 0; - rms.removeGauge(ageOfLastShippedOpKey); + rms.removeMetric(ageOfLastShippedOpKey); - rms.removeCounter(logEditsFilteredKey); - rms.removeCounter(logEditsReadKey); + rms.removeMetric(logEditsFilteredKey); + rms.removeMetric(logEditsReadKey); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java index b08f8f2..9e94004 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java @@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.io.hfile.LruBlockCache; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.MemStore; import org.apache.hadoop.hbase.regionserver.HStore; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.junit.experimental.categories.Category; @@ -313,12 +312,6 @@ public class TestHeapSize extends TestCase { assertEquals(expected, actual); } - // SchemaConfigured - LOG.debug("Heap size for: " + SchemaConfigured.class.getName()); - SchemaConfigured sc = new SchemaConfigured(null, "myTable", "myCF"); - assertEquals(ClassSize.estimateBase(SchemaConfigured.class, true), - sc.heapSize()); - // Store Overhead cl = HStore.class; actual = HStore.FIXED_OVERHEAD; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java index c73c491..6eb26ba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.MultithreadedTestUtil; import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.util.ChecksumType; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; public class CacheTestUtils { @@ -281,17 +280,6 @@ public class CacheTestUtils { } }; } - - @Override - public BlockType getBlockType() { - return BlockType.DATA; - } - - @Override - public SchemaMetrics getSchemaMetrics() { - return SchemaMetrics.getUnknownInstanceForTest(); - } - } private static HFileBlockPair[] generateHFileBlocks(int blockSize, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java index 84460e9..42ec110 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java @@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.util.BloomFilterFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; @@ -360,13 +359,9 @@ public class TestCacheOnWrite { (LruBlockCache) new CacheConfig(conf).getBlockCache(); blockCache.clearCache(); assertEquals(0, blockCache.getBlockTypeCountsForTest().size()); - Map metricsBefore = SchemaMetrics.getMetricsSnapshot(); region.compactStores(); LOG.debug("compactStores() returned"); - SchemaMetrics.validateMetricChanges(metricsBefore); - Map compactionMetrics = SchemaMetrics.diffMetrics( - metricsBefore, SchemaMetrics.getMetricsSnapshot()); - LOG.debug(SchemaMetrics.formatMetrics(compactionMetrics)); + Map blockTypesInCache = blockCache.getBlockTypeCountsForTest(); LOG.debug("Block types in cache: " + blockTypesInCache); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java index 7a712b4..d8e09fc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java @@ -20,8 +20,6 @@ package org.apache.hadoop.hbase.io.hfile; import java.nio.ByteBuffer; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; - import junit.framework.TestCase; import org.apache.hadoop.hbase.SmallTests; import org.junit.experimental.categories.Category; @@ -137,15 +135,6 @@ public class TestCachedBlockQueue extends TestCase { return null; } - @Override - public BlockType getBlockType() { - return BlockType.DATA; - } - - @Override - public SchemaMetrics getSchemaMetrics() { - return SchemaMetrics.ALL_SCHEMA_METRICS; - } }, accessTime, false); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java index 5f8214e..7eb4916 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java @@ -35,8 +35,6 @@ import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -102,7 +100,7 @@ public class TestForceCacheImportantBlocks { TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, BLOCK_SIZE); - SchemaMetrics.setUseTableNameInTest(false); + HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes(CF)) .setMaxVersions(MAX_VERSIONS) @@ -112,16 +110,12 @@ public class TestForceCacheImportantBlocks { hcd.setBlockCacheEnabled(cfCacheEnabled); HRegion region = TEST_UTIL.createTestRegion(TABLE, hcd); writeTestData(region); - Map metricsBefore = SchemaMetrics.getMetricsSnapshot(); + for (int i = 0; i < NUM_ROWS; ++i) { Get get = new Get(Bytes.toBytes("row" + i)); region.get(get, null); } - SchemaMetrics.validateMetricChanges(metricsBefore); - Map metricsAfter = SchemaMetrics.getMetricsSnapshot(); - Map metricsDelta = SchemaMetrics.diffMetrics(metricsBefore, - metricsAfter); - SchemaMetrics metrics = SchemaMetrics.getInstance(TABLE, CF); + List importantBlockCategories = new ArrayList(); importantBlockCategories.add(BlockCategory.BLOOM); @@ -129,30 +123,8 @@ public class TestForceCacheImportantBlocks { // We only have index blocks for HFile v2. importantBlockCategories.add(BlockCategory.INDEX); } - - for (BlockCategory category : importantBlockCategories) { - String hitsMetricName = getMetricName(metrics, category); - assertTrue("Metric " + hitsMetricName + " was not incremented", - metricsDelta.containsKey(hitsMetricName)); - long hits = metricsDelta.get(hitsMetricName); - assertTrue("Invalid value of " + hitsMetricName + ": " + hits, hits > 0); - } - - if (!cfCacheEnabled) { - // Caching is turned off for the CF, so make sure we are not caching data - // blocks. - String dataHitMetricName = getMetricName(metrics, BlockCategory.DATA); - assertFalse("Nonzero value for metric " + dataHitMetricName, - metricsDelta.containsKey(dataHitMetricName)); - } } - private String getMetricName(SchemaMetrics metrics, BlockCategory category) { - String hitsMetricName = - metrics.getBlockMetricName(category, SchemaMetrics.NO_COMPACTION, - BlockMetricType.CACHE_HIT); - return hitsMetricName; - } private void writeTestData(HRegion region) throws IOException { for (int i = 0; i < NUM_ROWS; ++i) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java index ff97fe6..a5362ca 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java @@ -33,8 +33,6 @@ import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; import org.apache.hadoop.hbase.io.encoding.RedundantKVGenerator; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.util.ChecksumType; import org.junit.After; import org.junit.Before; @@ -52,8 +50,6 @@ public class TestHFileDataBlockEncoder { new HBaseTestingUtility(); private HFileDataBlockEncoderImpl blockEncoder; private RedundantKVGenerator generator = new RedundantKVGenerator(); - private SchemaConfigured UNKNOWN_TABLE_AND_CF = - SchemaConfigured.createUnknown(); private boolean includesMemstoreTS; /** @@ -75,7 +71,6 @@ public class TestHFileDataBlockEncoder { @Before public void setUp() { conf = TEST_UTIL.getConfiguration(); - SchemaMetrics.configureGlobally(conf); } /** @@ -161,7 +156,6 @@ public class TestHFileDataBlockEncoder { HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, 0, includesMemstoreTS, HFileReaderV2.MAX_MINOR_VERSION, 0, ChecksumType.NULL.getCode(), 0); - UNKNOWN_TABLE_AND_CF.passSchemaMetricsTo(b); return b; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java index 6f6e4e6..3194238 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.SmallTests; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.util.Bytes; import org.junit.After; @@ -46,21 +45,13 @@ public class TestHFileReaderV1 { private Configuration conf; private FileSystem fs; - private Map startingMetrics; private static final int N = 1000; @Before public void setUp() throws IOException { - startingMetrics = SchemaMetrics.getMetricsSnapshot(); conf = TEST_UTIL.getConfiguration(); fs = FileSystem.get(conf); - SchemaMetrics.configureGlobally(conf); - } - - @After - public void tearDown() throws Exception { - SchemaMetrics.validateMetricChanges(startingMetrics); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java index ebb78bf..290ba4a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java @@ -26,11 +26,10 @@ import java.util.Collection; import java.util.Map; import java.util.Random; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.hfile.LruBlockCache.EvictionThread; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; -import org.apache.hadoop.hbase.regionserver.metrics.TestSchemaMetrics; import org.apache.hadoop.hbase.util.ClassSize; import org.junit.After; import org.junit.Before; @@ -47,30 +46,9 @@ import org.junit.runners.Parameterized.Parameters; * evictions run when they're supposed to and do what they should, * and that cached blocks are accessible when expected to be. */ -@RunWith(Parameterized.class) @Category(SmallTests.class) public class TestLruBlockCache { - private Map startingMetrics; - - public TestLruBlockCache(boolean useTableName) { - SchemaMetrics.setUseTableNameInTest(useTableName); - } - - @Parameters - public static Collection parameters() { - return TestSchemaMetrics.parameters(); - } - - @Before - public void setUp() throws Exception { - startingMetrics = SchemaMetrics.getMetricsSnapshot(); - } - - @After - public void tearDown() throws Exception { - SchemaMetrics.validateMetricChanges(startingMetrics); - } @Test public void testBackgroundEvictionThread() throws Exception { @@ -673,16 +651,6 @@ public class TestLruBlockCache { } @Override - public BlockType getBlockType() { - return BlockType.DATA; - } - - @Override - public SchemaMetrics getSchemaMetrics() { - return SchemaMetrics.getUnknownInstanceForTest(); - } - - @Override public int getSerializedLength() { return 0; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java index 7fb7a7c..0179724 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java @@ -39,8 +39,6 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.InternalScanner; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; import org.junit.Test; @@ -146,9 +144,6 @@ public class TestScannerSelectionUsingTTL { Set accessedFiles = cache.getCachedFileNamesForTest(); LOG.debug("Files accessed during scan: " + accessedFiles); - Map metricsBeforeCompaction = - SchemaMetrics.getMetricsSnapshot(); - // Exercise both compaction codepaths. if (explicitCompaction) { region.getStore(FAMILY_BYTES).compactRecentForTesting(totalNumFiles); @@ -156,18 +151,6 @@ public class TestScannerSelectionUsingTTL { region.compactStores(); } - SchemaMetrics.validateMetricChanges(metricsBeforeCompaction); - Map compactionMetrics = - SchemaMetrics.diffMetrics(metricsBeforeCompaction, - SchemaMetrics.getMetricsSnapshot()); - long compactionDataBlocksRead = SchemaMetrics.getLong( - compactionMetrics, - SchemaMetrics.getInstance(TABLE, FAMILY).getBlockMetricName( - BlockCategory.DATA, true, BlockMetricType.READ_COUNT)); - assertEquals("Invalid number of blocks accessed during compaction. " + - "We only expect non-expired files to be accessed.", - numFreshFiles, compactionDataBlocksRead); region.close(); } - } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/HRegionServerMetricsWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/HRegionServerMetricsWrapperStub.java new file mode 100644 index 0000000..0750296 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/HRegionServerMetricsWrapperStub.java @@ -0,0 +1,138 @@ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.regionserver.metrics.HRegionServerMetricsWrapper; + +public class HRegionServerMetricsWrapperStub implements HRegionServerMetricsWrapper { + + @Override public String getServerName() { + return "test"; + } + + @Override public String getClusterId() { + return "tClusterId"; + } + + @Override public String getZookeeperQuorum() { + return "zk"; + } + + @Override public String getCoprocessors() { + return "co-process"; + } + + @Override public long getRegionServerStartTime() { + return 100; + } + + @Override public long getNumOnlineRegions() { + return 101; + } + + @Override public long getNumStores() { + return 2; + } + + @Override public long getNumStoreFiles() { + return 300; + } + + @Override public long getMemstoreSize() { + return 1025; + } + + @Override public long getStoreFileSize() { + return 1900; + } + + @Override public double getRequestsPerSecond() { + return 0; + } + + @Override public long getTotalRequestCount() { + return 899; + } + + @Override public long getReadRequestsCount() { + return 997; + } + + @Override public long getWriteRequestsCount() { + return 707; + } + + @Override public long getCheckAndMutateChecksFailed() { + return 401; + } + + @Override public long getCheckAndMutateChecksPassed() { + return 405; + } + + @Override public long getStoreFileIndexSize() { + return 406; + } + + @Override public long getTotalStaticIndexSize() { + return 407; + } + + @Override public long getTotalStaticBloomSize() { + return 408; + } + + @Override public long getNumPutsWithoutWAL() { + return 409; + } + + @Override public long getDataInMemoryWithoutWAL() { + return 410; + } + + @Override public int getPercentFileLocal() { + return 99; + } + + @Override public int getCompactionQueueSize() { + return 411; + } + + @Override public int getFlushQueueSize() { + return 412; + } + + @Override public long getBlockCacheFreeSize() { + return 413; + } + + @Override public long getBlockCacheCount() { + return 414; + } + + @Override public long getBlockCacheSize() { + return 415; + } + + @Override public long getBlockCacheHitCount() { + return 416; + } + + @Override public long getBlockCacheMissCount() { + return 417; + } + + @Override public long getBlockCacheEvictedCount() { + return 418; + } + + @Override public int getBlockCacheHitPercent() { + return 98; + } + + @Override public int getBlockCacheHitCachingPercent() { + return 97; + } + + @Override public void forceRecompute() { + //IGNORED. + } + } \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java index 19c690f..f8db58b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestBlocksScanned.java @@ -28,9 +28,6 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.hfile.Compression; -import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Assert; import org.junit.Test; @@ -52,7 +49,7 @@ public class TestBlocksScanned extends HBaseTestCase { @Override public void setUp() throws Exception { super.setUp(); - SchemaMetrics.setUseTableNameInTest(true); + TEST_UTIL = new HBaseTestingUtility(); TESTTABLEDESC = new HTableDescriptor(TABLE); @@ -72,11 +69,6 @@ public class TestBlocksScanned extends HBaseTestCase { addContent(r, FAMILY, COL); r.flushcache(); - // Get the per-cf metrics - SchemaMetrics schemaMetrics = - SchemaMetrics.getInstance(Bytes.toString(TABLE), Bytes.toString(FAMILY)); - Map schemaMetricSnapshot = SchemaMetrics.getMetricsSnapshot(); - // Do simple test of getting one row only first. Scan scan = new Scan(Bytes.toBytes("aaa"), Bytes.toBytes("aaz")); scan.addColumn(FAMILY, COL); @@ -92,26 +84,6 @@ public class TestBlocksScanned extends HBaseTestCase { int kvPerBlock = (int) Math.ceil(BLOCK_SIZE / (double) results.get(0).getLength()); Assert.assertEquals(2, kvPerBlock); - - long expectDataBlockRead = (long) Math.ceil(expectResultSize / (double) kvPerBlock); - long expectIndexBlockRead = expectDataBlockRead; - - verifyDataAndIndexBlockRead(schemaMetricSnapshot, schemaMetrics, - expectDataBlockRead, expectIndexBlockRead); } - private void verifyDataAndIndexBlockRead(Map previousMetricSnapshot, - SchemaMetrics schemaMetrics, long expectDataBlockRead, long expectedIndexBlockRead){ - Map currentMetricsSnapshot = SchemaMetrics.getMetricsSnapshot(); - Map diffs = - SchemaMetrics.diffMetrics(previousMetricSnapshot, currentMetricsSnapshot); - - long dataBlockRead = SchemaMetrics.getLong(diffs, - schemaMetrics.getBlockMetricName(BlockCategory.DATA, false, BlockMetricType.READ_COUNT)); - long indexBlockRead = SchemaMetrics.getLong(diffs, - schemaMetrics.getBlockMetricName(BlockCategory.INDEX, false, BlockMetricType.READ_COUNT)); - - Assert.assertEquals(expectDataBlockRead, dataBlockRead); - Assert.assertEquals(expectedIndexBlockRead, indexBlockRead); - } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java index 1fb16cd..17cd2d8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCacheOnWriteInSchema.java @@ -194,7 +194,6 @@ public class TestCacheOnWriteInSchema { BlockCache cache = cacheConf.getBlockCache(); StoreFile sf = new StoreFile(fs, path, conf, cacheConf, BloomType.ROWCOL, null); - store.passSchemaMetricsTo(sf); HFileReaderV2 reader = (HFileReaderV2) sf.createReader().getHFileReader(); try { // Open a scanner with (on read) caching disabled diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index b908763..de7ebd7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -78,7 +78,6 @@ import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl; import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.hbase.regionserver.wal.HLogFactory; import org.apache.hadoop.hbase.regionserver.wal.HLogUtil; @@ -132,14 +131,11 @@ public class TestHRegion extends HBaseTestCase { protected final byte [] row2 = Bytes.toBytes("rowB"); - private Map startingMetrics; - /** * @see org.apache.hadoop.hbase.HBaseTestCase#setUp() */ @Override protected void setUp() throws Exception { - startingMetrics = SchemaMetrics.getMetricsSnapshot(); super.setUp(); } @@ -147,7 +143,6 @@ public class TestHRegion extends HBaseTestCase { protected void tearDown() throws Exception { super.tearDown(); EnvironmentEdgeManagerTestHelper.reset(); - SchemaMetrics.validateMetricChanges(startingMetrics); } ////////////////////////////////////////////////////////////////////////////// @@ -3676,9 +3671,6 @@ public class TestHRegion extends HBaseTestCase { info = new HRegionInfo(htd.getName(), HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY, false); Path path = new Path(DIR + "testStatusSettingToAbortIfAnyExceptionDuringRegionInitilization"); - // no where we are instantiating HStore in this test case so useTableNameGlobally is null. To - // avoid NullPointerException we are setting useTableNameGlobally to false. - SchemaMetrics.setUseTableNameInTest(false); region = HRegion.newHRegion(path, null, fs, conf, info, htd, null); // region initialization throws IOException and set task state to ABORTED. region.initialize(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMXBean.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMXBean.java deleted file mode 100644 index 9d5a531..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMXBean.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.regionserver; - -import junit.framework.Assert; - -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.master.HMaster; -import org.apache.hadoop.hbase.MediumTests; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(MediumTests.class) -public class TestMXBean { - - private static final HBaseTestingUtility TEST_UTIL = - new HBaseTestingUtility(); - - @BeforeClass - public static void setup() throws Exception { - TEST_UTIL.startMiniCluster(1, 1); - } - - @AfterClass - public static void teardown() throws Exception { - TEST_UTIL.shutdownMiniCluster(); - } - - @Test - public void testInfo() { - HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); - HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); - MXBeanImpl info = MXBeanImpl.init(rs); - - Assert.assertEquals(rs.getServerName().getServerName(), - info.getServerName()); - Assert.assertEquals(rs.getCoprocessors().length, - info.getCoprocessors().length); - rs.getConfiguration().setInt("hbase.master.info.port", - master.getServerName().getPort()); - Assert.assertEquals(rs.getZooKeeperWatcher().getQuorum(), - info.getZookeeperQuorum()); - } - -} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java index f1ab788..96f90cf 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java @@ -35,7 +35,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.regionserver.HStore.ScanInfo; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.util.Bytes; import com.google.common.base.Joiner; @@ -61,7 +60,6 @@ public class TestMemStore extends TestCase { super.setUp(); this.mvcc = new MultiVersionConsistencyControl(); this.memstore = new MemStore(); - SchemaMetrics.setUseTableNameInTest(false); } public void testPutSameKey() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java index 9490691..1fd82fb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java @@ -47,8 +47,6 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.Compression; -import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Before; import org.junit.Test; @@ -121,12 +119,6 @@ public class TestMultiColumnScanner { assertTrue(TIMESTAMPS[i] < TIMESTAMPS[i + 1]); } - @Before - public void setUp() { - SchemaMetrics.configureGlobally(TEST_UTIL.getConfiguration()); - } - - @Parameters public static final Collection parameters() { List parameters = new ArrayList(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java index a0ee77f..59abd16 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRSStatusServlet.java @@ -59,8 +59,7 @@ public class TestRSStatusServlet { new ServerName("localhost", FAKE_IPC_PORT, 11111); private final GetServerInfoResponse fakeResponse = ResponseConverter.buildGetServerInfoResponse(fakeServerName, FAKE_WEB_PORT); - private final RegionServerMetrics metrics = - new RegionServerMetrics(); + private final ServerName fakeMasterAddress = new ServerName("localhost", 60010, 1212121212); @@ -71,8 +70,6 @@ public class TestRSStatusServlet { .when(rs).getConfiguration(); Mockito.doReturn(fakeResponse).when(rs).getServerInfo( (RpcController)Mockito.any(), (GetServerInfoRequest)Mockito.any()); - Mockito.doReturn(metrics).when(rs).getMetrics(); - // Fake ZKW ZooKeeperWatcher zkw = Mockito.mock(ZooKeeperWatcher.class); Mockito.doReturn("fakequorum").when(zkw).getQuorum(); @@ -82,6 +79,10 @@ public class TestRSStatusServlet { MasterAddressTracker mat = Mockito.mock(MasterAddressTracker.class); Mockito.doReturn(fakeMasterAddress).when(mat).getMasterAddress(); Mockito.doReturn(mat).when(rs).getMasterAddressManager(); + + RegionServerMetrics rms = Mockito.mock(RegionServerMetrics.class); + Mockito.doReturn(new HRegionServerMetricsWrapperStub()).when(rms).getRsWrapper(); + Mockito.doReturn(rms).when(rs).getMetrics(); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index 822f5b2..2df3244 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -1,393 +1,182 @@ -/* - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hbase.regionserver; -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; -import java.util.Set; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CompatibilityFactory; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.MediumTests; -import org.apache.hadoop.hbase.client.Append; -import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.regionserver.metrics.RegionMetricsStorage; +import org.apache.hadoop.hbase.regionserver.metrics.RegionServerGeneralMetricsSource; import org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics. - StoreMetricType; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; -import org.junit.After; -import org.junit.Before; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; - -/** - * Test metrics incremented on region server operations. - */ @Category(MediumTests.class) public class TestRegionServerMetrics { + private static final Log LOG = LogFactory.getLog(TestRegionServerMetrics.class); + private static MetricsAssertHelper metricsHelper; - private static final Log LOG = - LogFactory.getLog(TestRegionServerMetrics.class.getName()); - - private final static String TABLE_NAME = - TestRegionServerMetrics.class.getSimpleName() + "Table"; - private String[] FAMILIES = new String[] { "cf1", "cf2", "anotherCF" }; - private static final int MAX_VERSIONS = 1; - private static final int NUM_COLS_PER_ROW = 15; - private static final int NUM_FLUSHES = 3; - private static final int NUM_REGIONS = 4; - - private static final SchemaMetrics ALL_METRICS = - SchemaMetrics.ALL_SCHEMA_METRICS; + static { + Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG); + } - private final HBaseTestingUtility TEST_UTIL = - new HBaseTestingUtility(); + private static MiniHBaseCluster cluster; + private static HRegionServer rs; + private static Configuration conf; + private static HBaseTestingUtility TEST_UTIL; + private static RegionServerMetrics regionServerMetrics; + private static RegionServerGeneralMetricsSource sourceGeneral; - private Map startingMetrics; - private final int META_AND_ROOT = 2; + @BeforeClass + public static void startCluster() throws Exception { + metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class); + TEST_UTIL = new HBaseTestingUtility(); + conf = TEST_UTIL.getConfiguration(); + conf.getLong("hbase.splitlog.max.resubmit", 0); + // Make the failure test faster + conf.setInt("zookeeper.recovery.retry", 0); + conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1); - @Before - public void setUp() throws Exception { - SchemaMetrics.setUseTableNameInTest(true); - startingMetrics = SchemaMetrics.getMetricsSnapshot(); - TEST_UTIL.startMiniCluster(); - } + TEST_UTIL.startMiniCluster(1, 1); + cluster = TEST_UTIL.getHBaseCluster(); - @After - public void tearDown() throws Exception { - TEST_UTIL.shutdownMiniCluster(); - SchemaMetrics.validateMetricChanges(startingMetrics); - } + cluster.waitForActiveAndReadyMaster(); - private void assertTimeVaryingMetricCount(int expectedCount, String table, String cf, - String regionName, String metricPrefix) { - - Integer expectedCountInteger = new Integer(expectedCount); - - if (cf != null) { - String cfKey = - SchemaMetrics.TABLE_PREFIX + table + "." + - SchemaMetrics.CF_PREFIX + cf + "." + metricPrefix; - Pair cfPair = RegionMetricsStorage.getTimeVaryingMetric(cfKey); - assertEquals(expectedCountInteger, cfPair.getSecond()); + while (cluster.getLiveRegionServerThreads().size() < 1) { + Threads.sleep(100); } - if (regionName != null) { - String rKey = - SchemaMetrics.TABLE_PREFIX + table + "." + - SchemaMetrics.REGION_PREFIX + regionName + "." + metricPrefix; + rs = cluster.getRegionServer(0); + regionServerMetrics = rs.getMetrics(); + sourceGeneral = regionServerMetrics.getMetricsSource(); + } - Pair regionPair = RegionMetricsStorage.getTimeVaryingMetric(rKey); - assertEquals(expectedCountInteger, regionPair.getSecond()); + @AfterClass + public static void after() throws Exception { + if (TEST_UTIL != null) { + TEST_UTIL.shutdownMiniCluster(); } } - - private void assertStoreMetricEquals(long expected, - SchemaMetrics schemaMetrics, StoreMetricType storeMetricType) { - final String storeMetricName = - schemaMetrics.getStoreMetricName(storeMetricType); - Long startValue = startingMetrics.get(storeMetricName); - assertEquals("Invalid value for store metric " + storeMetricName - + " (type " + storeMetricType + ")", expected, - RegionMetricsStorage.getNumericMetric(storeMetricName) - - (startValue != null ? startValue : 0)); - } - - @Test - public void testOperationMetrics() throws IOException { - String cf = "OPCF"; - String otherCf = "otherCF"; - String rk = "testRK"; - String icvCol = "icvCol"; - String appendCol = "appendCol"; - String regionName = null; - HTable hTable = - TEST_UTIL.createTable(TABLE_NAME.getBytes(), - new byte[][] { cf.getBytes(), otherCf.getBytes() }); - Set regionInfos = hTable.getRegionLocations().keySet(); - - regionName = regionInfos.toArray(new HRegionInfo[regionInfos.size()])[0].getEncodedName(); - - //Do a multi put that has one cf. Since they are in different rk's - //The lock will still be obtained and everything will be applied in one multiput. - Put pOne = new Put(rk.getBytes()); - pOne.add(cf.getBytes(), icvCol.getBytes(), Bytes.toBytes(0L)); - Put pTwo = new Put("ignored1RK".getBytes()); - pTwo.add(cf.getBytes(), "ignored".getBytes(), Bytes.toBytes(0L)); - - hTable.put(Arrays.asList(new Put[] {pOne, pTwo})); - - // Do a multiput where the cf doesn't stay consistent. - Put pThree = new Put("ignored2RK".getBytes()); - pThree.add(cf.getBytes(), "ignored".getBytes(), Bytes.toBytes("TEST1")); - Put pFour = new Put("ignored3RK".getBytes()); - pFour.add(otherCf.getBytes(), "ignored".getBytes(), Bytes.toBytes(0L)); - - hTable.put(Arrays.asList(new Put[] { pThree, pFour })); - - hTable.incrementColumnValue(rk.getBytes(), cf.getBytes(), icvCol.getBytes(), 1L); - Get g = new Get(rk.getBytes()); - g.addColumn(cf.getBytes(), appendCol.getBytes()); - hTable.get(g); + @Test(timeout = 300000) + public void testRegionCount() throws Exception { + String regionMetricsKey = "regionCount"; - Append a = new Append(rk.getBytes()); - a.add(cf.getBytes(), appendCol.getBytes(), Bytes.toBytes("-APPEND")); - hTable.append(a); + long regions = metricsHelper.getGaugeLong(regionMetricsKey, sourceGeneral); + // Creating a table should add one region + TEST_UTIL.createTable(Bytes.toBytes("table"), Bytes.toBytes("cf")); - Delete dOne = new Delete(rk.getBytes()); - dOne.deleteFamily(cf.getBytes()); - hTable.delete(dOne); - - Delete dTwo = new Delete(rk.getBytes()); - hTable.delete(dTwo); - - // There should be one multi put where the cf is consistent - assertTimeVaryingMetricCount(1, TABLE_NAME, cf, null, "multiput_"); + metricsHelper.assertGaugeGt(regionMetricsKey, regions, sourceGeneral); + } - // There were two multiputs to the cf. - assertTimeVaryingMetricCount(2, TABLE_NAME, null, regionName, "multiput_"); + @Test + public void testRequestCount() throws Exception { + byte[] tName = Bytes.toBytes("testRequestCount"); + byte[] cfName = Bytes.toBytes("d"); + byte[] row = Bytes.toBytes("rk"); + byte[] qualifier = Bytes.toBytes("qual"); + byte[] initValue = Bytes.toBytes("Value"); + byte[] nextValue = Bytes.toBytes("NEXT VAL"); - // There was one multiput where the cf was not consistent. - assertTimeVaryingMetricCount(1, TABLE_NAME, "__unknown", null, "multiput_"); - // One increment and one append - assertTimeVaryingMetricCount(1, TABLE_NAME, cf, regionName, "increment_"); - assertTimeVaryingMetricCount(1, TABLE_NAME, cf, regionName, "append_"); + TEST_UTIL.createTable(tName, cfName); - // One delete where the cf is known - assertTimeVaryingMetricCount(1, TABLE_NAME, cf, null, "multidelete_"); + new HTable(conf, tName).close(); //wait for the table to come up. + regionServerMetrics.getRsWrapper().forceRecompute(); + long requests = metricsHelper.getCounter("totalRequestCount", sourceGeneral); + long readRequests = metricsHelper.getCounter("readRequestCount", sourceGeneral); + long writeRequests = metricsHelper.getCounter("writeRequestCount", sourceGeneral); - // two deletes in the region. - assertTimeVaryingMetricCount(2, TABLE_NAME, null, regionName, "multidelete_"); + HTable table = new HTable(conf, tName); - // Three gets. one for gets. One for append. One for increment. - assertTimeVaryingMetricCount(3, TABLE_NAME, cf, regionName, "get_"); + Put p = new Put(row); - } - private void assertCheckAndMutateMetrics(final HRegionServer rs, - long expectedPassed, long expectedFailed) { - rs.doMetrics(); - RegionServerMetrics metrics = rs.getMetrics(); - assertEquals("checkAndMutatePassed metrics incorrect", - expectedPassed, metrics.checkAndMutateChecksPassed.get()); - assertEquals("checkAndMutateFailed metrics incorrect", - expectedFailed, metrics.checkAndMutateChecksFailed.get()); - } + p.add(cfName, qualifier, initValue); - @Test - public void testCheckAndMutateMetrics() throws Exception { - final HRegionServer rs = - TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); - byte [] tableName = Bytes.toBytes("testCheckAndMutateMetrics"); - byte [] family = Bytes.toBytes("family"); - byte [] qualifier = Bytes.toBytes("qualifier"); - byte [] row = Bytes.toBytes("row1"); - HTable table = TEST_UTIL.createTable(tableName, family); - long expectedPassed = 0; - long expectedFailed = 0; - - // checkAndPut success - Put put = new Put(row); - byte [] val1 = Bytes.toBytes("val1"); - put.add(family, qualifier, val1); - table.checkAndPut(row, family, qualifier, null, put); - expectedPassed++; - assertCheckAndMutateMetrics(rs, expectedPassed, expectedFailed); - - // checkAndPut failure - byte [] val2 = Bytes.toBytes("val2"); - table.checkAndPut(row, family, qualifier, val2, put); - expectedFailed++; - assertCheckAndMutateMetrics(rs, expectedPassed, expectedFailed); - - // checkAndDelete success - Delete delete = new Delete(row); - delete.deleteColumn(family, qualifier); - table.checkAndDelete(row, family, qualifier, val1, delete); - expectedPassed++; - assertCheckAndMutateMetrics(rs, expectedPassed, expectedFailed); - - // checkAndDelete failure - table.checkAndDelete(row, family, qualifier, val1, delete); - expectedFailed++; - assertCheckAndMutateMetrics(rs, expectedPassed, expectedFailed); - } - - @Test - public void testRemoveRegionMetrics() throws IOException, InterruptedException { - String cf = "REMOVECF"; - HTable hTable = TEST_UTIL.createTable(TABLE_NAME.getBytes(), cf.getBytes()); - HRegionInfo[] regionInfos = - hTable.getRegionLocations().keySet() - .toArray(new HRegionInfo[hTable.getRegionLocations().keySet().size()]); - - String regionName = regionInfos[0].getEncodedName(); - - // Do some operations so there are metrics. - Put pOne = new Put("TEST".getBytes()); - pOne.add(cf.getBytes(), "test".getBytes(), "test".getBytes()); - hTable.put(pOne); - - Get g = new Get("TEST".getBytes()); - g.addFamily(cf.getBytes()); - hTable.get(g); - assertTimeVaryingMetricCount(1, TABLE_NAME, cf, regionName, "get_"); - HBaseAdmin admin = TEST_UTIL.getHBaseAdmin(); - admin.disableTable(TABLE_NAME.getBytes()); - admin.deleteTable(TABLE_NAME.getBytes()); - - assertTimeVaryingMetricCount(0, TABLE_NAME, cf, regionName, "get_"); - } - - @Test - public void testMultipleRegions() throws IOException, InterruptedException { - - TEST_UTIL.createRandomTable( - TABLE_NAME, - Arrays.asList(FAMILIES), - MAX_VERSIONS, NUM_COLS_PER_ROW, NUM_FLUSHES, NUM_REGIONS, 1000); - - final HRegionServer rs = - TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); - - assertEquals(NUM_REGIONS + META_AND_ROOT, ProtobufUtil.getOnlineRegions(rs).size()); - - rs.doMetrics(); - for (HRegion r : TEST_UTIL.getMiniHBaseCluster().getRegions( - Bytes.toBytes(TABLE_NAME))) { - for (Map.Entry storeEntry : r.getStores().entrySet()) { - LOG.info("For region " + r.getRegionNameAsString() + ", CF " + - Bytes.toStringBinary(storeEntry.getKey()) + " found store files " + - ": " + storeEntry.getValue().getStorefiles()); - } + for (int i=0; i< 30; i++) { + table.put(p); } - assertStoreMetricEquals(NUM_FLUSHES * NUM_REGIONS * FAMILIES.length - + META_AND_ROOT, ALL_METRICS, StoreMetricType.STORE_FILE_COUNT); - for (String cf : FAMILIES) { - SchemaMetrics schemaMetrics = SchemaMetrics.getInstance(TABLE_NAME, cf); - assertStoreMetricEquals(NUM_FLUSHES * NUM_REGIONS, schemaMetrics, - StoreMetricType.STORE_FILE_COUNT); - } + table.flushCommits(); - // ensure that the max value is also maintained - final String storeMetricName = ALL_METRICS - .getStoreMetricNameMax(StoreMetricType.STORE_FILE_COUNT); - assertEquals("Invalid value for store metric " + storeMetricName, - NUM_FLUSHES, RegionMetricsStorage.getNumericMetric(storeMetricName)); - } + Get g = new Get(row); + for (int i=0; i< 10; i++) { + table.get(g); + } + Put pTwo = new Put(row); + pTwo.add(cfName, qualifier, nextValue); + table.checkAndPut(row, cfName, qualifier, initValue, pTwo); + table.checkAndPut(row, cfName, qualifier, initValue, pTwo); - private void assertSizeMetric(String table, String[] cfs, int[] metrics) { - // we have getsize & nextsize for each column family - assertEquals(cfs.length * 2, metrics.length); - for (int i =0; i < cfs.length; ++i) { - String prefix = SchemaMetrics.generateSchemaMetricsPrefix(table, cfs[i]); - String getMetric = prefix + SchemaMetrics.METRIC_GETSIZE; - String nextMetric = prefix + SchemaMetrics.METRIC_NEXTSIZE; - // verify getsize and nextsize matches - int getSize = RegionMetricsStorage.getNumericMetrics().containsKey(getMetric) ? - RegionMetricsStorage.getNumericMetrics().get(getMetric).intValue() : 0; - int nextSize = RegionMetricsStorage.getNumericMetrics().containsKey(nextMetric) ? - RegionMetricsStorage.getNumericMetrics().get(nextMetric).intValue() : 0; + regionServerMetrics.getRsWrapper().forceRecompute(); + metricsHelper.assertCounterGt("totalRequestCount", requests + 39, sourceGeneral); + metricsHelper.assertCounter("readRequestCount", readRequests + 12, sourceGeneral); + metricsHelper.assertCounter("writeRequestCount", writeRequests + 31, sourceGeneral); + metricsHelper.assertCounter("checkMutatePassedCount", 1, sourceGeneral); + metricsHelper.assertCounter("checkMutateFailedCount", 1, sourceGeneral); - assertEquals(metrics[i], getSize); - assertEquals(metrics[cfs.length + i], nextSize); - } } @Test - public void testGetNextSize() throws IOException, InterruptedException { - String rowName = "row1"; - byte[] ROW = Bytes.toBytes(rowName); - String tableName = "SizeMetricTest"; - byte[] TABLE = Bytes.toBytes(tableName); - String cf1Name = "cf1"; - String cf2Name = "cf2"; - String[] cfs = new String[] {cf1Name, cf2Name}; - byte[] CF1 = Bytes.toBytes(cf1Name); - byte[] CF2 = Bytes.toBytes(cf2Name); - - long ts = 1234; - HTable hTable = TEST_UTIL.createTable(TABLE, new byte[][]{CF1, CF2}); - HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); - - Put p = new Put(ROW); - p.add(CF1, CF1, ts, CF1); - p.add(CF2, CF2, ts, CF2); - hTable.put(p); - - KeyValue kv1 = new KeyValue(ROW, CF1, CF1, ts, CF1); - KeyValue kv2 = new KeyValue(ROW, CF2, CF2, ts, CF2); - int kvLength = kv1.getLength(); - assertEquals(kvLength, kv2.getLength()); - - // only cf1.getsize is set on Get - hTable.get(new Get(ROW).addFamily(CF1)); - assertSizeMetric(tableName, cfs, new int[] {kvLength, 0, 0, 0}); - - // only cf2.getsize is set on Get - hTable.get(new Get(ROW).addFamily(CF2)); - assertSizeMetric(tableName, cfs, new int[] {kvLength, kvLength, 0, 0}); - - // only cf2.nextsize is set - for (Result res : hTable.getScanner(CF2)) { - } - assertSizeMetric(tableName, cfs, - new int[] {kvLength, kvLength, 0, kvLength}); - - // only cf2.nextsize is set - for (Result res : hTable.getScanner(CF1)) { - } - assertSizeMetric(tableName, cfs, - new int[] {kvLength, kvLength, kvLength, kvLength}); - - // getsize/nextsize should not be set on flush or compaction - for (HRegion hr : TEST_UTIL.getMiniHBaseCluster().getRegions(TABLE)) { - hr.flushcache(); - hr.compactStores(); - } - assertSizeMetric(tableName, cfs, - new int[] {kvLength, kvLength, kvLength, kvLength}); + public void testStoreCount() throws Exception { + byte[] tableName = Bytes.toBytes("testStoreCount"); + byte[] cf = Bytes.toBytes("d"); + byte[] row = Bytes.toBytes("rk"); + byte[] qualifier = Bytes.toBytes("qual"); + byte[] val = Bytes.toBytes("Value"); + + regionServerMetrics.getRsWrapper().forceRecompute(); + long stores = metricsHelper.getGaugeLong("storeCount", sourceGeneral); + long storeFiles = metricsHelper.getGaugeLong("storeFileCount", sourceGeneral); + + TEST_UTIL.createTable(tableName, cf); + + //Force a hfile. + HTable t = new HTable(conf, tableName); + Put p = new Put(row); + p.add(cf, qualifier, val); + t.put(p); + t.flushCommits(); + TEST_UTIL.getHBaseAdmin().flush(tableName); + + regionServerMetrics.getRsWrapper().forceRecompute(); + metricsHelper.assertGauge("storeCount", stores +1, sourceGeneral); + metricsHelper.assertGauge("storeFileCount", storeFiles + 1, sourceGeneral); } } - diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java index 3f07e32..e7481d3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java @@ -50,7 +50,6 @@ import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl; import org.apache.hadoop.hbase.io.hfile.HFileScanner; import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder; import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.util.BloomFilterFactory; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; @@ -72,22 +71,17 @@ public class TestStoreFile extends HBaseTestCase { private CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration()); private static String ROOT_DIR = TEST_UTIL.getDataTestDir("TestStoreFile").toString(); - private Map startingMetrics; - private static final ChecksumType CKTYPE = ChecksumType.CRC32; private static final int CKBYTES = 512; @Override public void setUp() throws Exception { super.setUp(); - this.startingMetrics = SchemaMetrics.getMetricsSnapshot(); } @Override public void tearDown() throws Exception { super.tearDown(); - LOG.info("Verifying metrics for " + getName() + ": " + this.startingMetrics); - SchemaMetrics.validateMetricChanges(this.startingMetrics); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java index 5e1e5d3..d7910a0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.KeyValueTestUtil; import org.apache.hadoop.hbase.MediumTests; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.regionserver.HStore.ScanInfo; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdge; import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; @@ -54,7 +53,6 @@ public class TestStoreScanner extends TestCase { public void setUp() throws Exception { super.setUp(); - SchemaMetrics.setUseTableNameInTest(false); } /* diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestRegionServerGeneralMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestRegionServerGeneralMetrics.java new file mode 100644 index 0000000..9bd3a69 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestRegionServerGeneralMetrics.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.metrics; + +import org.apache.hadoop.hbase.CompatibilityFactory; +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.regionserver.HRegionServerMetricsWrapperStub; +import org.apache.hadoop.hbase.test.MetricsAssertHelper; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertNotNull; + +/** + * Unit test version of rs metrics tests. + */ +@Category(SmallTests.class) +public class TestRegionServerGeneralMetrics { + + public MetricsAssertHelper HELPER = CompatibilityFactory.getInstance(MetricsAssertHelper.class); + + @Test + public void testWrapperSource() { + + RegionServerMetrics rsm = new RegionServerMetrics(new HRegionServerMetricsWrapperStub()); + RegionServerGeneralMetricsSource source = rsm.getMetricsSource(); + HELPER.assertTag("serverName", "test", source); + HELPER.assertTag("clusterId", "tClusterId", source); + HELPER.assertTag("zookeeperQuorum", "zk", source); + HELPER.assertGauge("regionServerStartTime", 100, source); + HELPER.assertGauge("regionCount", 101, source); + HELPER.assertGauge("storeCount", 2, source); + HELPER.assertGauge("storeFileCount", 300, source); + HELPER.assertGauge("memstoreSize", 1025, source); + HELPER.assertGauge("storeFileSize", 1900, source); + HELPER.assertCounter("totalRequestCount", 899, source); + HELPER.assertCounter("readRequestCount", 997, source); + HELPER.assertCounter("writeRequestCount", 707, source); + HELPER.assertCounter("checkMutateFailedCount", 401, source); + HELPER.assertCounter("checkMutatePassedCount", 405, source); + HELPER.assertGauge("storeFileIndexSize", 406, source); + HELPER.assertGauge("staticIndexSize", 407, source); + HELPER.assertGauge("staticBloomSize", 408, source); + HELPER.assertCounter("putsWithoutWALCount", 409, source); + HELPER.assertGauge("putsWithoutWALSize", 410, source); + HELPER.assertGauge("percentFilesLocal", 99, source); + HELPER.assertGauge("compactionQueueLength", 411, source); + HELPER.assertGauge("flushQueueLength", 412, source); + HELPER.assertGauge("blockCacheFreeSize", 413, source); + HELPER.assertGauge("blockCacheCount", 414, source); + HELPER.assertGauge("blockCacheSize", 415, source); + HELPER.assertCounter("blockCacheHitCount", 416, source); + HELPER.assertCounter("blockCacheMissCount", 417, source); + HELPER.assertCounter("blockCacheEvictionCount", 418, source); + HELPER.assertGauge("blockCountHitPercent", 98, source); + HELPER.assertGauge("blockCacheExpressHitPercent", 97, source); + } + + @Test + public void testConstuctor() { + RegionServerMetrics rsm = new RegionServerMetrics(new HRegionServerMetricsWrapperStub()); + assertNotNull("There should be a hadoop1/hadoop2 metrics source", rsm.getMetricsSource() ); + assertNotNull("The RegionServerMetricsWrapper should be accessable", rsm.getRsWrapper()); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaConfigured.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaConfigured.java deleted file mode 100644 index 0ef0775..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaConfigured.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hbase.regionserver.metrics; - -import static org.junit.Assert.*; - -import java.util.regex.Pattern; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.SmallTests; -import org.apache.hadoop.hbase.regionserver.HRegion; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONStringer; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(SmallTests.class) -public class TestSchemaConfigured { - private static final Log LOG = LogFactory.getLog(TestSchemaConfigured.class); - private final String TABLE_NAME = "myTable"; - private final String CF_NAME = "myColumnFamily"; - - private static final Path TMP_HFILE_PATH = new Path( - "/hbase/myTable/myRegion/" + HRegion.REGION_TEMP_SUBDIR + "/hfilename"); - - /** Test if toString generates real JSON */ - @Test - public void testToString() throws JSONException { - SchemaConfigured sc = new SchemaConfigured(null, TABLE_NAME, CF_NAME); - JSONStringer json = new JSONStringer(); - json.object(); - json.key("tableName"); - json.value(TABLE_NAME); - json.key("cfName"); - json.value(CF_NAME); - json.endObject(); - assertEquals(json.toString(), sc.schemaConfAsJSON()); - } - - /** Don't allow requesting metrics before setting table/CF name */ - @Test - public void testDelayedInitialization() { - SchemaConfigured unconfigured = new SchemaConfigured(); - try { - unconfigured.getSchemaMetrics(); - fail(IllegalStateException.class.getSimpleName() + " expected"); - } catch (IllegalStateException ex) { - assertTrue("Unexpected exception message: " + ex.getMessage(), - Pattern.matches(".* metrics requested before .* initialization.*", - ex.getMessage())); - LOG.debug("Expected exception: " + ex.getMessage()); - } - - SchemaMetrics.setUseTableNameInTest(false); - SchemaConfigured other = new SchemaConfigured(null, TABLE_NAME, CF_NAME); - other.passSchemaMetricsTo(unconfigured); - unconfigured.getSchemaMetrics(); // now this should succeed - } - - /** Don't allow setting table/CF name twice */ - @Test - public void testInitializingTwice() { - Configuration conf = HBaseConfiguration.create(); - for (int i = 0; i < 4; ++i) { - SchemaConfigured sc = new SchemaConfigured(conf, TABLE_NAME, CF_NAME); - SchemaConfigured target = - new SchemaConfigured(conf, TABLE_NAME + (i % 2 == 1 ? "1" : ""), - CF_NAME + ((i & 2) != 0 ? "1" : "")); - if (i == 0) { - sc.passSchemaMetricsTo(target); // No exception expected. - continue; - } - - String testDesc = - "Trying to re-configure " + target.schemaConfAsJSON() + " with " - + sc.schemaConfAsJSON(); - try { - sc.passSchemaMetricsTo(target); - fail(IllegalArgumentException.class.getSimpleName() + " expected"); - } catch (IllegalArgumentException ex) { - final String errorMsg = testDesc + ". Unexpected exception message: " + - ex.getMessage(); - final String exceptionRegex = "Trying to change table .* CF .*"; - assertTrue(errorMsg, Pattern.matches(exceptionRegex, ex.getMessage())); - LOG.debug("Expected exception: " + ex.getMessage()); - } - } - } - - @Test(expected=IllegalStateException.class) - public void testConfigureWithUnconfigured1() { - SchemaConfigured unconfigured = new SchemaConfigured(null, "t1", null); - SchemaConfigured target = new SchemaConfigured(); - unconfigured.passSchemaMetricsTo(target); - } - - @Test(expected=IllegalStateException.class) - public void testConfigureWithUnconfigured2() { - SchemaConfigured unconfigured = new SchemaConfigured(null, null, "cf1"); - SchemaConfigured target = new SchemaConfigured(); - unconfigured.passSchemaMetricsTo(target); - } - - /** - * Configuring with an uninitialized object is equivalent to re-setting - * schema metrics configuration. - */ - public void testConfigureWithNull() { - SchemaConfigured unconfigured = new SchemaConfigured(); - SchemaConfigured target = new SchemaConfigured(null, "t1", "cf1"); - unconfigured.passSchemaMetricsTo(target); - assertTrue(target.getTableName() == null); - assertTrue(target.getColumnFamilyName() == null); - } - - public void testConfigurePartiallyDefined() { - final SchemaConfigured sc = new SchemaConfigured(null, "t1", "cf1"); - final SchemaConfigured target1 = new SchemaConfigured(null, "t2", null); - sc.passSchemaMetricsTo(target1); - assertEquals("t2", target1.getColumnFamilyName()); - assertEquals("cf1", target1.getColumnFamilyName()); - - final SchemaConfigured target2 = new SchemaConfigured(null, null, "cf2"); - sc.passSchemaMetricsTo(target2); - assertEquals("t1", target2.getColumnFamilyName()); - assertEquals("cf2", target2.getColumnFamilyName()); - - final SchemaConfigured target3 = new SchemaConfigured(null, null, null); - sc.passSchemaMetricsTo(target3); - assertEquals("t1", target2.getColumnFamilyName()); - assertEquals("cf1", target2.getColumnFamilyName()); - } - - @Test(expected=IllegalArgumentException.class) - public void testConflictingConf() { - SchemaConfigured sc = new SchemaConfigured(null, "t1", "cf1"); - SchemaConfigured target = new SchemaConfigured(null, "t2", "cf1"); - sc.passSchemaMetricsTo(target); - } - - /** We allow setting CF to unknown and then reconfiguring it */ - public void testReconfigureUnknownCF() { - SchemaConfigured sc = new SchemaConfigured(null, "t1", "cf1"); - SchemaConfigured target = - new SchemaConfigured(null, "t1", SchemaMetrics.UNKNOWN); - sc.passSchemaMetricsTo(target); - } - - /** - * When the "column family" deduced from the path is ".tmp" (this happens - * for files written on compaction) we allow re-setting the CF to another - * value. - */ - @Test - public void testTmpPath() { - SchemaConfigured sc = new SchemaConfigured(null, "myTable", "myCF"); - SchemaConfigured target = new SchemaConfigured(TMP_HFILE_PATH); - sc.passSchemaMetricsTo(target); - } - - /** - * Even if CF is initially undefined (".tmp"), we don't allow to change - * table name. - */ - @Test(expected=IllegalArgumentException.class) - public void testTmpPathButInvalidTable() { - SchemaConfigured sc = new SchemaConfigured(null, "anotherTable", "myCF"); - SchemaConfigured target = new SchemaConfigured(TMP_HFILE_PATH); - sc.passSchemaMetricsTo(target); - } - - @Test - public void testSchemaConfigurationHook() { - SchemaConfigured sc = new SchemaConfigured(null, "myTable", "myCF"); - final StringBuilder newCF = new StringBuilder(); - final StringBuilder newTable = new StringBuilder(); - SchemaConfigured target = new SchemaConfigured() { - @Override - protected void schemaConfigurationChanged() { - newCF.append(getColumnFamilyName()); - newTable.append(getTableName()); - } - }; - sc.passSchemaMetricsTo(target); - assertEquals("myTable", newTable.toString()); - assertEquals("myCF", newCF.toString()); - } - - @Test - public void testResetSchemaMetricsConf() { - SchemaConfigured target = new SchemaConfigured(null, "t1", "cf1"); - SchemaConfigured.resetSchemaMetricsConf(target); - new SchemaConfigured(null, "t2", "cf2").passSchemaMetricsTo(target); - assertEquals("t2", target.getTableName()); - assertEquals("cf2", target.getColumnFamilyName()); - } - - @Test - public void testPathTooShort() { - // This has too few path components (four, the first one is empty). - SchemaConfigured sc1 = new SchemaConfigured(new Path("/a/b/c/d")); - assertEquals(SchemaMetrics.UNKNOWN, sc1.getTableName()); - assertEquals(SchemaMetrics.UNKNOWN, sc1.getColumnFamilyName()); - - SchemaConfigured sc2 = new SchemaConfigured(new Path("a/b/c/d")); - assertEquals(SchemaMetrics.UNKNOWN, sc2.getTableName()); - assertEquals(SchemaMetrics.UNKNOWN, sc2.getColumnFamilyName()); - - SchemaConfigured sc3 = new SchemaConfigured( - new Path("/hbase/tableName/regionId/cfName/hfileName")); - assertEquals("tableName", sc3.getTableName()); - assertEquals("cfName", sc3.getColumnFamilyName()); - - SchemaConfigured sc4 = new SchemaConfigured( - new Path("hbase/tableName/regionId/cfName/hfileName")); - assertEquals("tableName", sc4.getTableName()); - assertEquals("cfName", sc4.getColumnFamilyName()); - } - - -} - diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java deleted file mode 100644 index e89ac9e..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/metrics/TestSchemaMetrics.java +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hbase.regionserver.metrics; - -import static org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics. - BOOL_VALUES; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -import java.util.Collection; -import java.util.HashSet; -import java.util.Map; -import java.util.Random; -import java.util.Set; - -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.MediumTests; -import org.apache.hadoop.hbase.io.hfile.BlockType; -import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; -import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics. - BlockMetricType; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.ClassSize; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; - -@Category(MediumTests.class) -@RunWith(Parameterized.class) -public class TestSchemaMetrics { - - private final String TABLE_NAME = "myTable"; - private final String CF_NAME = "myColumnFamily"; - - private final boolean useTableName; - private Map startingMetrics; - - @Parameters - public static Collection parameters() { - return HBaseTestingUtility.BOOLEAN_PARAMETERIZED; - } - - public TestSchemaMetrics(boolean useTableName) { - this.useTableName = useTableName; - SchemaMetrics.setUseTableNameInTest(useTableName); - } - - @Before - public void setUp() { - startingMetrics = SchemaMetrics.getMetricsSnapshot(); - }; - - @Test - public void testNaming() { - final String metricPrefix = (useTableName ? "tbl." + - TABLE_NAME + "." : "") + "cf." + CF_NAME + "."; - SchemaMetrics schemaMetrics = SchemaMetrics.getInstance(TABLE_NAME, - CF_NAME); - SchemaMetrics ALL_CF_METRICS = SchemaMetrics.ALL_SCHEMA_METRICS; - - // fsReadTimeMetric - assertEquals(metricPrefix + "fsRead", schemaMetrics.getBlockMetricName( - BlockCategory.ALL_CATEGORIES, false, BlockMetricType.READ_TIME)); - - // compactionReadTimeMetric - assertEquals(metricPrefix + "compactionRead", - schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, true, - BlockMetricType.READ_TIME)); - - // fsBlockReadCntMetric - assertEquals(metricPrefix + "fsBlockReadCnt", - schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, false, - BlockMetricType.READ_COUNT)); - - // fsBlockReadCacheHitCntMetric - assertEquals(metricPrefix + "fsBlockReadCacheHitCnt", - schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, false, - BlockMetricType.CACHE_HIT)); - - // fsBlockReadCacheMissCntMetric - assertEquals(metricPrefix + "fsBlockReadCacheMissCnt", - schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, false, - BlockMetricType.CACHE_MISS)); - - // compactionBlockReadCntMetric - assertEquals(metricPrefix + "compactionBlockReadCnt", - schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, true, - BlockMetricType.READ_COUNT)); - - // compactionBlockReadCacheHitCntMetric - assertEquals(metricPrefix + "compactionBlockReadCacheHitCnt", - schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, true, - BlockMetricType.CACHE_HIT)); - - // compactionBlockReadCacheMissCntMetric - assertEquals(metricPrefix + "compactionBlockReadCacheMissCnt", - schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, true, - BlockMetricType.CACHE_MISS)); - - // fsMetaBlockReadCntMetric - assertEquals("fsMetaBlockReadCnt", ALL_CF_METRICS.getBlockMetricName( - BlockCategory.META, false, BlockMetricType.READ_COUNT)); - - // fsMetaBlockReadCacheHitCntMetric - assertEquals("fsMetaBlockReadCacheHitCnt", - ALL_CF_METRICS.getBlockMetricName(BlockCategory.META, false, - BlockMetricType.CACHE_HIT)); - - // fsMetaBlockReadCacheMissCntMetric - assertEquals("fsMetaBlockReadCacheMissCnt", - ALL_CF_METRICS.getBlockMetricName(BlockCategory.META, false, - BlockMetricType.CACHE_MISS)); - - // Per-(column family, block type) statistics. - assertEquals(metricPrefix + "bt.Index.fsBlockReadCnt", - schemaMetrics.getBlockMetricName(BlockCategory.INDEX, false, - BlockMetricType.READ_COUNT)); - - assertEquals(metricPrefix + "bt.Data.compactionBlockReadCacheHitCnt", - schemaMetrics.getBlockMetricName(BlockCategory.DATA, true, - BlockMetricType.CACHE_HIT)); - - // A special case for Meta blocks - assertEquals(metricPrefix + "compactionMetaBlockReadCacheHitCnt", - schemaMetrics.getBlockMetricName(BlockCategory.META, true, - BlockMetricType.CACHE_HIT)); - - // Cache metrics - assertEquals(metricPrefix + "blockCacheSize", - schemaMetrics.getBlockMetricName(BlockCategory.ALL_CATEGORIES, false, - BlockMetricType.CACHE_SIZE)); - - assertEquals(metricPrefix + "bt.Index.blockCacheNumEvicted", - schemaMetrics.getBlockMetricName(BlockCategory.INDEX, false, - BlockMetricType.EVICTED)); - - assertEquals("bt.Data.blockCacheNumCached", - ALL_CF_METRICS.getBlockMetricName(BlockCategory.DATA, false, - BlockMetricType.CACHED)); - - assertEquals("blockCacheNumCached", ALL_CF_METRICS.getBlockMetricName( - BlockCategory.ALL_CATEGORIES, false, BlockMetricType.CACHED)); - - // "Non-compaction aware" metrics - try { - ALL_CF_METRICS.getBlockMetricName(BlockCategory.ALL_CATEGORIES, true, - BlockMetricType.CACHE_SIZE); - fail("Exception expected"); - } catch (IllegalArgumentException ex) { - } - - // Bloom metrics - assertEquals("keyMaybeInBloomCnt", ALL_CF_METRICS.getBloomMetricName(true)); - assertEquals(metricPrefix + "keyNotInBloomCnt", - schemaMetrics.getBloomMetricName(false)); - - schemaMetrics.printMetricNames(); - } - - public void checkMetrics() { - SchemaMetrics.validateMetricChanges(startingMetrics); - } - - @Test - public void testIncrements() { - Random rand = new Random(23982737L); - for (int i = 1; i <= 3; ++i) { - final String tableName = "table" + i; - for (int j = 1; j <= 3; ++j) { - final String cfName = "cf" + j; - SchemaMetrics sm = SchemaMetrics.getInstance(tableName, cfName); - for (boolean isInBloom : BOOL_VALUES) { - sm.updateBloomMetrics(isInBloom); - checkMetrics(); - } - - for (BlockCategory blockCat : BlockType.BlockCategory.values()) { - if (blockCat == BlockCategory.ALL_CATEGORIES) { - continue; - } - - for (boolean isCompaction : BOOL_VALUES) { - sm.updateOnCacheHit(blockCat, isCompaction); - checkMetrics(); - sm.updateOnCacheMiss(blockCat, isCompaction, rand.nextInt()); - checkMetrics(); - } - - for (boolean isEviction : BOOL_VALUES) { - sm.updateOnCachePutOrEvict(blockCat, (isEviction ? -1 : 1) - * rand.nextInt(1024 * 1024), isEviction); - } - } - } - } - } - - @Test - public void testGenerateSchemaMetricsPrefix() { - String tableName = "table1"; - int numCF = 3; - - StringBuilder expected = new StringBuilder(); - if (useTableName) { - expected.append("tbl."); - expected.append(tableName); - expected.append("."); - } - expected.append("cf."); - Set families = new HashSet(); - for (int i = 1; i <= numCF; i++) { - String cf = "cf" + i; - families.add(Bytes.toBytes(cf)); - expected.append(cf); - if (i == numCF) { - expected.append("."); - } else { - expected.append("~"); - } - } - - String result = SchemaMetrics.generateSchemaMetricsPrefix(tableName, - families); - assertEquals(expected.toString(), result); - } - - -} -