From 764d91add1e9bff0ece9215df675f78c8c39d0d3 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 12 Apr 2019 19:29:01 -0700 Subject: [PATCH] HBASE-8868 add metric to report client shortcircuit reads Expose file system level read metrics for RegionServer. If the HBase RS runs on top of HDFS, calculate the aggregation of ReadStatistics of each HdfsFileInputStream. These metrics include: (1) total number of bytes read from HDFS. (2) total number of bytes read from local DataNode. (3) total number of bytes read locally through short-circuit read. (4) total number of bytes read locally through zero-copy read. Because HDFS ReadStatistics is calculated per input stream, it is not feasible to update the aggregated number in real time. Instead, the metrics are updated when an input stream is closed. Change-Id: I3a5bb7fed5a6afe2ecb432e1082dd249b29aef8b --- .../regionserver/MetricsRegionServerSource.java | 11 +++++ .../regionserver/MetricsRegionServerWrapper.java | 20 ++++++++ .../MetricsRegionServerSourceImpl.java | 12 +++++ .../hadoop/hbase/io/FSDataInputStreamWrapper.java | 54 +++++++++++++++++++++- .../MetricsRegionServerWrapperImpl.java | 21 +++++++++ .../MetricsRegionServerWrapperStub.java | 20 ++++++++ .../regionserver/TestRegionServerMetrics.java | 19 ++++++++ 7 files changed, 156 insertions(+), 1 deletion(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 6d9ce54..e9f98ba 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -477,6 +477,17 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String HEDGED_READ_WINS_DESC = "The number of times we started a hedged read and a hedged read won"; + String TOTAL_BYTES_READ = "totalBytesRead"; + String TOTAL_BYTES_READ_DESC = "The total number of bytes read"; + String LOCAL_BYTES_READ = "localBytesRead"; + String LOCAL_BYTES_READ_DESC = + "The number of bytes read locally"; + String SHORTCIRCUIT_BYTES_READ = "shortCircuitBytesRead"; + String SHORTCIRCUIT_BYTES_READ_DESC = "The number of bytes read through short circuit read"; + String ZEROCOPY_BYTES_READ = "zeroCopyBytesRead"; + String ZEROCOPY_BYTES_READ_DESC = + "The number of bytes read through zero copy"; + String BLOCKED_REQUESTS_COUNT = "blockedRequestCount"; String BLOCKED_REQUESTS_COUNT_DESC = "The number of blocked requests because of memstore size is " + "larger than blockingMemStoreSize"; diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index 03ebc4c..a4b65d7 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -453,6 +453,26 @@ public interface MetricsRegionServerWrapper { long getHedgedReadWins(); /** + * @return Number of total bytes read + */ + long getTotalBytesRead(); + + /** + * @return Number of bytes read locally + */ + long getLocalBytesRead(); + + /** + * @return Number of bytes read locally through short circuit + */ + long getShortCircuitBytesRead(); + + /** + * @return Number of bytes read locally through zero copy + */ + long getZeroCopyBytesRead(); + + /** * @return Count of requests blocked because the memstore size is larger than blockingMemStoreSize */ long getBlockedRequestsCount(); diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index 58c42a5..ac0460e 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -510,6 +510,18 @@ public class MetricsRegionServerSourceImpl .addGauge(Interns.info(PERCENT_FILES_LOCAL_SECONDARY_REGIONS, PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC), rsWrap.getPercentFileLocalSecondaryRegions()) + .addGauge(Interns.info(TOTAL_BYTES_READ, + TOTAL_BYTES_READ_DESC), + rsWrap.getTotalBytesRead()) + .addGauge(Interns.info(LOCAL_BYTES_READ, + LOCAL_BYTES_READ_DESC), + rsWrap.getLocalBytesRead()) + .addGauge(Interns.info(SHORTCIRCUIT_BYTES_READ, + SHORTCIRCUIT_BYTES_READ_DESC), + rsWrap.getShortCircuitBytesRead()) + .addGauge(Interns.info(ZEROCOPY_BYTES_READ, + ZEROCOPY_BYTES_READ_DESC), + rsWrap.getZeroCopyBytesRead()) .addGauge(Interns.info(SPLIT_QUEUE_LENGTH, SPLIT_QUEUE_LENGTH_DESC), rsWrap.getSplitQueueSize()) .addGauge(Interns.info(COMPACTION_QUEUE_LENGTH, COMPACTION_QUEUE_LENGTH_DESC), diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java index 6c73405..0dc4f60 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/FSDataInputStreamWrapper.java @@ -29,6 +29,8 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hdfs.DFSInputStream; +import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,6 +96,15 @@ public class FSDataInputStreamWrapper implements Closeable { // errors against Hadoop pre 2.6.4 and 2.7.1 versions. private Method unbuffer = null; + private final static ReadStatistics readStatistics = new ReadStatistics(); + + private static class ReadStatistics { + long totalBytesRead; + long totalLocalBytesRead; + long totalShortCircuitBytesRead; + long totalZeroCopyBytesRead; + } + public FSDataInputStreamWrapper(FileSystem fs, Path path) throws IOException { this(fs, path, false, -1L); } @@ -232,14 +243,55 @@ public class FSDataInputStreamWrapper implements Closeable { } } - /** Close stream(s) if necessary. */ + private void updateInputStreamStatistics(FSDataInputStream stream) { + // If the underlying file system is HDFS, update read statistics upon close. + if (stream instanceof HdfsDataInputStream) { + DFSInputStream.ReadStatistics stat = ((HdfsDataInputStream)stream).getReadStatistics(); + synchronized (readStatistics) { + readStatistics.totalBytesRead += stat.getTotalBytesRead(); + readStatistics.totalLocalBytesRead += stat.getTotalBytesRead(); + readStatistics.totalShortCircuitBytesRead += stat.getTotalShortCircuitBytesRead(); + readStatistics.totalZeroCopyBytesRead += stat.getTotalZeroCopyBytesRead(); + } + } + } + + public static long getTotalBytesRead() { + synchronized (readStatistics) { + return readStatistics.totalBytesRead; + } + } + + public static long getLocalBytesRead() { + synchronized (readStatistics) { + return readStatistics.totalLocalBytesRead; + } + } + + public static long getShortCircuitBytesRead() { + synchronized (readStatistics) { + return readStatistics.totalShortCircuitBytesRead; + } + } + + public static long getZeroCopyBytesRead() { + synchronized (readStatistics) { + return readStatistics.totalZeroCopyBytesRead; + } + } + + /** CloseClose stream(s) if necessary. */ @Override public void close() { if (!doCloseStreams) { return; } + updateInputStreamStatistics(this.streamNoFsChecksum); // we do not care about the close exception as it is for reading, no data loss issue. IOUtils.closeQuietly(streamNoFsChecksum); + + + updateInputStreamStatistics(stream); IOUtils.closeQuietly(stream); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 33a6ee0..c33fd15 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.CacheStats; import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache; @@ -898,6 +899,26 @@ class MetricsRegionServerWrapperImpl } @Override + public long getTotalBytesRead() { + return FSDataInputStreamWrapper.getTotalBytesRead(); + } + + @Override + public long getLocalBytesRead() { + return FSDataInputStreamWrapper.getLocalBytesRead(); + } + + @Override + public long getShortCircuitBytesRead() { + return FSDataInputStreamWrapper.getShortCircuitBytesRead(); + } + + @Override + public long getZeroCopyBytesRead() { + return FSDataInputStreamWrapper.getZeroCopyBytesRead(); + } + + @Override public long getBlockedRequestsCount() { return blockedRequestsCount; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index b003b44..7811638 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -396,6 +396,26 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe } @Override + public long getTotalBytesRead() { + return 0; + } + + @Override + public long getLocalBytesRead() { + return 0; + } + + @Override + public long getShortCircuitBytesRead() { + return 0; + } + + @Override + public long getZeroCopyBytesRead() { + return 0; + } + + @Override public long getBlockedRequestsCount() { return 0; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index 3778c20..89168ee 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hbase.regionserver; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -608,4 +609,22 @@ public class TestRegionServerMetrics { metricsRegionServer.getRegionServerWrapper().forceRecompute(); assertTrue(metricsHelper.getGaugeDouble("averageRegionSize", serverSource) > 0.0); } + + @Test + public void testReadBytes() throws Exception { + // Do a first put to be sure that the connection is established, meta is there and so on. + doNPuts(1, false); + doNGets(10, false); + TEST_UTIL.getAdmin().flush(tableName); + metricsRegionServer.getRegionServerWrapper().forceRecompute(); + + assertTrue("Total read bytes should be larger than 0", + metricsRegionServer.getRegionServerWrapper().getTotalBytesRead() > 0); + assertTrue("Total local read bytes should be larger than 0", + metricsRegionServer.getRegionServerWrapper().getLocalBytesRead() > 0); + assertEquals("Total short circuit read bytes should be equal to 0", 0, + metricsRegionServer.getRegionServerWrapper().getShortCircuitBytesRead()); + assertEquals("Total zero-byte read bytes should be equal to 0", 0, + metricsRegionServer.getRegionServerWrapper().getZeroCopyBytesRead()); + } } -- 2.5.3