diff --git common/src/java/org/apache/hadoop/hive/common/DiskRangeList.java common/src/java/org/apache/hadoop/hive/common/DiskRangeList.java index 616c5ec..0133b35 100644 --- common/src/java/org/apache/hadoop/hive/common/DiskRangeList.java +++ common/src/java/org/apache/hadoop/hive/common/DiskRangeList.java @@ -111,6 +111,21 @@ public int listSize() { return result; } + @Override + public int getLength() { + return super.getLength(); + } + + public long getTotalLength() { + long totalLength = getLength(); + DiskRangeList current = next; + while (current != null) { + totalLength += current.getLength(); + current = current.next; + } + return totalLength; + } + @VisibleForTesting public DiskRangeList[] listToArray() { DiskRangeList[] result = new DiskRangeList[listSize()]; diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java index ca874cd..3933f46 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java @@ -79,9 +79,11 @@ public void allocateMultiple(LlapMemoryBuffer[] dest, int size) { @Override public DiskRangeList getFileData(long fileId, DiskRangeList ranges, long baseOffset) { if (ranges == null) return null; - metrics.incrCacheRequestedBytes(ranges.getLength()); FileCache subCache = cache.get(fileId); - if (subCache == null || !subCache.incRef()) return ranges; + if (subCache == null || !subCache.incRef()) { + metrics.incrCacheRequestedBytes(ranges.getTotalLength()); + return ranges; + } try { DiskRangeList prev = ranges.prev; if (prev == null) { @@ -89,6 +91,7 @@ public DiskRangeList getFileData(long fileId, DiskRangeList ranges, long baseOff } DiskRangeList current = ranges; while (current != null) { + metrics.incrCacheRequestedBytes(current.getLength()); // We assume ranges in "ranges" are non-overlapping; thus, we will save next in advance. DiskRangeList next = current.next; getOverlappingRanges(baseOffset, current, subCache.cache); diff --git llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java index 4453e92..3ffa6e0 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java @@ -39,6 +39,8 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import com.google.common.annotations.VisibleForTesting; + /** * Llap daemon cache metrics source. */ @@ -115,6 +117,16 @@ public String getName() { return name; } + @VisibleForTesting + public long getCacheRequestedBytes() { + return cacheRequestedBytes.value(); + } + + @VisibleForTesting + public long getCacheHitBytes() { + return cacheHitBytes.value(); + } + @Override public void getMetrics(MetricsCollector collector, boolean b) { MetricsRecordBuilder rb = collector.addRecord(CacheMetrics) diff --git llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java index ef09796..667767d 100644 --- llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java +++ llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestLowLevelCacheImpl.java @@ -33,11 +33,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.DiskRange; import org.apache.hadoop.hive.common.DiskRangeList; import org.apache.hadoop.hive.common.DiskRangeList.DiskRangeListCreateHelper; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.io.api.cache.LlapMemoryBuffer; import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache.Priority; import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics; @@ -206,6 +204,70 @@ public void testStaleValueReplace() { } @Test + public void testCacheMetrics() { + DiskRangeListCreateHelper list = new DiskRangeListCreateHelper(); + list.addOrMerge(0, 100, true, false); + list.addOrMerge(100, 200, true, false); + list.addOrMerge(200, 300, true, false); + list.addOrMerge(300, 400, true, false); + list.addOrMerge(400, 500, true, false); + assertEquals(1, list.get().listSize()); + assertEquals(500, list.get().getTotalLength()); + list = new DiskRangeListCreateHelper(); + list.addOrMerge(0, 100, false, false); + list.addOrMerge(100, 200, false, false); + list.addOrMerge(200, 300, false, false); + list.addOrMerge(300, 400, false, false); + list.addOrMerge(400, 500, false, false); + assertEquals(5, list.get().listSize()); + assertEquals(500, list.get().getTotalLength()); + list = new DiskRangeListCreateHelper(); + list.addOrMerge(0, 100, true, false); + list.addOrMerge(100, 200, true, false); + list.addOrMerge(200, 300, false, false); + list.addOrMerge(300, 400, true, false); + list.addOrMerge(400, 500, true, false); + assertEquals(2, list.get().listSize()); + assertEquals(500, list.get().getTotalLength()); + + LlapDaemonCacheMetrics metrics = LlapDaemonCacheMetrics.create("test", "1"); + LowLevelCacheImpl cache = new LowLevelCacheImpl(metrics, + new DummyCachePolicy(), new DummyAllocator(), -1); // no cleanup thread + long fn = 1; + LlapMemoryBuffer[] fakes = new LlapMemoryBuffer[]{fb(), fb(), fb()}; + cache.putFileData(fn, new DiskRange[]{dr(0, 100), dr(300, 500), dr(800, 1000)}, + fakes, 0, Priority.NORMAL); + assertEquals(0, metrics.getCacheRequestedBytes()); + assertEquals(0, metrics.getCacheHitBytes()); + list = new DiskRangeListCreateHelper(); + list.addOrMerge(0, 1000, true, false); + cache.getFileData(fn, list.get(), 0); + assertEquals(1000, metrics.getCacheRequestedBytes()); + assertEquals(500, metrics.getCacheHitBytes()); + + list = new DiskRangeListCreateHelper(); + list.addOrMerge(0, 100, true, false); + cache.getFileData(fn, list.get(), 0); + assertEquals(1100, metrics.getCacheRequestedBytes()); + assertEquals(600, metrics.getCacheHitBytes()); + + list = new DiskRangeListCreateHelper(); + list.addOrMerge(0, 100, true, false); + list.addOrMerge(300, 500, true, false); + list.addOrMerge(800, 1000, true, false); + cache.getFileData(fn, list.get(), 0); + assertEquals(1600, metrics.getCacheRequestedBytes()); + assertEquals(1100, metrics.getCacheHitBytes()); + + list = new DiskRangeListCreateHelper(); + list.addOrMerge(300, 500, true, false); + list.addOrMerge(1000, 2000, true, false); + cache.getFileData(fn, list.get(), 0); + assertEquals(2800, metrics.getCacheRequestedBytes()); + assertEquals(1300, metrics.getCacheHitBytes()); + } + + @Test public void testMTTWithCleanup() { final LowLevelCacheImpl cache = new LowLevelCacheImpl( LlapDaemonCacheMetrics.create("test", "1"), new DummyCachePolicy(), new DummyAllocator(), 1); diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index 298b70d..c6b2e52 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -39,13 +39,8 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Hdfs; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSClient; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hive.common.DiskRange; import org.apache.hadoop.hive.common.DiskRangeList; import org.apache.hadoop.hive.common.DiskRangeList.DiskRangeListCreateHelper;