diff --git llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/MetadataCache.java llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/MetadataCache.java index 10bd736486..ce1e23670a 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/MetadataCache.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/MetadataCache.java @@ -63,7 +63,7 @@ public MetadataCache(BuddyAllocator allocator, MemoryManager memoryManager, this.policy = policy; this.metrics = metrics; this.estimateErrors = useEstimateCache - ? new ConcurrentHashMap() : null; + ? new ConcurrentHashMap<>() : null; } public void putIncompleteCbs(Object fileKey, DiskRange[] ranges, long baseOffset, AtomicBoolean isStopped) { @@ -235,7 +235,7 @@ private LlapBufferOrBuffers wrapBbForFile(LlapBufferOrBuffers result, if (maxAlloc < length) { largeBuffers = new LlapMetadataBuffer[length / maxAlloc]; for (int i = 0; i < largeBuffers.length; ++i) { - largeBuffers[i] = new LlapMetadataBuffer(fileKey, tag); + largeBuffers[i] = new LlapMetadataBuffer<>(fileKey, tag); } allocator.allocateMultiple(largeBuffers, maxAlloc, null, isStopped); for (int i = 0; i < largeBuffers.length; ++i) { @@ -256,7 +256,7 @@ private LlapBufferOrBuffers wrapBbForFile(LlapBufferOrBuffers result, LlapMetadataBuffer[] cacheData = new LlapMetadataBuffer[largeBuffers.length + 1]; System.arraycopy(largeBuffers, 0, cacheData, 0, largeBuffers.length); cacheData[largeBuffers.length] = smallBuffer[0]; - return new LlapMetadataBuffers(cacheData); + return new LlapMetadataBuffers<>(cacheData); } } } @@ -340,16 +340,16 @@ public void decRefBuffer(MemoryBufferOrBuffers buffer) { if (result != null) return result; if (tailBuffer.remaining() <= allocator.getMaxAllocation()) { // The common case by far. - return wrapSmallBb(new LlapMetadataBuffer(key, tag), tailBuffer, isStopped); + return wrapSmallBb(new LlapMetadataBuffer<>(key, tag), tailBuffer, isStopped); } else { int allocCount = determineAllocCount(tailBuffer); @SuppressWarnings("unchecked") LlapMetadataBuffer[] results = new LlapMetadataBuffer[allocCount]; for (int i = 0; i < allocCount; ++i) { - results[i] = new LlapMetadataBuffer(key, tag); + results[i] = new LlapMetadataBuffer<>(key, tag); } wrapLargeBb(results, tailBuffer, isStopped); - return new LlapMetadataBuffers(results); + return new LlapMetadataBuffers<>(results); } } @@ -406,7 +406,7 @@ private boolean lockBuffer(LlapBufferOrBuffers buffers, boolean doNotifyPolicy) for (int i = 0; i < bufferArray.length; ++i) { if (lockOneBuffer(bufferArray[i], doNotifyPolicy)) continue; for (int j = 0; j < i; ++j) { - unlockSingleBuffer(buffer, true); + unlockSingleBuffer(bufferArray[j], true); } discardMultiBuffer(buffers); return false; @@ -497,7 +497,7 @@ public boolean equals(Object obj) { } } - public static interface LlapBufferOrBuffers extends MemoryBufferOrBuffers { + public interface LlapBufferOrBuffers extends MemoryBufferOrBuffers { LlapAllocatorBuffer getSingleLlapBuffer(); LlapAllocatorBuffer[] getMultipleLlapBuffers(); } diff --git llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java index e5953e2b95..84d628554d 100644 --- llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java +++ llap-server/src/test/org/apache/hadoop/hive/llap/cache/TestOrcMetadataCache.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.llap.io.metadata.MetadataCache.LlapMetadataBuffer; import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics; import org.apache.hadoop.hive.ql.io.orc.encoded.IncompleteCb; +import org.junit.Assert; import org.junit.Test; public class TestOrcMetadataCache { @@ -97,6 +98,30 @@ public void updateMaxSize(long maxSize) { } } + @Test + public void testCaseSomePartialBuffersAreEvicted() { + final DummyMemoryManager mm = new DummyMemoryManager(); + final DummyCachePolicy cp = new DummyCachePolicy(); + final int MAX_ALLOC = 64; + final LlapDaemonCacheMetrics metrics = LlapDaemonCacheMetrics.create("", ""); + final BuddyAllocator alloc = new BuddyAllocator(false, false, 8, MAX_ALLOC, 1, 4096, 0, null, mm, metrics, null, true); + final MetadataCache cache = new MetadataCache(alloc, mm, cp, true, metrics); + final Object fileKey1 = new Object(); + final Random rdm = new Random(); + final ByteBuffer smallBuffer = ByteBuffer.allocate(2 * MAX_ALLOC); + rdm.nextBytes(smallBuffer.array()); + //put some metadata in the cache that needs multiple buffers (2 * MAX_ALLOC) + final LlapBufferOrBuffers result = cache.putFileMetadata(fileKey1, smallBuffer, null, null); + // assert that we have our 2 buffers + Assert.assertEquals(2, result.getMultipleLlapBuffers().length); + final LlapAllocatorBuffer[] buffers = result.getMultipleLlapBuffers(); + //test setup where one buffer is evicted and therefore can not be locked + buffers[1].decRef(); + buffers[1].invalidateAndRelease(); + //Try to get the buffer should lead to cleaning the cache since some part was evicted. + Assert.assertNull(cache.getFileMetadata(fileKey1)); + } + @Test public void testBuffers() throws Exception { DummyMemoryManager mm = new DummyMemoryManager();