diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java index e012d7dbf9..62d7e55344 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java @@ -453,11 +453,13 @@ public Allocator getAllocator() { public void debugDumpShort(StringBuilder sb) { sb.append("\nORC cache state "); int allLocked = 0, allUnlocked = 0, allEvicted = 0, allMoving = 0; + long totalUsedSpace = 0; for (Map.Entry>> e : cache.entrySet()) { if (!e.getValue().incRef()) continue; try { int fileLocked = 0, fileUnlocked = 0, fileEvicted = 0, fileMoving = 0; + long fileMemoryUsage = 0; if (e.getValue().getCache().isEmpty()) continue; List lockedBufs = null; if (LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) { @@ -483,6 +485,7 @@ public void debugDumpShort(StringBuilder sb) { ++fileUnlocked; } } finally { + fileMemoryUsage += e2.getValue().allocSize; e2.getValue().decRef(); } } @@ -490,8 +493,21 @@ public void debugDumpShort(StringBuilder sb) { allUnlocked += fileUnlocked; allEvicted += fileEvicted; allMoving += fileMoving; - sb.append("\n file " + e.getKey() + ": " + fileLocked + " locked, " + fileUnlocked - + " unlocked, " + fileEvicted + " evicted, " + fileMoving + " being moved"); + totalUsedSpace += fileMemoryUsage; + + sb.append("\n file " + + e.getKey() + + ": " + + fileLocked + + " locked, " + + fileUnlocked + + " unlocked, " + + fileEvicted + + " evicted, " + + fileMoving + + " being moved," + + fileMemoryUsage + + " total used byte"); if (fileLocked > 0 && LlapIoImpl.LOCKING_LOGGER.isTraceEnabled()) { LlapIoImpl.LOCKING_LOGGER.trace("locked-buffers: {}", lockedBufs); } @@ -499,7 +515,16 @@ public void debugDumpShort(StringBuilder sb) { e.getValue().decRef(); } } - sb.append("\nORC cache summary: " + allLocked + " locked, " + allUnlocked + " unlocked, " - + allEvicted + " evicted, " + allMoving + " being moved"); + sb.append("\nORC cache summary: " + + allLocked + + " locked, " + + allUnlocked + + " unlocked, " + + allEvicted + + " evicted, " + + allMoving + + " being moved," + + totalUsedSpace + + "total used space"); } } diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCachePolicy.java llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCachePolicy.java index 45829dd5c4..aa5ad66314 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCachePolicy.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCachePolicy.java @@ -20,11 +20,59 @@ import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority; +/** + * Actor managing the eviction requests. + * Cache policy relies notifications from the actual {@link LowLevelCache} to keep track of buffer access. + */ public interface LowLevelCachePolicy extends LlapIoDebugDump { + + /** + * Signals to the policy the addition of a new page to the cache directory. + * + * @param buffer buffer to be cached + * @param priority the priority of cached element + */ void cache(LlapCacheableBuffer buffer, Priority priority); + + /** + * Notifies the policy that this buffer is locked, thus take it out of the free list. + * Note that this notification is a hint and can not be the source of truth about what can be evicted + * currently the source of truth is the counter of reference to the buffer see {@link LlapCacheableBuffer#isLocked()}. + * + * @param buffer buffer to be locked. + */ void notifyLock(LlapCacheableBuffer buffer); + + /** + * Notifies the policy that a buffer is unlocked after been used. This notification signals to the policy that an + * access to this page occurred thus can be used to track what page got a read request + * + * @param buffer buffer that just got unlocked + */ void notifyUnlock(LlapCacheableBuffer buffer); + + /** + * Signals to the policy that it has to evict some pages to make room incoming buffers. + * Policy has to at least evict the amount requested. + * Policy does not now about the shape of evicted buffers and only can reason about total size. + * Not that is method will block until at least {@code memoryToReserve} bytes are evicted. + * + * @param memoryToReserve amount of bytes to be evicted + * @return actual amount of evicted bytes. + */ long evictSomeBlocks(long memoryToReserve); + + /** + * Sets the eviction listener dispatcher. + * + * @param listener eviction listener actor + */ void setEvictionListener(EvictionListener listener); + + /** + * Signals to the policy to evict all the unlocked used buffers. + * + * @return amount (bytes) of memory evicted. + */ long purge(); } diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java index e552fee534..704f2f14d3 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java @@ -181,7 +181,7 @@ public long purge() { try { LlapCacheableBuffer current = oldTail = listTail; while (current != null) { - boolean canEvict = LlapCacheableBuffer.INVALIDATE_OK != current.invalidate(); + boolean canEvict = LlapCacheableBuffer.INVALIDATE_OK == current.invalidate(); current.indexInHeap = LlapCacheableBuffer.NOT_IN_CACHE; if (canEvict) { current = current.prev; diff --git llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java index fedade5c9c..0d16703011 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java @@ -20,8 +20,41 @@ import java.util.concurrent.atomic.AtomicBoolean; +/** + * Memory Manager is an accountant over a fixed size of memory. + * It does is the following. + * 1 - tracks the amount of memory (bytes) reserved out of a given maximum size to be shared between IO Threads. + * 2 - when a reservation can not be fulfilled form the current free space it has to notify Evictor to free up some + * space. + *

+ * Note that it does not know about the actual shape, content or owners of memory, all it cares about is bytes usage. + */ public interface MemoryManager { + + /** + * Signals to the Memory manager the release of some memory bytes that are free to be used. + * + * @param memUsage amount of memory bytes that are released to be added to the ledger as free. + */ void releaseMemory(long memUsage); + + /** + * Sets the amount of bytes that the memory manager is managing. + * + * @param maxSize total amount of available bytes to be allocated. + */ void updateMaxSize(long maxSize); + + /** + * Reserves some amount of bytes within the managed pool of memory. + *

+ * Callers expect that the memory manager will always fulfill the request by notifying the Evictor about how much + * need to be evicted to accommodate the reserve request. + * Note that this method will block until reservation is fulfilled. + * + * @param memoryToReserve Amount of bytes to reserve. + * @param isStopped Caller state to indicate if it is still running while the memory manager is trying to + * allocate the space. + */ void reserveMemory(long memoryToReserve, AtomicBoolean isStopped); } diff --git llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/MetadataCache.java llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/MetadataCache.java index 2b3bca6ec1..8400fe9841 100644 --- llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/MetadataCache.java +++ llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/MetadataCache.java @@ -127,9 +127,22 @@ public void notifyEvicted(OrcFileEstimateErrors buffer) { @Override public void debugDumpShort(StringBuilder sb) { - // TODO: perhaps add counters for separate things and multiple buffer cases. - sb.append("\nMetadata cache state: ").append(metadata.size()).append( - " files and stripes, ").append(estimateErrors.size()).append(" files w/ORC estimate"); + sb.append("\nMetadata cache state: ") + .append(metadata.size()) + .append(" files and stripes, ") + .append(metadata.values().parallelStream().mapToLong(value -> { + if (value.getSingleLlapBuffer() != null) { + return value.getSingleLlapBuffer().allocSize; + } + long sum = 0; + for (LlapAllocatorBuffer llapMetadataBuffer : value.getMultipleLlapBuffers()) { + sum += llapMetadataBuffer.allocSize; + } + return sum; + }).sum()) + .append(" total used bytes, ") + .append(estimateErrors.size()) + .append(" files w/ORC estimate"); } @Override