From b334018b8971cb1ef076d98b051adea942aa66f0 Mon Sep 17 00:00:00 2001 From: Panos Garefalakis Date: Mon, 18 May 2020 15:42:30 +0100 Subject: [PATCH] Track HT load usage in Bytes (operator level) Change-Id: I44300d737abf2470b7189fb45808c1bd4f80009e --- .../java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java | 3 ++- .../org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java | 5 +++++ .../mapjoin/fast/VectorMapJoinFastHashTableLoader.java | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java index 6681522281d..7ad4fecda5b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableLoader.java @@ -30,7 +30,8 @@ public interface HashTableLoader { enum HashTableLoaderCounters { - HASHTABLE_LOAD_TIME_MS + HASHTABLE_LOAD_TIME_MS, + HASHTABLE_LOAD_BYTES }; void init(ExecMapperContext context, MapredContext mrContext, Configuration hconf, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java index 9cb00000dca..c3d22aab335 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java @@ -70,6 +70,7 @@ private TezContext tezContext; private String cacheKey; private TezCounter htLoadCounter; + private TezCounter htUsageCounter; @Override public void init(ExecMapperContext context, MapredContext mrContext, Configuration hconf, @@ -82,6 +83,8 @@ public void init(ExecMapperContext context, MapredContext mrContext, Configurati String vertexName = hconf.get(Operator.CONTEXT_NAME_KEY, ""); String counterName = Utilities.getVertexCounterName(HashTableLoaderCounters.HASHTABLE_LOAD_TIME_MS.name(), vertexName); this.htLoadCounter = tezContext.getTezProcessorContext().getCounters().findCounter(counterGroup, counterName); + counterName = Utilities.getVertexCounterName(HashTableLoaderCounters.HASHTABLE_LOAD_BYTES.name(), vertexName); + this.htUsageCounter = tezContext.getTezProcessorContext().getCounters().findCounter(counterGroup, counterName); } @Override @@ -269,6 +272,8 @@ public void load(MapJoinTableContainer[] mapJoinTables, } long delta = System.currentTimeMillis() - startTime; htLoadCounter.increment(delta); + htUsageCounter.increment(tableContainer.getEstimatedMemorySize()); + tableContainer.seal(); mapJoinTables[pos] = tableContainer; if (doMemCheck) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java index a011027ccf8..d731bc3038e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastHashTableLoader.java @@ -58,6 +58,7 @@ private TezContext tezContext; private String cacheKey; private TezCounter htLoadCounter; + private TezCounter htUsageCounter; @Override public void init(ExecMapperContext context, MapredContext mrContext, @@ -70,6 +71,8 @@ public void init(ExecMapperContext context, MapredContext mrContext, String vertexName = hconf.get(Operator.CONTEXT_NAME_KEY, ""); String counterName = Utilities.getVertexCounterName(HashTableLoaderCounters.HASHTABLE_LOAD_TIME_MS.name(), vertexName); this.htLoadCounter = tezContext.getTezProcessorContext().getCounters().findCounter(counterGroup, counterName); + counterName = Utilities.getVertexCounterName(HashTableLoaderCounters.HASHTABLE_LOAD_BYTES.name(), vertexName); + this.htUsageCounter = tezContext.getTezProcessorContext().getCounters().findCounter(counterGroup, counterName); } @Override @@ -158,6 +161,7 @@ public void load(MapJoinTableContainer[] mapJoinTables, } long delta = System.currentTimeMillis() - startTime; htLoadCounter.increment(delta); + htUsageCounter.increment(vectorMapJoinFastTableContainer.getEstimatedMemorySize()); vectorMapJoinFastTableContainer.seal(); mapJoinTables[pos] = vectorMapJoinFastTableContainer; -- 2.20.1 (Apple Git-117)