diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java index 536b92c..4740f4a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.exec.tez; import java.io.IOException; +import java.lang.management.ManagementFactory; import java.util.Collections; import java.util.Map; @@ -77,11 +78,20 @@ public void load(MapJoinTableContainer[] mapJoinTables, boolean useOptimizedTables = HiveConf.getBoolVar( hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE); + float hashtableMemoryUsage = HiveConf.getFloatVar( + hconf, HiveConf.ConfVars.HIVEHASHTABLEFOLLOWBYGBYMAXMEMORYUSAGE); boolean useHybridGraceHashJoin = desc.isHybridHashJoin(); boolean isFirstKey = true; // TODO remove this after memory manager is in long noConditionalTaskThreshold = HiveConf.getLongVar( hconf, HiveConf.ConfVars.HIVECONVERTJOINNOCONDITIONALTASKTHRESHOLD); + long processMaxMemory = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); + if (noConditionalTaskThreshold > processMaxMemory) { + LOG.warn("noConditionalTaskThreshold value of " + noConditionalTaskThreshold + + " is greater than the max memory size of " + processMaxMemory); + // Don't want to attempt to grab more memory than we have available .. percentage is a bit arbitrary + noConditionalTaskThreshold = (long) (processMaxMemory * hashtableMemoryUsage); + } // Only applicable to n-way Hybrid Grace Hash Join HybridHashTableConf nwayConf = null;