diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 1dee6fb..7331e51 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2684,6 +2684,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "hive.tez.exec.inplace.progress", true, "Updates tez job execution progress in-place in the terminal."), + TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION("hive.tez.task.scale.memory.reserve.fraction", 0.5f, + "The fraction of JVM memory which Tez will reserve for the processor. This ratio shouldn't" + + " be smaller than 0.3"), // The default is different on the client and server, so it's null here. LLAP_IO_ENABLED("hive.llap.io.enabled", null, "Whether the LLAP IO layer is enabled."), LLAP_IO_NONVECTOR_WRAPPER_ENABLED("hive.llap.io.nonvector.wrapper.enabled", true, diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index 9e114c0..c58d66e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -89,6 +89,7 @@ private static final String CLASS_NAME = TezTask.class.getName(); private final PerfLogger perfLogger = SessionState.getPerfLogger(); + private static final String TEZ_MEMORY_RESERVE_FRACTION = "tez.task.scale.memory.reserve-fraction"; private TezCounters counters; @@ -341,6 +342,12 @@ DAG build(JobConf conf, TezWork work, Path scratchDir, // translate work to vertex perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_CREATE_VERTEX + w.getName()); + // Reserve a customized (usually more) percentage of memory for DAG that has Mapjoins + if (w.getHasMapJoin()) { + dag.setConf(TEZ_MEMORY_RESERVE_FRACTION, + Float.toString(super.conf.getFloatVar(HiveConf.ConfVars.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION))); + } + if (w instanceof UnionWork) { // Special case for unions. These items translate to VertexGroups diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java index 461ba37..21494af 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java @@ -244,6 +244,7 @@ public Object process(Node nd, Stack stack, // mapjoin later if (!context.mapJoinWorkMap.containsKey(mj)) { List workItems = new LinkedList(); + work.setHasMapJoin(true); workItems.add(work); context.mapJoinWorkMap.put(mj, workItems); } else { diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java index 20f787b..2199576 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java @@ -74,6 +74,8 @@ public BaseWork(String name) { protected boolean llapMode = false; protected boolean uberMode = false; + private boolean hasMapJoin = false; + public void setGatheringStats(boolean gatherStats) { this.gatheringStats = gatherStats; } @@ -223,6 +225,14 @@ public boolean getLlapMode() { return llapMode; } + public boolean getHasMapJoin() { + return hasMapJoin; + } + + public void setHasMapJoin(boolean hasMapJoin) { + this.hasMapJoin = hasMapJoin; + } + public abstract void configureJobConf(JobConf job); public void setTag(int tag) {