diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index d287b45..ae71fca 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2848,7 +2848,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal LLAP_ALLOW_PERMANENT_FNS("hive.llap.allow.permanent.fns", true, "Whether LLAP decider should allow permanent UDFs."), LLAP_EXECUTION_MODE("hive.llap.execution.mode", "none", - new StringSet("auto", "none", "all", "map"), + new StringSet("auto", "none", "all", "map", "all_fail_if_disalllowed"), "Chooses whether query fragments will run in container or in llap"), LLAP_OBJECT_CACHE_ENABLED("hive.llap.object.cache.enabled", true, "Cache objects (plans, hashtables, etc) in llap"), diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java index f46e2d7..fe97b24 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.optimizer.physical; import static org.apache.hadoop.hive.ql.optimizer.physical.LlapDecider.LlapMode.all; +import static org.apache.hadoop.hive.ql.optimizer.physical.LlapDecider.LlapMode.all_fail_if_disallowed; import static org.apache.hadoop.hive.ql.optimizer.physical.LlapDecider.LlapMode.auto; import static org.apache.hadoop.hive.ql.optimizer.physical.LlapDecider.LlapMode.map; import static org.apache.hadoop.hive.ql.optimizer.physical.LlapDecider.LlapMode.none; @@ -58,6 +59,7 @@ import org.apache.hadoop.hive.ql.lib.Rule; import org.apache.hadoop.hive.ql.lib.RuleRegExp; import org.apache.hadoop.hive.ql.lib.TaskGraphWalker; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.BaseWork; @@ -94,8 +96,9 @@ public enum LlapMode { map, // map operators only - all, // all operators + all, // all operators. Launch containers if user code etc prevents running inside llap. none, // no operators + all_fail_if_disallowed, // all operators. Fail if user code etc prevents running inside llap. auto // please hive, choose for me } @@ -107,6 +110,7 @@ private final boolean arePermanentFnsAllowed; private final boolean shouldUber; private List mapJoinOpList; + private final Map rules; public LlapDecisionDispatcher(PhysicalContext pctx, LlapMode mode) { conf = pctx.getConf(); @@ -115,6 +119,7 @@ public LlapDecisionDispatcher(PhysicalContext pctx, LlapMode mode) { // Don't user uber in "all" mode - everything can go into LLAP, which is better than uber. shouldUber = HiveConf.getBoolVar(conf, ConfVars.LLAP_AUTO_ALLOW_UBER) && (mode != all); mapJoinOpList = new ArrayList(); + rules = getRules(); } @Override @@ -175,6 +180,7 @@ private boolean evaluateWork(TezWork tezWork, BaseWork work) return false; } + // first we check if we *can* run in llap. If we need to use // user code to do so (script/udf) we don't. /*if (work instanceof MapWork && ((MapWork)work).isUseOneNullRowInputFormat()) { @@ -184,6 +190,11 @@ private boolean evaluateWork(TezWork tezWork, BaseWork work) if (!evaluateOperators(work)) { LOG.info("some operators cannot be run in llap"); + if (mode == all_fail_if_disallowed) { + throw new RuntimeException("Cannot run all parts of query in llap. Failing since " + + ConfVars.LLAP_EXECUTION_MODE.varname + " is set to " + all_fail_if_disallowed.name()); + } + return false; } @@ -388,7 +399,7 @@ public Object process(Node n, Stack s, NodeProcessorCtx c, Object... os) { private boolean evaluateOperators(BaseWork work) throws SemanticException { // lets take a look at the operators. we're checking for user // code in those. we will not run that in llap. - Dispatcher disp = new DefaultRuleDispatcher(null, getRules(), null); + Dispatcher disp = new DefaultRuleDispatcher(null, rules, null); GraphWalker ogw = new DefaultGraphWalker(disp); ArrayList topNodes = new ArrayList(); @@ -400,7 +411,6 @@ private boolean evaluateOperators(BaseWork work) throws SemanticException { for (Node n : nodeOutput.keySet()) { if (nodeOutput.get(n) != null) { if (!((Boolean)nodeOutput.get(n))) { - LOG.info("Cannot run in LLAP mode."); return false; } }