diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java index d3ec7ff..1e0ffc3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java @@ -77,7 +77,7 @@ * graph as "llap", which in turn causes them to be submitted to an * llap daemon instead of a regular yarn container. * - * The actual algoritm used is driven by LLAP_EXECUTION_MODE. "all", + * The actual algorithm used is driven by LLAP_EXECUTION_MODE. "all", * "none" and "map" mechanically tag those elements. "auto" tries to * be smarter by looking for suitable vertices. * @@ -116,7 +116,7 @@ public LlapDecisionDispatcher(PhysicalContext pctx, LlapMode mode) { @Override public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) - throws SemanticException { + throws SemanticException { @SuppressWarnings("unchecked") Task currTask = (Task) nd; if (currTask instanceof TezTask) { @@ -132,6 +132,12 @@ private void handleWork(TezWork tezWork, BaseWork work) throws SemanticException { if (evaluateWork(tezWork, work)) { convertWork(tezWork, work); + } else { + if (mode == all) { + throw new SemanticException("Llap mode is set to all but cannot run work in llap mode." + + "Set " + HiveConf.ConfVars.LLAP_EXECUTION_MODE + " = auto or set " + + HiveConf.ConfVars.HIVE_EXECUTION_MODE + " = container"); + } } } @@ -152,7 +158,7 @@ private void convertWork(TezWork tezWork, BaseWork work) } private boolean evaluateWork(TezWork tezWork, BaseWork work) - throws SemanticException { + throws SemanticException { LOG.info("Evaluating work item: " + work.getName()); @@ -228,6 +234,7 @@ private boolean evaluateWork(TezWork tezWork, BaseWork work) } // couldn't convince you otherwise? well then let's llap. + LOG.info("Can run work " + work.getName() + " in llap mode."); return true; } @@ -251,7 +258,7 @@ private boolean checkExpression(ExprNodeDesc expr) { if (!isBuiltIn) { if (!arePermanentFnsAllowed) { LOG.info("Not a built-in function: " + cur.getExprString() - + " (permanent functions are disabled)"); + + " (permanent functions are disabled)"); return false; } if (!FunctionRegistry.isPermanentFunction(funcDesc)) { @@ -304,6 +311,7 @@ private boolean checkAggregators(Collection aggs) { @Override public Object process(Node n, Stack s, NodeProcessorCtx c, Object... os) { + LOG.info("Cannot run operator [" + n + "] in llap mode."); return new Boolean(false); } }); @@ -313,7 +321,9 @@ public Object process(Node n, Stack s, NodeProcessorCtx c, public Object process(Node n, Stack s, NodeProcessorCtx c, Object... os) { ExprNodeDesc expr = ((FilterOperator)n).getConf().getPredicate(); - return new Boolean(checkExpression(expr)); + Boolean retval = new Boolean(checkExpression(expr)); + LOG.info("Can filter operator [" + n + "] can be run in llap mode? " + retval); + return new Boolean(retval); } }); opRules.put(new RuleRegExp("No user code in gby", GroupByOperator.getOperatorName() + "%"), @@ -321,9 +331,11 @@ public Object process(Node n, Stack s, NodeProcessorCtx c, @Override public Object process(Node n, Stack s, NodeProcessorCtx c, Object... os) { - @SuppressWarnings("unchecked") - List aggs = ((Operator) n).getConf().getAggregators(); - return new Boolean(checkAggregators(aggs)); + @SuppressWarnings("unchecked") + List aggs = ((Operator) n).getConf().getAggregators(); + Boolean retval = new Boolean(checkAggregators(aggs)); + LOG.info("Can group by operator [" + n + "] be run in llap mode? " + retval); + return new Boolean(retval); } }); opRules.put(new RuleRegExp("No user code in select", SelectOperator.getOperatorName() + "%"), @@ -331,9 +343,11 @@ public Object process(Node n, Stack s, NodeProcessorCtx c, @Override public Object process(Node n, Stack s, NodeProcessorCtx c, Object... os) { - @SuppressWarnings({ "unchecked" }) - List exprs = ((Operator) n).getConf().getColList(); - return new Boolean(checkExpressions(exprs)); + @SuppressWarnings({ "unchecked" }) + List exprs = ((Operator) n).getConf().getColList(); + Boolean retval = new Boolean(checkExpressions(exprs)); + LOG.info("Can select operator [" + n + "] be run in llap mode? " + retval); + return new Boolean(retval); } }); @@ -376,10 +390,10 @@ private boolean checkInputsVectorized(MapWork mapWork) { for (String path : mapWork.getPathToPartitionInfo().keySet()) { PartitionDesc pd = mapWork.getPathToPartitionInfo().get(path); List> interfaceList = - Arrays.asList(pd.getInputFileFormatClass().getInterfaces()); + Arrays.asList(pd.getInputFileFormatClass().getInterfaces()); if (!interfaceList.contains(VectorizedInputFormatInterface.class)) { LOG.info("Input format: " + pd.getInputFileFormatClassName() - + ", doesn't provide vectorized input"); + + ", doesn't provide vectorized input"); return false; } }