diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index fadbc20..0445960 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -922,7 +922,8 @@ private boolean verifyAndSetVectorPartDesc(PartitionDesc pd, boolean isAcidTable mapWork.setVectorizationEnabledConditionsNotMet(enabledConditionsNotMetList); // We consider this an enable issue, not a not vectorized issue. - LOG.info("Cannot enable vectorization because input file format(s) " + inputFileFormatClassNameSet + + setOperatorIssue( + "Cannot enable vectorization because input file format(s) " + inputFileFormatClassNameSet + " do not met conditions " + VectorizationCondition.addBooleans(enabledConditionsNotMetList, false)); return new ImmutablePair(false, true); } @@ -1038,7 +1039,9 @@ private boolean verifyAndSetVectorPartDesc(PartitionDesc pd, boolean isAcidTable private boolean validateMapWork(MapWork mapWork, VectorTaskColumnInfo vectorTaskColumnInfo, boolean isTezOrSpark) throws SemanticException { - LOG.info("Validating MapWork..."); + if (LOG.isDebugEnabled()) { + LOG.debug("Validating MapWork..."); + } ImmutablePair onlyOneTableScanPair = verifyOnlyOneTableScanOperator(mapWork); if (onlyOneTableScanPair == null) { @@ -1094,7 +1097,9 @@ private boolean validateMapWork(MapWork mapWork, VectorTaskColumnInfo vectorTask private void vectorizeMapWork(MapWork mapWork, VectorTaskColumnInfo vectorTaskColumnInfo, boolean isTezOrSpark) throws SemanticException { - LOG.info("Vectorizing MapWork..."); + if (LOG.isDebugEnabled()) { + LOG.debug("Vectorizing MapWork..."); + } mapWork.setVectorMode(true); Map opRules = new LinkedHashMap(); MapWorkVectorizationNodeProcessor vnp = @@ -1215,7 +1220,9 @@ private void addReduceWorkRules(Map opRules, NodeProcessor private boolean validateReduceWork(ReduceWork reduceWork, VectorTaskColumnInfo vectorTaskColumnInfo) throws SemanticException { - LOG.info("Validating ReduceWork..."); + if (LOG.isDebugEnabled()) { + LOG.debug("Validating ReduceWork..."); + } // Validate input to ReduceWork. if (!getOnlyStructObjectInspectors(reduceWork, vectorTaskColumnInfo)) { @@ -1246,7 +1253,9 @@ private boolean validateReduceWork(ReduceWork reduceWork, private void vectorizeReduceWork(ReduceWork reduceWork, VectorTaskColumnInfo vectorTaskColumnInfo) throws SemanticException { - LOG.info("Vectorizing ReduceWork..."); + if (LOG.isDebugEnabled()) { + LOG.debug("Vectorizing ReduceWork..."); + } reduceWork.setVectorMode(true); // For some reason, the DefaultGraphWalker does not descend down from the reducer Operator as @@ -1261,8 +1270,10 @@ private void vectorizeReduceWork(ReduceWork reduceWork, // iterator the reduce operator tree ArrayList topNodes = new ArrayList(); topNodes.add(reduceWork.getReducer()); - LOG.info("vectorizeReduceWork reducer Operator: " + - reduceWork.getReducer().getName() + "..."); + if (LOG.isDebugEnabled()) { + LOG.debug("vectorizeReduceWork reducer Operator: " + + reduceWork.getReducer().getName() + "..."); + } HashMap nodeOutput = new HashMap(); ogw.startWalking(topNodes, nodeOutput); @@ -1454,12 +1465,7 @@ public VectorizationContext walkStackToFindVectorizationContext(Stack stac if (vectorOpParent instanceof VectorizationContextRegion) { VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOpParent; vContext = vcRegion.getOuputVectorizationContext(); - LOG.info("walkStackToFindVectorizationContext " + vectorOpParent.getName() + " has new vectorization context " + vContext.toString()); - } else { - LOG.info("walkStackToFindVectorizationContext " + vectorOpParent.getName() + " does not have new vectorization context"); } - } else { - LOG.info("walkStackToFindVectorizationContext " + opParent.getName() + " is not vectorized"); } --i; } @@ -1520,10 +1526,13 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, if (op instanceof TableScanOperator) { if (taskVectorizationContext == null) { taskVectorizationContext = getVectorizationContext(op.getName(), vectorTaskColumnInfo); + if (LOG.isInfoEnabled()) { + LOG.info("mapColumnNames " + vectorTaskColumnInfo.allColumnNames.toString()); + LOG.info("mapTypeInfos " + vectorTaskColumnInfo.allTypeInfos.toString()); + } } vContext = taskVectorizationContext; } else { - LOG.debug("MapWorkVectorizationNodeProcessor process going to walk the operator stack to get vectorization context for " + op.getName()); vContext = walkStackToFindVectorizationContext(stack, op); if (vContext == null) { // No operator has "pushed" a new context -- so use the task vectorization context. @@ -1584,8 +1593,10 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, currentOperator = op; if (op.getParentOperators().size() == 0) { - LOG.info("ReduceWorkVectorizationNodeProcessor process reduceColumnNames " + vectorTaskColumnInfo.allColumnNames.toString()); - + if (LOG.isInfoEnabled()) { + LOG.info("reduceColumnNames " + vectorTaskColumnInfo.allColumnNames.toString()); + LOG.info("reduceTypeInfos " + vectorTaskColumnInfo.allTypeInfos.toString()); + } vContext = new VectorizationContext("__Reduce_Shuffle__", vectorTaskColumnInfo.allColumnNames, hiveConf); taskVectorizationContext = vContext; @@ -1595,7 +1606,6 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, LOG.debug("Vectorized ReduceWork reduce shuffle vectorization context " + vContext.toString()); } } else { - LOG.info("ReduceWorkVectorizationNodeProcessor process going to walk the operator stack to get vectorization context for " + op.getName()); vContext = walkStackToFindVectorizationContext(stack, op); if (vContext == null) { // If we didn't find a context among the operators, assume the top -- reduce shuffle's @@ -1605,8 +1615,9 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } assert vContext != null; - LOG.info("ReduceWorkVectorizationNodeProcessor process operator " + op.getName() + " using vectorization context" + vContext.toString()); - + if (LOG.isDebugEnabled()) { + LOG.debug("ReduceWorkVectorizationNodeProcessor process operator " + op.getName() + " using vectorization context" + vContext.toString()); + } Operator vectorOp = doVectorize(op, vContext, true); if (LOG.isDebugEnabled()) { @@ -1862,7 +1873,7 @@ private boolean validateMapJoinDesc(MapJoinDesc desc) { return false; } if (desc.getResidualFilterExprs() != null && !desc.getResidualFilterExprs().isEmpty()) { - LOG.info("Cannot vectorize outer join with complex ON clause"); + setExpressionIssue("Residual Filters", "Cannot vectorize outer join with complex ON clause"); return false; } return true; @@ -2039,9 +2050,10 @@ private boolean validateGroupByOperator(GroupByOperator op, boolean isReduce, bo vectorDesc.setProcessingMode(processingMode); - LOG.info("Vector GROUP BY operator will use processing mode " + processingMode.name() + - ", isVectorOutput " + vectorDesc.isVectorOutput()); - + if (LOG.isDebugEnabled()) { + LOG.debug("Vector GROUP BY operator will use processing mode " + processingMode.name() + + ", isVectorOutput " + vectorDesc.isVectorOutput()); + } return true; } @@ -2304,16 +2316,13 @@ private boolean isBigTableOnlyResults(MapJoinDesc desc) { int smallTableIndicesSize; if (desc.getValueIndices() != null && desc.getValueIndices().get(posSingleVectorMapJoinSmallTable) != null) { smallTableIndices = desc.getValueIndices().get(posSingleVectorMapJoinSmallTable); - LOG.info("Vectorizer isBigTableOnlyResults smallTableIndices " + Arrays.toString(smallTableIndices)); smallTableIndicesSize = smallTableIndices.length; } else { smallTableIndices = null; - LOG.info("Vectorizer isBigTableOnlyResults smallTableIndices EMPTY"); smallTableIndicesSize = 0; } List smallTableRetainList = desc.getRetainList().get(posSingleVectorMapJoinSmallTable); - LOG.info("Vectorizer isBigTableOnlyResults smallTableRetainList " + smallTableRetainList); int smallTableRetainSize = smallTableRetainList.size(); if (smallTableIndicesSize > 0) { @@ -2331,7 +2340,6 @@ private boolean isBigTableOnlyResults(MapJoinDesc desc) { return false; } - LOG.info("Vectorizer isBigTableOnlyResults returning true"); return true; } @@ -2373,7 +2381,6 @@ private boolean isBigTableOnlyResults(MapJoinDesc desc) { List bigTableKeyExprs = keyExprs.get(posBigTable); if (bigTableKeyExprs.size() == 1) { TypeInfo typeInfo = bigTableKeyExprs.get(0).getTypeInfo(); - LOG.info("Vectorizer vectorizeOperator map join typeName " + typeInfo.getTypeName()); switch (((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) { case BOOLEAN: hashTableKeyType = HashTableKeyType.BOOLEAN; @@ -2424,8 +2431,6 @@ private boolean isBigTableOnlyResults(MapJoinDesc desc) { throw new HiveException("Unknown join type " + joinType); } - LOG.info("Vectorizer vectorizeOperator map join hashTableKind " + hashTableKind.name() + " hashTableKeyType " + hashTableKeyType.name()); - switch (hashTableKeyType) { case BOOLEAN: case BYTE: @@ -2501,7 +2506,9 @@ private boolean isBigTableOnlyResults(MapJoinDesc desc) { vectorOp = OperatorFactory.getVectorOperator( opClass, op.getCompilationOpContext(), op.getConf(), vContext); - LOG.info("Vectorizer vectorizeOperator map join class " + vectorOp.getClass().getSimpleName()); + if (LOG.isDebugEnabled()) { + LOG.debug("Vectorizer vectorizeOperator map join class " + vectorOp.getClass().getSimpleName()); + } return vectorOp; } @@ -2912,7 +2919,6 @@ private boolean canSpecializeMapJoin(Operator op, MapJoi // Look for single column optimization. if (reduceSinkKeyColumnVectorTypes.length == 1) { - LOG.info("Vectorizer vectorizeOperator groupby typeName " + vectorReduceSinkInfo.getReduceSinkKeyTypeInfos()[0]); Type columnVectorType = reduceSinkKeyColumnVectorTypes[0]; switch (columnVectorType) { case LONG: @@ -2962,7 +2968,9 @@ private boolean canSpecializeMapJoin(Operator op, MapJoi vectorOp = OperatorFactory.getVectorOperator( opClass, op.getCompilationOpContext(), op.getConf(), vContext); - LOG.info("Vectorizer vectorizeOperator reduce sink class " + vectorOp.getClass().getSimpleName()); + if (LOG.isDebugEnabled()) { + LOG.debug("Vectorizer vectorizeOperator reduce sink class " + vectorOp.getClass().getSimpleName()); + } return vectorOp; } @@ -3404,9 +3412,6 @@ private boolean usesVectorUDFAdaptor(VectorExpression[] vecExprs) { vectorTaskColumnInfo.setAllNative(false); } - LOG.debug("vectorizeOperator " + vectorOp.getClass().getName()); - LOG.debug("vectorizeOperator " + vectorOp.getConf().getClass().getName()); - if (vectorOp != op) { fixupParentChildOperators(op, vectorOp); ((AbstractOperatorDesc) vectorOp.getConf()).setVectorMode(true);