diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index f8e715d..0e87498 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2999,6 +2999,11 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "internal use only, used to override the hive.vectorized.execution.enabled setting and\n" + "turn off vectorization. The default is false, of course", true), + HIVE_TEST_VECTORIZATION_SUPPRESS_EXPLAIN_EXECUTION_MODE( + "hive.test.vectorization.suppress.explain.execution.mode", false, + "internal use only, used to suppress \"Execution mode: vectorized\" EXPLAIN display.\n" + + "The default is false, of course", + true), HIVE_TYPE_CHECK_ON_INSERT("hive.typecheck.on.insert", true, "This property has been extended to control " + "whether to check, convert, and normalize partition value to conform to its column type in " diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java index 0f3c5f2..d508d02 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java @@ -85,7 +85,10 @@ private void initialize(HiveConf hiveConf) { // Vectorization should be the last optimization, because it doesn't modify the plan // or any operators. It makes a very low level transformation to the expressions to // run in the vectorized mode. - if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)) { + if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED) || + HiveConf.getVar(hiveConf, + HiveConf.ConfVars.HIVE_TEST_VECTORIZATION_ENABLED_OVERRIDE).equalsIgnoreCase( + "enable")) { resolvers.add(new Vectorizer()); } if (!"none".equalsIgnoreCase(hiveConf.getVar(HiveConf.ConfVars.HIVESTAGEIDREARRANGE))) { diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 13a2fc4..c839536 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -231,6 +231,7 @@ import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hive.common.util.AnnotationUtils; import org.apache.hadoop.util.ReflectionUtils; + import com.google.common.collect.ImmutableSet; import com.google.common.base.Preconditions; @@ -300,6 +301,7 @@ boolean isVectorizationEnabled; private VectorizationEnabledOverride vectorizationEnabledOverride; + boolean isTestForcedVectorizationEnable; private boolean useVectorizedInputFileFormat; private boolean useVectorDeserialize; @@ -336,6 +338,8 @@ supportedAcidInputFormats.add(OneNullRowInputFormat.class.getName()); } + private boolean isTestVectorizationSuppressExplainExecutionMode; + private BaseWork currentBaseWork; private Operator currentOperator; private Collection> vectorizedInputFormatExcludes; @@ -651,6 +655,10 @@ public void transferToBaseWork(BaseWork baseWork) { baseWork.setAllNative(allNative); baseWork.setUsesVectorUDFAdaptor(usesVectorUDFAdaptor); + + baseWork.setIsTestForcedVectorizationEnable(isTestForcedVectorizationEnable); + baseWork.setIsTestVectorizationSuppressExplainExecutionMode( + isTestVectorizationSuppressExplainExecutionMode); } } @@ -923,19 +931,30 @@ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) Task currTask = (Task) nd; if (currTask instanceof MapRedTask) { MapredWork mapredWork = ((MapRedTask) currTask).getWork(); - convertMapWork(mapredWork.getMapWork(), false); + + MapWork mapWork = mapredWork.getMapWork(); + setMapWorkExplainConditions(mapWork); + convertMapWork(mapredWork.getMapWork(), /* isTezOrSpark */ false); + logMapWorkExplainVectorization(mapWork); + ReduceWork reduceWork = mapredWork.getReduceWork(); if (reduceWork != null) { + // Always set the EXPLAIN conditions. setReduceWorkExplainConditions(reduceWork); // We do not vectorize MR Reduce. + + logReduceWorkExplainVectorization(reduceWork); } } else if (currTask instanceof TezTask) { TezWork work = ((TezTask) currTask).getWork(); for (BaseWork baseWork: work.getAllWork()) { if (baseWork instanceof MapWork) { - convertMapWork((MapWork) baseWork, true); + MapWork mapWork = (MapWork) baseWork; + setMapWorkExplainConditions(mapWork); + convertMapWork(mapWork, /* isTezOrSpark */ true); + logMapWorkExplainVectorization(mapWork); } else if (baseWork instanceof ReduceWork) { ReduceWork reduceWork = (ReduceWork) baseWork; @@ -946,13 +965,17 @@ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) if (isReduceVectorizationEnabled) { convertReduceWork(reduceWork); } + logReduceWorkExplainVectorization(reduceWork); } } } else if (currTask instanceof SparkTask) { SparkWork sparkWork = (SparkWork) currTask.getWork(); for (BaseWork baseWork : sparkWork.getAllWork()) { if (baseWork instanceof MapWork) { - convertMapWork((MapWork) baseWork, true); + MapWork mapWork = (MapWork) baseWork; + setMapWorkExplainConditions(mapWork); + convertMapWork(mapWork, /* isTezOrSpark */ true); + logMapWorkExplainVectorization(mapWork); } else if (baseWork instanceof ReduceWork) { ReduceWork reduceWork = (ReduceWork) baseWork; @@ -962,6 +985,7 @@ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) if (isReduceVectorizationEnabled) { convertReduceWork(reduceWork); } + logReduceWorkExplainVectorization(reduceWork); } } } @@ -969,28 +993,109 @@ public Object dispatch(Node nd, Stack stack, Object... nodeOutputs) return null; } - private void convertMapWork(MapWork mapWork, boolean isTezOrSpark) throws SemanticException { - - mapWork.setVectorizationExamined(true); + private void setExplainConditions(BaseWork baseWork) { // Global used when setting errors, etc. - currentBaseWork = mapWork; + currentBaseWork = baseWork; - VectorTaskColumnInfo vectorTaskColumnInfo = new VectorTaskColumnInfo(); - vectorTaskColumnInfo.assume(); + baseWork.setVectorizedVertexNum(++vectorizedVertexNum); + baseWork.setVectorizationExamined(true); + } - mapWork.setVectorizedVertexNum(++vectorizedVertexNum); + private void setMapWorkExplainConditions(MapWork mapWork) { + setExplainConditions(mapWork); + } - if (!validateAndVectorizeMapWork(mapWork, vectorTaskColumnInfo, isTezOrSpark)) { - if (currentBaseWork.getVectorizationEnabled()) { - VectorizerReason notVectorizedReason = currentBaseWork.getNotVectorizedReason(); - if (notVectorizedReason == null) { - LOG.info("Cannot vectorize: unknown"); - } else { - LOG.info("Cannot vectorize: " + notVectorizedReason.toString()); - } + private void setReduceWorkExplainConditions(ReduceWork reduceWork) { + + setExplainConditions(reduceWork); + + reduceWork.setReduceVectorizationEnabled(isReduceVectorizationEnabled); + reduceWork.setVectorReduceEngine( + HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE)); + } + + private boolean logExplainVectorization(BaseWork baseWork, String name) { + + if (!baseWork.getVectorizationExamined()) { + return false; + } + + LOG.info(name + " vectorization enabled: " + baseWork.getVectorizationEnabled()); + boolean isVectorized = baseWork.getVectorMode(); + LOG.info(name + " vectorized: " + isVectorized); + if (!isVectorized) { + VectorizerReason notVectorizedReason = baseWork.getNotVectorizedReason(); + if (notVectorizedReason != null) { + LOG.info(name + " notVectorizedReason: " + notVectorizedReason.toString()); + } + } + LOG.info(name + " vectorizedVertexNum: " + baseWork.getVectorizedVertexNum()); + + if (LOG.isDebugEnabled() && isVectorized) { + VectorizedRowBatchCtx batchContext = baseWork.getVectorizedRowBatchCtx(); + LOG.debug(name + " dataColumnCount: " + batchContext.getDataColumnCount()); + int[] dataColumnNums = batchContext.getDataColumnNums(); + if (dataColumnNums != null) { + LOG.debug(name + " includeColumns: " + Arrays.toString(dataColumnNums)); + } + LOG.debug(name + " partitionColumnCount: " + batchContext.getPartitionColumnCount()); + LOG.debug(name + " dataColumns: " + + BaseWork.BaseExplainVectorization.getColumns( + batchContext, 0, batchContext.getDataColumnCount())); + LOG.debug(name + " scratchColumnTypeNames: " + + BaseWork.BaseExplainVectorization.getScratchColumns(batchContext)); + VirtualColumn[] neededVirtualColumns = batchContext.getNeededVirtualColumns(); + if (neededVirtualColumns != null && neededVirtualColumns.length != 0) { + LOG.debug(name + " neededVirtualColumns: " + Arrays.toString(neededVirtualColumns)); } } + return true; + } + + private void logMapWorkExplainVectorization(MapWork mapWork) { + + if (!logExplainVectorization(mapWork, "Map")) { + return; + } + + // Conditions. + List enabledConditionsMet = mapWork.getVectorizationEnabledConditionsMet(); + if (enabledConditionsMet != null && !enabledConditionsMet.isEmpty()) { + LOG.info("Map enabledConditionsMet: " + enabledConditionsMet.toString()); + } + List enabledConditionsNotMet = mapWork.getVectorizationEnabledConditionsNotMet(); + if (enabledConditionsNotMet != null && !enabledConditionsNotMet.isEmpty()) { + LOG.info("Map enabledConditionsNotMet: " + enabledConditionsNotMet.toString()); + } + Set inputFileFormatClassNameSet = + mapWork.getVectorizationInputFileFormatClassNameSet(); + if (inputFileFormatClassNameSet != null && !inputFileFormatClassNameSet.isEmpty()) { + LOG.info("Map inputFileFormatClassNameSet: " + inputFileFormatClassNameSet.toString()); + } + } + + private void logReduceWorkExplainVectorization(ReduceWork reduceWork) { + + if (!logExplainVectorization(reduceWork, "Reduce")) { + return; + } + + // Conditions. + LOG.info("Reducer " + HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCE_ENABLED.varname + + ": " + reduceWork.getReduceVectorizationEnabled()); + LOG.info("Reducer engine: " + reduceWork.getVectorReduceEngine()); + } + + private void convertMapWork(MapWork mapWork, boolean isTezOrSpark) throws SemanticException { + + // We have to evaluate the input format to see if vectorization is enabled, so + // we do not set it right here. + + VectorTaskColumnInfo vectorTaskColumnInfo = new VectorTaskColumnInfo(); + vectorTaskColumnInfo.assume(); + + validateAndVectorizeMapWork(mapWork, vectorTaskColumnInfo, isTezOrSpark); } private void addMapWorkRules(Map opRules, NodeProcessor np) { @@ -1595,7 +1700,7 @@ private boolean isInputFormatExcluded(String inputFileFormatClassName, Collectio return new ImmutablePair(true, false); } - private boolean validateAndVectorizeMapWork(MapWork mapWork, VectorTaskColumnInfo vectorTaskColumnInfo, + private void validateAndVectorizeMapWork(MapWork mapWork, VectorTaskColumnInfo vectorTaskColumnInfo, boolean isTezOrSpark) throws SemanticException { //-------------------------------------------------------------------------------------------- @@ -1607,7 +1712,7 @@ private boolean validateAndVectorizeMapWork(MapWork mapWork, VectorTaskColumnInf VectorizerReason notVectorizedReason = currentBaseWork.getNotVectorizedReason(); Preconditions.checkState(notVectorizedReason != null); mapWork.setVectorizationEnabledConditionsNotMet(Arrays.asList(new String[] {notVectorizedReason.toString()})); - return false; + return; } String alias = onlyOneTableScanPair.left; TableScanOperator tableScanOperator = onlyOneTableScanPair.right; @@ -1625,7 +1730,7 @@ private boolean validateAndVectorizeMapWork(MapWork mapWork, VectorTaskColumnInf Preconditions.checkState(notVectorizedReason != null); mapWork.setVectorizationEnabledConditionsNotMet(Arrays.asList(new String[] {notVectorizedReason.toString()})); } - return false; + return; } final int dataColumnCount = @@ -1705,7 +1810,7 @@ private boolean validateAndVectorizeMapWork(MapWork mapWork, VectorTaskColumnInf * Validate and vectorize the Map operator tree. */ if (!validateAndVectorizeMapOperators(mapWork, tableScanOperator, isTezOrSpark, vectorTaskColumnInfo)) { - return false; + return; } //-------------------------------------------------------------------------------------------- @@ -1714,11 +1819,7 @@ private boolean validateAndVectorizeMapWork(MapWork mapWork, VectorTaskColumnInf mapWork.setVectorMode(true); - if (LOG.isDebugEnabled()) { - debugDisplayVertexInfo(mapWork); - } - - return true; + return; } private boolean validateAndVectorizeMapOperators(MapWork mapWork, TableScanOperator tableScanOperator, @@ -1859,47 +1960,26 @@ private void vectorizeTableScanOperatorInPlace(TableScanOperator tableScanOperat return newChildren; } - private void setReduceWorkExplainConditions(ReduceWork reduceWork) { - - reduceWork.setVectorizationExamined(true); - - reduceWork.setReduceVectorizationEnabled(isReduceVectorizationEnabled); - reduceWork.setVectorReduceEngine( - HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE)); - } - - private void convertReduceWork(ReduceWork reduceWork) throws SemanticException { + private void convertReduceWork(ReduceWork reduceWork) throws SemanticException { - // Global used when setting errors, etc. - currentBaseWork = reduceWork; - currentBaseWork.setVectorizationEnabled(true); + reduceWork.setVectorizationEnabled(true); VectorTaskColumnInfo vectorTaskColumnInfo = new VectorTaskColumnInfo(); vectorTaskColumnInfo.assume(); - reduceWork.setVectorizedVertexNum(++vectorizedVertexNum); reduceWork.setVectorizedTestingReducerBatchSize(vectorizedTestingReducerBatchSize); - if (!validateAndVectorizeReduceWork(reduceWork, vectorTaskColumnInfo)) { - if (currentBaseWork.getVectorizationEnabled()) { - VectorizerReason notVectorizedReason = currentBaseWork.getNotVectorizedReason(); - if (notVectorizedReason == null) { - LOG.info("Cannot vectorize: unknown"); - } else { - LOG.info("Cannot vectorize: " + notVectorizedReason.toString()); - } - } - } + validateAndVectorizeReduceWork(reduceWork, vectorTaskColumnInfo); } - private boolean validateAndVectorizeReduceWork(ReduceWork reduceWork, + private void validateAndVectorizeReduceWork(ReduceWork reduceWork, VectorTaskColumnInfo vectorTaskColumnInfo) throws SemanticException { Operator reducer = reduceWork.getReducer(); // Validate input to ReduceWork. if (!getOnlyStructObjectInspectors(reduceWork, vectorTaskColumnInfo)) { - return false; + return; } //-------------------------------------------------------------------------------------------- @@ -1908,7 +1988,7 @@ private boolean validateAndVectorizeReduceWork(ReduceWork reduceWork, * Validate and vectorize the Reduce operator tree. */ if (!validateAndVectorizeReduceOperators(reduceWork, vectorTaskColumnInfo)) { - return false; + return; } //-------------------------------------------------------------------------------------------- @@ -1917,11 +1997,7 @@ private boolean validateAndVectorizeReduceWork(ReduceWork reduceWork, reduceWork.setVectorMode(true); - if (LOG.isDebugEnabled()) { - debugDisplayVertexInfo(reduceWork); - } - - return true; + return; } private boolean validateAndVectorizeReduceOperators(ReduceWork reduceWork, @@ -2073,6 +2149,7 @@ public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticE HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED); final boolean weCanAttemptVectorization; + isTestForcedVectorizationEnable = false; switch (vectorizationEnabledOverride) { case NONE: weCanAttemptVectorization = isVectorizationEnabled; @@ -2082,6 +2159,7 @@ public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticE break; case ENABLE: weCanAttemptVectorization = true; + isTestForcedVectorizationEnable = !isVectorizationEnabled; break; default: throw new RuntimeException("Unexpected vectorization enabled override " + @@ -2162,6 +2240,10 @@ public PhysicalContext resolve(PhysicalContext physicalContext) throws SemanticE hiveVectorAdaptorUsageMode = HiveVectorAdaptorUsageMode.getHiveConfValue(hiveConf); + isTestVectorizationSuppressExplainExecutionMode = + HiveConf.getBoolVar(hiveConf, + HiveConf.ConfVars.HIVE_TEST_VECTORIZATION_SUPPRESS_EXPLAIN_EXECUTION_MODE); + // create dispatcher and graph walker Dispatcher disp = new VectorizationDispatcher(); TaskGraphWalker ogw = new TaskGraphWalker(disp); @@ -4778,27 +4860,4 @@ private static VectorPTFInfo createVectorPTFInfo(Operator getColumns(VectorizedRowBatchCtx vectorizedRowBatchCtx, + int startIndex, int count) { + String[] rowColumnNames = vectorizedRowBatchCtx.getRowColumnNames(); + TypeInfo[] rowColumnTypeInfos = vectorizedRowBatchCtx.getRowColumnTypeInfos(); + DataTypePhysicalVariation[] dataTypePhysicalVariations = + vectorizedRowBatchCtx.getRowdataTypePhysicalVariations(); + + List result = new ArrayList(count); + final int end = startIndex + count; + for (int i = startIndex; i < end; i++) { + String displayString = rowColumnNames[i] + ":" + rowColumnTypeInfos[i]; + if (dataTypePhysicalVariations != null && + dataTypePhysicalVariations[i] != DataTypePhysicalVariation.NONE) { + displayString += "/" + dataTypePhysicalVariations[i].toString(); + } + result.add(displayString); + } + return result; + } + + public static String getScratchColumns(VectorizedRowBatchCtx vectorizedRowBatchCtx) { + String[] scratchColumnTypeNames = vectorizedRowBatchCtx.getScratchColumnTypeNames(); + DataTypePhysicalVariation[] scratchDataTypePhysicalVariations = vectorizedRowBatchCtx.getScratchDataTypePhysicalVariations(); + final int size = scratchColumnTypeNames.length; + List result = new ArrayList(size); + for (int i = 0; i < size; i++) { + String displayString = scratchColumnTypeNames[i]; + if (scratchDataTypePhysicalVariations != null && scratchDataTypePhysicalVariations[i] != DataTypePhysicalVariation.NONE) { + displayString += "/" + scratchDataTypePhysicalVariations[i].toString(); + } + result.add(displayString); + } + return result.toString(); + } + @Explain(vectorization = Vectorization.SUMMARY, displayName = "usesVectorUDFAdaptor", explainLevels = { Level.DEFAULT, Level.EXTENDED }) public Boolean usesVectorUDFAdaptor() { if (!baseWork.getVectorMode()) { @@ -336,33 +392,14 @@ public RowBatchContextExplainVectorization(VectorizedRowBatchCtx vectorizedRowBa this.vectorizedRowBatchCtx = vectorizedRowBatchCtx; } - private List getColumns(int startIndex, int count) { - String[] rowColumnNames = vectorizedRowBatchCtx.getRowColumnNames(); - TypeInfo[] rowColumnTypeInfos = vectorizedRowBatchCtx.getRowColumnTypeInfos(); - DataTypePhysicalVariation[] dataTypePhysicalVariations = - vectorizedRowBatchCtx.getRowdataTypePhysicalVariations(); - - List result = new ArrayList(count); - final int end = startIndex + count; - for (int i = startIndex; i < end; i++) { - String displayString = rowColumnNames[i] + ":" + rowColumnTypeInfos[i]; - if (dataTypePhysicalVariations != null && - dataTypePhysicalVariations[i] != DataTypePhysicalVariation.NONE) { - displayString += "/" + dataTypePhysicalVariations[i].toString(); - } - result.add(displayString); - } - return result; - } - @Explain(vectorization = Vectorization.DETAIL, displayName = "dataColumns", explainLevels = { Level.DEFAULT, Level.EXTENDED }) public List getDataColumns() { - return getColumns(0, vectorizedRowBatchCtx.getDataColumnCount()); + return getColumns(vectorizedRowBatchCtx, 0, vectorizedRowBatchCtx.getDataColumnCount()); } @Explain(vectorization = Vectorization.DETAIL, displayName = "partitionColumns", explainLevels = { Level.DEFAULT, Level.EXTENDED }) public List getPartitionColumns() { - return getColumns(vectorizedRowBatchCtx.getDataColumnCount(), vectorizedRowBatchCtx.getPartitionColumnCount()); + return getColumns(vectorizedRowBatchCtx, vectorizedRowBatchCtx.getDataColumnCount(), vectorizedRowBatchCtx.getPartitionColumnCount()); } @Explain(vectorization = Vectorization.DETAIL, displayName = "includeColumns", explainLevels = { Level.DEFAULT, Level.EXTENDED }) @@ -371,7 +408,7 @@ public String getDataColumnNums() { if (dataColumnNums == null) { return null; } - return Arrays.toString(vectorizedRowBatchCtx.getDataColumnNums()); + return Arrays.toString(dataColumnNums); } @Explain(vectorization = Vectorization.DETAIL, displayName = "dataColumnCount", explainLevels = { Level.DEFAULT, Level.EXTENDED }) @@ -386,18 +423,7 @@ public int getPartitionColumnCount() { @Explain(vectorization = Vectorization.DETAIL, displayName = "scratchColumnTypeNames", explainLevels = { Level.DEFAULT, Level.EXTENDED }) public String getScratchColumnTypeNames() { - String[] scratchColumnTypeNames = vectorizedRowBatchCtx.getScratchColumnTypeNames(); - DataTypePhysicalVariation[] scratchDataTypePhysicalVariations = vectorizedRowBatchCtx.getScratchDataTypePhysicalVariations(); - final int size = scratchColumnTypeNames.length; - List result = new ArrayList(size); - for (int i = 0; i < size; i++) { - String displayString = scratchColumnTypeNames[i]; - if (scratchDataTypePhysicalVariations != null && scratchDataTypePhysicalVariations[i] != DataTypePhysicalVariation.NONE) { - displayString += "/" + scratchDataTypePhysicalVariations[i].toString(); - } - result.add(displayString); - } - return result.toString(); + return getScratchColumns(vectorizedRowBatchCtx); } @Explain(vectorization = Vectorization.DETAIL, displayName = "neededVirtualColumns", explainLevels = { Level.DEFAULT, Level.EXTENDED }) diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java index 1f139c8..bd00fdc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/FetchWork.java @@ -347,6 +347,8 @@ public boolean getVectorizationExamined() { private final FetchWork fetchWork; + private VectorizationCondition[] fetchWorkVectorizationConditions; + public FetchExplainVectorization(FetchWork fetchWork) { this.fetchWork = fetchWork; } @@ -356,9 +358,34 @@ public boolean enabled() { return false; } - @Explain(vectorization = Vectorization.SUMMARY, displayName = "enabledConditionsNotMet", explainLevels = { Level.DEFAULT, Level.EXTENDED }) - public List enabledConditionsNotMet() { - return VectorizationCondition.getConditionsSupported(false); + private VectorizationCondition[] createMergeWorkExplainVectorizationConditions() { + + boolean enabled = false; + + VectorizationCondition[] conditions = new VectorizationCondition[] { + new VectorizationCondition( + enabled, + "Vectorizing Fetch Supported") + }; + return conditions; + } + + @Explain(vectorization = Vectorization.SUMMARY, displayName = "enableConditionsMet", + explainLevels = { Level.DEFAULT, Level.EXTENDED }) + public List getEnableConditionsMet() { + if (fetchWorkVectorizationConditions == null) { + fetchWorkVectorizationConditions = createMergeWorkExplainVectorizationConditions(); + } + return VectorizationCondition.getConditionsMet(fetchWorkVectorizationConditions); + } + + @Explain(vectorization = Vectorization.SUMMARY, displayName = "enableConditionsNotMet", + explainLevels = { Level.DEFAULT, Level.EXTENDED }) + public List getEnableConditionsNotMet() { + if (fetchWorkVectorizationConditions == null) { + fetchWorkVectorizationConditions = createMergeWorkExplainVectorizationConditions(); + } + return VectorizationCondition.getConditionsNotMet(fetchWorkVectorizationConditions); } } @@ -369,6 +396,7 @@ public FetchExplainVectorization getMapExplainVectorization() { } return new FetchExplainVectorization(this); } + @Explain(displayName = "Cached Query Result", displayOnlyOnTrue = true, explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public boolean isCachedResult() { return isCachedResult; diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java index f147309..5bf0625 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java @@ -468,7 +468,9 @@ private void setAliases() { @Explain(displayName = "Execution mode", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }, vectorization = Vectorization.SUMMARY_PATH) public String getExecutionMode() { - if (vectorMode) { + if (vectorMode && + !(getIsTestForcedVectorizationEnable() && + getIsTestVectorizationSuppressExplainExecutionMode())) { if (llapMode) { if (uberMode) { return "vectorized, uber"; diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java index b0ae64a..aa82d30 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java @@ -29,7 +29,9 @@ import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; +import org.apache.hadoop.hive.ql.plan.BaseWork.BaseExplainVectorization; import org.apache.hadoop.hive.ql.plan.Explain.Level; +import org.apache.hadoop.hive.ql.plan.Explain.Vectorization; import org.apache.hadoop.mapred.JobConf; public class MergeJoinWork extends BaseWork { @@ -175,8 +177,63 @@ public void setLlapMode(boolean llapMode) { public boolean getLlapMode() { return getMainWork().getLlapMode(); } - + public void addDummyOp(HashTableDummyOperator dummyOp) { getMainWork().addDummyOp(dummyOp); } + + /** + * For now, this class just says in EXPLAIN VECTORIZATION we don't support vectorization of the + * Merge Join vertex instead of being silent about it. + */ + public class MergeJoinExplainVectorization extends BaseExplainVectorization { + + private final MergeJoinWork mergeJoinWork; + + private VectorizationCondition[] mergeWorkVectorizationConditions; + + public MergeJoinExplainVectorization(MergeJoinWork mergeJoinWork) { + super(mergeJoinWork); + this.mergeJoinWork = mergeJoinWork; + } + + private VectorizationCondition[] createMergeWorkExplainVectorizationConditions() { + + boolean enabled = false; + + VectorizationCondition[] conditions = new VectorizationCondition[] { + new VectorizationCondition( + enabled, + "Vectorizing MergeJoin Supported") + }; + return conditions; + } + + @Explain(vectorization = Vectorization.SUMMARY, displayName = "enableConditionsMet", + explainLevels = { Level.DEFAULT, Level.EXTENDED }) + public List getEnableConditionsMet() { + if (mergeWorkVectorizationConditions == null) { + mergeWorkVectorizationConditions = createMergeWorkExplainVectorizationConditions(); + } + return VectorizationCondition.getConditionsMet(mergeWorkVectorizationConditions); + } + + @Explain(vectorization = Vectorization.SUMMARY, displayName = "enableConditionsNotMet", + explainLevels = { Level.DEFAULT, Level.EXTENDED }) + public List getEnableConditionsNotMet() { + if (mergeWorkVectorizationConditions == null) { + mergeWorkVectorizationConditions = createMergeWorkExplainVectorizationConditions(); + } + return VectorizationCondition.getConditionsNotMet(mergeWorkVectorizationConditions); + } + } + + @Explain(vectorization = Vectorization.SUMMARY, displayName = "MergeJoin Vectorization", + explainLevels = { Level.DEFAULT, Level.EXTENDED }) + public MergeJoinExplainVectorization getReduceExplainVectorization() { + if (!getVectorizationExamined()) { + return null; + } + return new MergeJoinExplainVectorization(this); + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java index ff5acbb..51298ce 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java @@ -126,15 +126,17 @@ public void setTagToValueDesc(final List tagToValueDesc) { @Explain(displayName = "Execution mode", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }, vectorization = Vectorization.SUMMARY_PATH) public String getExecutionMode() { - if (vectorMode) { + if (vectorMode && + !(getIsTestForcedVectorizationEnable() && + getIsTestVectorizationSuppressExplainExecutionMode())) { if (llapMode) { - if (uberMode) { - return "vectorized, uber"; - } else { - return "vectorized, llap"; - } + if (uberMode) { + return "vectorized, uber"; + } else { + return "vectorized, llap"; + } } else { - return "vectorized"; + return "vectorized"; } } else if (llapMode) { return uberMode? "uber" : "llap";