diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java deleted file mode 100644 index c299d3a..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExtractOperator.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec; - -import java.io.Serializable; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.ExtractDesc; -import org.apache.hadoop.hive.ql.plan.api.OperatorType; - -/** - * Extract operator implementation Extracts a subobject and passes that on. - **/ -public class ExtractOperator extends Operator implements - Serializable { - private static final long serialVersionUID = 1L; - protected transient ExprNodeEvaluator eval; - - @Override - protected void initializeOp(Configuration hconf) throws HiveException { - eval = ExprNodeEvaluatorFactory.get(conf.getCol()); - outputObjInspector = eval.initialize(inputObjInspectors[0]); - initializeChildren(hconf); - } - - @Override - public void processOp(Object row, int tag) throws HiveException { - forward(eval.evaluate(row), outputObjInspector); - } - - @Override - public OperatorType getType() { - return OperatorType.EXTRACT; - } - - @Override - public boolean acceptLimitPushdown() { - return true; - } - - /** - * @return the name of the operator - */ - @Override - public String getName() { - return getOperatorName(); - } - - static public String getOperatorName() { - return "EX"; - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java index f3c382a..b82fcb2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/OperatorFactory.java @@ -19,9 +19,6 @@ package org.apache.hadoop.hive.ql.exec; import org.apache.hadoop.hive.ql.exec.vector.VectorAppMasterEventOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorExtractOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorAppMasterEventOperator; -import org.apache.hadoop.hive.ql.exec.vector.VectorExtractOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorFileSinkOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorFilterOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator; @@ -39,7 +36,6 @@ import org.apache.hadoop.hive.ql.plan.DummyStoreDesc; import org.apache.hadoop.hive.ql.plan.DynamicPruningEventDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.ExtractDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; import org.apache.hadoop.hive.ql.plan.ForwardDesc; @@ -89,7 +85,6 @@ opvec.add(new OpTuple(ScriptDesc.class, ScriptOperator.class)); opvec.add(new OpTuple(PTFDesc.class, PTFOperator.class)); opvec.add(new OpTuple(ReduceSinkDesc.class, ReduceSinkOperator.class)); - opvec.add(new OpTuple(ExtractDesc.class, ExtractOperator.class)); opvec.add(new OpTuple(GroupByDesc.class, GroupByOperator.class)); opvec.add(new OpTuple(JoinDesc.class, JoinOperator.class)); opvec.add(new OpTuple(MapJoinDesc.class, MapJoinOperator.class)); @@ -143,7 +138,6 @@ vectorOpvec.add(new OpTuple(FileSinkDesc.class, VectorFileSinkOperator.class)); vectorOpvec.add(new OpTuple(FilterDesc.class, VectorFilterOperator.class)); vectorOpvec.add(new OpTuple(LimitDesc.class, VectorLimitOperator.class)); - vectorOpvec.add(new OpTuple(ExtractDesc.class, VectorExtractOperator.class)); } private static final class OpTuple { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java index 2e6a880..e95505c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java @@ -68,7 +68,6 @@ @Override protected void initializeOp(Configuration jobConf) throws HiveException { hiveConf = jobConf; - // if the parent is ExtractOperator, this invocation is from reduce-side isMapOperator = conf.isMapSide(); reconstructQueryDef(hiveConf); @@ -157,7 +156,7 @@ protected void setupKeysWrapper(ObjectInspector inputOI) throws HiveException { /* * Why cannot we just use the ExprNodeEvaluator on the column? * - because on the reduce-side it is initialized based on the rowOI of the HiveTable - * and not the OI of the ExtractOp ( the parent of this Operator on the reduce-side) + * and not the OI of the parent of this Operator on the reduce-side */ keyFields[i] = ExprNodeEvaluatorFactory.get(exprDef.getExprNode()); keyOIs[i] = keyFields[i].initialize(inputOI); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 9ed2c61..6b7c463 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -31,7 +31,6 @@ import java.io.DataInput; import java.io.EOFException; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractOperator.java deleted file mode 100644 index 7f4bb64..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractOperator.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.exec.vector; - -import java.util.List; -import java.util.ArrayList; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.exec.ExtractOperator; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.plan.ExtractDesc; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; - -/** - * Vectorized extract operator implementation. - **/ -public class VectorExtractOperator extends ExtractOperator implements VectorizationContextRegion { - private static final long serialVersionUID = 1L; - - private List reduceTypeInfos; - - // Create a new outgoing vectorization context because we will project just the values. - private VectorizationContext vOutContext; - - private int[] projectedColumns; - - private String removeValueDotPrefix(String columnName) { - return columnName.substring("VALUE.".length()); - } - public VectorExtractOperator(VectorizationContext vContext, OperatorDesc conf) - throws HiveException { - this(); - this.conf = (ExtractDesc) conf; - - List reduceColumnNames = vContext.getProjectionColumnNames(); - int reduceColCount = reduceColumnNames.size(); - - /* - * Create a new vectorization context as projection of just the values columns, but - * keep same output column manager must be inherited to track the scratch the columns. - */ - vOutContext = new VectorizationContext(vContext); - - // Set a fileKey with vectorization context. - vOutContext.setFileKey(vContext.getFileKey() + "/_EXTRACT_"); - - // Remove "VALUE." prefix from value columns and create a new projection - vOutContext.resetProjectionColumns(); - for (int i = 0; i < reduceColCount; i++) { - String columnName = reduceColumnNames.get(i); - if (columnName.startsWith("VALUE.")) { - vOutContext.addProjectionColumn(removeValueDotPrefix(columnName), i); - } - } - } - - public VectorExtractOperator() { - super(); - } - - /* - * Called by the Vectorizer class to pass the types from reduce shuffle. - */ - public void setReduceTypeInfos(List reduceTypeInfos) { - this.reduceTypeInfos = reduceTypeInfos; - } - - @Override - protected void initializeOp(Configuration hconf) throws HiveException { - // Create the projection of the values and the output object inspector - // for just the value without their "VALUE." prefix. - int projectionSize = vOutContext.getProjectedColumns().size(); - projectedColumns = new int[projectionSize]; - List columnNames = new ArrayList(); - List ois = new ArrayList(); - for (int i = 0; i < projectionSize; i++) { - int projectedIndex = vOutContext.getProjectedColumns().get(i); - projectedColumns[i] = projectedIndex; - String colName = vOutContext.getProjectionColumnNames().get(i); - columnNames.add(colName); - TypeInfo typeInfo = reduceTypeInfos.get(projectedIndex); - ObjectInspector oi = TypeInfoUtils - .getStandardWritableObjectInspectorFromTypeInfo(typeInfo); - ois.add(oi); - } - outputObjInspector = ObjectInspectorFactory. - getStandardStructObjectInspector(columnNames, ois); - initializeChildren(hconf); - } - - - @Override - // Remove the key columns and forward the values (and scratch columns). - public void processOp(Object row, int tag) throws HiveException { - VectorizedRowBatch vrg = (VectorizedRowBatch) row; - - int[] originalProjections = vrg.projectedColumns; - int originalProjectionSize = vrg.projectionSize; - - // Temporarily substitute our projection. - vrg.projectionSize = projectedColumns.length; - vrg.projectedColumns = projectedColumns; - - forward(vrg, null); - - // Revert the projected columns back, because vrg will be re-used. - vrg.projectionSize = originalProjectionSize; - vrg.projectedColumns = originalProjections; - } - - @Override - public VectorizationContext getOuputVectorizationContext() { - return vOutContext; - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java index 24ca89f..76cc540 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/BucketingSortingReduceSinkOptimizer.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; -import org.apache.hadoop.hive.ql.exec.ExtractOperator; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.Operator; @@ -84,7 +83,7 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { // process reduce sink added by hive.enforce.bucketing or hive.enforce.sorting opRules.put(new RuleRegExp("R1", ReduceSinkOperator.getOperatorName() + "%" + - ExtractOperator.getOperatorName() + "%" + + SelectOperator.getOperatorName() + "%" + FileSinkOperator.getOperatorName() + "%"), getBucketSortReduceSinkProc(pctx)); @@ -362,8 +361,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, // If the reduce sink has not been introduced due to bucketing/sorting, ignore it FileSinkOperator fsOp = (FileSinkOperator) nd; - ExtractOperator exOp = (ExtractOperator) fsOp.getParentOperators().get(0); - ReduceSinkOperator rsOp = (ReduceSinkOperator) exOp.getParentOperators().get(0); + ReduceSinkOperator rsOp = (ReduceSinkOperator) fsOp.getParentOperators().get(0).getParentOperators().get(0); List rsOps = pGraphContext .getReduceSinkOperatorsAddedByEnforceBucketingSorting(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index e16ba6c..e7cf835 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -29,14 +29,15 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.ql.exec.ColumnInfo; -import org.apache.hadoop.hive.ql.exec.ExtractOperator; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorFactory; import org.apache.hadoop.hive.ql.exec.OperatorUtils; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.exec.SelectOperator; import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.exec.Utilities.ReduceField; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; @@ -57,12 +58,12 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.ExtractDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.ListBucketingCtx; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; +import org.apache.hadoop.hive.ql.plan.SelectDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; @@ -232,21 +233,34 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, outRR, parseCtx); rsOp.setColumnExprMap(colExprMap); - // Create ExtractDesc + List valCols = rsConf.getValueCols(); + List descs = new ArrayList(valCols.size()); + List colNames = new ArrayList(); + String colName; + for (ExprNodeDesc valCol : valCols) { + if(new String("'"+BUCKET_NUMBER_COL_NAME+"'").equals(valCol.getExprString())) { + colName = BUCKET_NUMBER_COL_NAME; + } else { + colName = valCol.getExprString(); + } + + colNames.add(colName); + descs.add(new ExprNodeColumnDesc(valCol.getTypeInfo(), ReduceField.VALUE.toString()+"."+colName, null, false)); + } + // Create SelectDesc + SelectDesc selConf = new SelectDesc(descs, colNames); ObjectPair exPair = copyRowResolver(outRR); - RowResolver exRR = exPair.getSecond(); - ExtractDesc exConf = new ExtractDesc(new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, - Utilities.ReduceField.VALUE.toString(), "", false)); + RowResolver selRR = exPair.getSecond(); - // Create Extract Operator - ExtractOperator exOp = (ExtractOperator) putOpInsertMap( - OperatorFactory.getAndMakeChild(exConf, new RowSchema(exRR.getColumnInfos()), rsOp), - exRR, parseCtx); + // Create Select Operator + SelectOperator selOp = (SelectOperator) putOpInsertMap( + OperatorFactory.getAndMakeChild(selConf, new RowSchema(selRR.getColumnInfos()), rsOp), + selRR, parseCtx); - // link EX to FS + // link SEL to FS fsOp.getParentOperators().clear(); - fsOp.getParentOperators().add(exOp); - exOp.getChildOperators().add(fsOp); + fsOp.getParentOperators().add(selOp); + selOp.getChildOperators().add(fsOp); // Set if partition sorted or partition bucket sorted fsOp.getConf().setDpSortState(FileSinkDesc.DPSortState.PARTITION_SORTED); @@ -259,13 +273,13 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, .getSchema().getSignature()); fsOp.getConf().setPartitionCols(partitionColumns); - LOG.info("Inserted " + rsOp.getOperatorId() + " and " + exOp.getOperatorId() + LOG.info("Inserted " + rsOp.getOperatorId() + " and " + selOp.getOperatorId() + " as parent of " + fsOp.getOperatorId() + " and child of " + fsParent.getOperatorId()); return null; } - // Remove RS and EX introduced by enforce bucketing/sorting config - // Convert PARENT -> RS -> EX -> FS to PARENT -> FS + // Remove RS and SEL introduced by enforce bucketing/sorting config + // Convert PARENT -> RS -> SEL -> FS to PARENT -> FS private boolean removeRSInsertedByEnforceBucketing(FileSinkOperator fsOp) { HiveConf hconf = parseCtx.getConf(); boolean enforceBucketing = HiveConf.getBoolVar(hconf, ConfVars.HIVEENFORCEBUCKETING); @@ -300,7 +314,7 @@ private boolean removeRSInsertedByEnforceBucketing(FileSinkOperator fsOp) { Operator rsChild = rsToRemove.getChildOperators().get(0); Operator rsGrandChild = rsChild.getChildOperators().get(0); - if (rsChild instanceof ExtractOperator) { + if (rsChild instanceof SelectOperator) { // if schema size cannot be matched, then it could be because of constant folding // converting partition column expression to constant expression. The constant // expression will then get pruned by column pruner since it will not reference to diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java index dc906e8..5a5776a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationUtilities.java @@ -30,7 +30,6 @@ import java.util.Set; import org.apache.hadoop.hive.ql.exec.ColumnInfo; -import org.apache.hadoop.hive.ql.exec.ExtractOperator; import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.ForwardOperator; import org.apache.hadoop.hive.ql.exec.GroupByOperator; @@ -239,7 +238,6 @@ protected static int indexOf(ExprNodeDesc cexpr, ExprNodeDesc[] pexprs, Operator } if (!(cursor instanceof SelectOperator || cursor instanceof FilterOperator - || cursor instanceof ExtractOperator || cursor instanceof ForwardOperator || cursor instanceof ScriptOperator || cursor instanceof ReduceSinkOperator)) { @@ -351,7 +349,6 @@ protected static SelectOperator replaceOperatorWithSelect(Operator operator, SelectDesc select = new SelectDesc(null, null); Operator parent = getSingleParent(operator); - Operator child = getSingleChild(operator); parent.getChildOperators().clear(); @@ -365,10 +362,6 @@ protected static SelectOperator replaceOperatorWithSelect(Operator operator, for (Operator ch : operator.getChildOperators()) { ch.replaceParent(operator, sel); } - if (child instanceof ExtractOperator) { - removeOperator(child, getSingleChild(child), sel, context); - procCtx.addRemovedOperator(child); - } operator.setChildOperators(null); operator.setParentOperators(null); procCtx.addRemovedOperator(operator); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java index 3fead79..404b759 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.exec.ExtractOperator; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; @@ -146,6 +145,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, public abstract static class AbsctractReducerReducerProc implements NodeProcessor { + @Override public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { ReduceSinkDeduplicateProcCtx dedupCtx = (ReduceSinkDeduplicateProcCtx) procCtx; @@ -164,7 +164,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } return false; } - if (child instanceof ExtractOperator || child instanceof SelectOperator) { + if (child instanceof SelectOperator) { return process(cRS, dedupCtx); } return false; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java index d6a6ed6..3672d97 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.ql.exec.ColumnInfo; -import org.apache.hadoop.hive.ql.exec.ExtractOperator; import org.apache.hadoop.hive.ql.exec.ForwardOperator; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.JoinOperator; @@ -461,12 +460,6 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, lCtx.getIndex().putDependency(rop, col_infos.get(cnt++), ExprProcFactory.getExprDependency(lCtx, inpOp, expr)); } - } else if (op instanceof ExtractOperator) { - ArrayList col_infos = rop.getSchema().getSignature(); - for(ExprNodeDesc expr : rop.getConf().getValueCols()) { - lCtx.getIndex().putDependency(rop, col_infos.get(cnt++), - ExprProcFactory.getExprDependency(lCtx, inpOp, expr)); - } } else { RowResolver resolver = lCtx.getParseCtx().getOpParseCtx().get(rop).getRowResolver(); ReduceSinkDesc desc = rop.getConf(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java index 7954767..f370d4d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java @@ -23,7 +23,6 @@ import java.util.List; import java.util.Map; -import org.apache.hadoop.hive.ql.exec.ExtractOperator; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.FilterOperator; import org.apache.hadoop.hive.ql.exec.ForwardOperator; @@ -112,9 +111,6 @@ private void inferBucketingSorting(List mapRedTasks) throws Semantic BucketingSortingOpProcFactory.getJoinProc()); opRules.put(new RuleRegExp("R5", FileSinkOperator.getOperatorName() + "%"), BucketingSortingOpProcFactory.getFileSinkProc()); - // Matches only ExtractOperators which are reducers - opRules.put(new RuleExactMatch("R6", ExtractOperator.getOperatorName() + "%"), - BucketingSortingOpProcFactory.getExtractProc()); opRules.put(new RuleRegExp("R7", FilterOperator.getOperatorName() + "%"), BucketingSortingOpProcFactory.getFilterProc()); opRules.put(new RuleRegExp("R8", LimitOperator.getOperatorName() + "%"), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java index cf02bec..aa41200 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingOpProcFactory.java @@ -26,7 +26,6 @@ import java.util.Stack; import org.apache.hadoop.hive.ql.exec.ColumnInfo; -import org.apache.hadoop.hive.ql.exec.ExtractOperator; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.ForwardOperator; import org.apache.hadoop.hive.ql.exec.GroupByOperator; @@ -487,49 +486,13 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } - /** - * Processor for Extract operator. - * - * Only handles the case where the tree looks like - * - * ReduceSinkOperator --- ExtractOperator - * - * This is the case for distribute by, sort by, order by, cluster by operators. - */ - public static class ExtractInferrer extends DefaultInferrer implements NodeProcessor { - @Override - public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, - Object... nodeOutputs) throws SemanticException { - - BucketingSortingCtx bctx = (BucketingSortingCtx)procCtx; - ExtractOperator exop = (ExtractOperator)nd; - - // As of writing this, there is no case where this could be false, this is just protection - // from possible future changes - if (exop.getParentOperators().size() != 1) { - return null; - } - - Operator parent = exop.getParentOperators().get(0); - - // The caller of this method should guarantee this - if (parent instanceof ReduceSinkOperator) { - extractTraits(bctx, (ReduceSinkOperator)parent, exop); - } - - return null; - } - } - - static void extractTraits(BucketingSortingCtx bctx, ReduceSinkOperator rop, Operator exop) + static void extractTraits(BucketingSortingCtx bctx, ReduceSinkOperator rop, Operator childop) throws SemanticException { List outputValues = Collections.emptyList(); - if (exop instanceof ExtractOperator) { - outputValues = rop.getConf().getValueCols(); - } else if (exop instanceof SelectOperator) { - SelectDesc select = ((SelectOperator)exop).getConf(); - outputValues = ExprNodeDescUtils.backtrack(select.getColList(), exop, rop); + if (childop instanceof SelectOperator) { + SelectDesc select = ((SelectOperator)childop).getConf(); + outputValues = ExprNodeDescUtils.backtrack(select.getColList(), childop, rop); } if (outputValues.isEmpty()) { return; @@ -543,16 +506,16 @@ static void extractTraits(BucketingSortingCtx bctx, ReduceSinkOperator rop, Oper // These represent the sorted columns List sortCols = extractSortCols(rop, outputValues); - List colInfos = exop.getSchema().getSignature(); + List colInfos = childop.getSchema().getSignature(); if (!bucketCols.isEmpty()) { List newBucketCols = getNewBucketCols(bucketCols, colInfos); - bctx.setBucketedCols(exop, newBucketCols); + bctx.setBucketedCols(childop, newBucketCols); } if (!sortCols.isEmpty()) { List newSortCols = getNewSortCols(sortCols, colInfos); - bctx.setSortedCols(exop, newSortCols); + bctx.setSortedCols(childop, newSortCols); } } @@ -778,10 +741,6 @@ public static NodeProcessor getFileSinkProc() { return new FileSinkInferrer(); } - public static NodeProcessor getExtractProc() { - return new ExtractInferrer(); - } - public static NodeProcessor getFilterProc() { return new ForwardingInferrer(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 94b4621..96901c9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hive.ql.exec.mr.MapRedTask; import org.apache.hadoop.hive.ql.exec.spark.SparkTask; import org.apache.hadoop.hive.ql.exec.tez.TezTask; -import org.apache.hadoop.hive.ql.exec.vector.VectorExtractOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext; @@ -76,7 +75,6 @@ import org.apache.hadoop.hive.ql.plan.ReduceWork; import org.apache.hadoop.hive.ql.plan.SMBJoinDesc; import org.apache.hadoop.hive.ql.plan.SparkWork; -import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.TezWork; import org.apache.hadoop.hive.ql.plan.VectorGroupByDesc; @@ -283,7 +281,7 @@ public Vectorizer() { class VectorizationDispatcher implements Dispatcher { - private PhysicalContext pctx; + private final PhysicalContext pctx; private List reduceColumnNames; private List reduceTypeInfos; @@ -449,9 +447,8 @@ private boolean getOnlyStructObjectInspectors(ReduceWork reduceWork) throws Sema } private void addReduceWorkRules(Map opRules, NodeProcessor np) { - opRules.put(new RuleRegExp("R1", ExtractOperator.getOperatorName() + ".*"), np); - opRules.put(new RuleRegExp("R2", GroupByOperator.getOperatorName() + ".*"), np); - opRules.put(new RuleRegExp("R3", SelectOperator.getOperatorName() + ".*"), np); + opRules.put(new RuleRegExp("R1", GroupByOperator.getOperatorName() + ".*"), np); + opRules.put(new RuleRegExp("R2", SelectOperator.getOperatorName() + ".*"), np); } private boolean validateReduceWork(ReduceWork reduceWork) throws SemanticException { @@ -485,7 +482,7 @@ private boolean validateReduceWork(ReduceWork reduceWork) throws SemanticExcepti private void vectorizeReduceWork(ReduceWork reduceWork) throws SemanticException { LOG.info("Vectorizing ReduceWork..."); reduceWork.setVectorMode(true); - + // For some reason, the DefaultGraphWalker does not descend down from the reducer Operator as // expected. We need to descend down, otherwise it breaks our algorithm that determines // VectorizationContext... Do we use PreOrderWalker instead of DefaultGraphWalker. @@ -506,11 +503,6 @@ private void vectorizeReduceWork(ReduceWork reduceWork) throws SemanticException // Necessary since we are vectorizing the root operator in reduce. reduceWork.setReducer(vnp.getRootVectorOp()); - Operator reducer = reduceWork.getReducer(); - if (reducer.getType().equals(OperatorType.EXTRACT)) { - ((VectorExtractOperator)reducer).setReduceTypeInfos(reduceTypeInfos); - } - Map> allScratchColumnVectorTypeMaps = vnp.getAllScratchColumnVectorTypeMaps(); reduceWork.setAllScratchColumnVectorTypeMaps(allScratchColumnVectorTypeMaps); Map> allColumnVectorMaps = vnp.getAllColumnVectorMaps(); @@ -525,8 +517,8 @@ private void vectorizeReduceWork(ReduceWork reduceWork) throws SemanticException class MapWorkValidationNodeProcessor implements NodeProcessor { - private MapWork mapWork; - private boolean isTez; + private final MapWork mapWork; + private final boolean isTez; public MapWorkValidationNodeProcessor(MapWork mapWork, boolean isTez) { this.mapWork = mapWork; @@ -658,7 +650,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, throw new SemanticException("Must be overridden"); } } - + class MapWorkVectorizationNodeProcessor extends VectorizationNodeProcessor { private final MapWork mWork; @@ -723,8 +715,6 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, if (LOG.isDebugEnabled()) { LOG.debug("Vectorized MapWork operator " + vectorOp.getName() + " vectorization context " + vContext.toString()); if (vectorOp instanceof VectorizationContextRegion) { - VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp; - VectorizationContext vOutContext = vcRegion.getOuputVectorizationContext(); LOG.debug("Vectorized MapWork operator " + vectorOp.getName() + " added vectorization context " + vContext.toString()); } } @@ -735,8 +725,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, class ReduceWorkVectorizationNodeProcessor extends VectorizationNodeProcessor { - private List reduceColumnNames; - + private final List reduceColumnNames; + private VectorizationContext reduceShuffleVectorizationContext; private Operator rootVectorOp; @@ -801,8 +791,6 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, if (LOG.isDebugEnabled()) { LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() + " vectorization context " + vContext.toString()); if (vectorOp instanceof VectorizationContextRegion) { - VectorizationContextRegion vcRegion = (VectorizationContextRegion) vectorOp; - VectorizationContext vOutContext = vcRegion.getOuputVectorizationContext(); LOG.debug("Vectorized ReduceWork operator " + vectorOp.getName() + " added vectorization context " + vContext.toString()); } } @@ -897,9 +885,6 @@ boolean validateMapWorkOperator(Operator op, MapWork mWo boolean validateReduceWorkOperator(Operator op) { boolean ret = false; switch (op.getType()) { - case EXTRACT: - ret = validateExtractOperator((ExtractOperator) op); - break; case MAPJOIN: // Does MAPJOIN actually get planned in Reduce? if (op instanceof MapJoinOperator) { @@ -1034,7 +1019,7 @@ private boolean validateMapJoinOperator(MapJoinOperator op) { MapJoinDesc desc = op.getConf(); return validateMapJoinDesc(desc); } - + private boolean validateMapJoinDesc(MapJoinDesc desc) { byte posBigTable = (byte) desc.getPosBigTable(); List filterExprs = desc.getFilters().get(posBigTable); @@ -1123,15 +1108,6 @@ private boolean validateGroupByOperator(GroupByOperator op, boolean isReduce, bo return true; } - private boolean validateExtractOperator(ExtractOperator op) { - ExprNodeDesc expr = op.getConf().getCol(); - boolean ret = validateExprNodeDesc(expr); - if (!ret) { - return false; - } - return true; - } - private boolean validateFileSinkOperator(FileSinkOperator op) { return true; } @@ -1300,7 +1276,7 @@ private VectorizationContext getVectorizationContext(Operator op, return vContext; } - private void fixupParentChildOperators(Operator op, + private void fixupParentChildOperators(Operator op, Operator vectorOp) { if (op.getParentOperators() != null) { vectorOp.setParentOperators(op.getParentOperators()); @@ -1354,7 +1330,7 @@ private boolean isVirtualColumn(ColumnInfo column) { return false; } - public void debugDisplayAllMaps(Map> allColumnVectorMaps, + public void debugDisplayAllMaps(Map> allColumnVectorMaps, Map> allScratchColumnVectorTypeMaps) { // Context keys grow in length since they are a path... diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 4364f28..64a325d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -94,6 +94,7 @@ import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.AcidOutputFormat; import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.io.AcidUtils.Operation; import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; @@ -149,7 +150,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.ExtractDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; @@ -229,12 +229,12 @@ private HashMap opToPartPruner; private HashMap opToPartList; private HashMap> topOps; - private HashMap> topSelOps; + private final HashMap> topSelOps; private LinkedHashMap, OpParseContext> opParseCtx; private List loadTableWork; private List loadFileWork; - private Map joinContext; - private Map smbMapJoinContext; + private final Map joinContext; + private final Map smbMapJoinContext; private final HashMap topToTable; private final Map fsopToTable; private final List reduceSinkOperatorsAddedByEnforceBucketingSorting; @@ -5980,8 +5980,13 @@ private Operator genBucketingSortingDest(String dest, Operator input, QB qb, maxReducers = numBuckets; } - input = genReduceSinkPlanForSortingBucketing(dest_tab, input, - sortCols, sortOrders, partnCols, maxReducers); + StringBuilder order = new StringBuilder(); + for (int sortOrder : sortOrders) { + order.append(sortOrder == BaseSemanticAnalyzer.HIVE_COLUMN_ORDER_ASC ? '+' : '-'); + } + input = genReduceSinkPlan(input, partnCols, sortCols, order.toString(), maxReducers, + (isAcidTable(dest_tab) ? getAcidType() : AcidUtils.Operation.NOT_ACID)); + reduceSinkOperatorsAddedByEnforceBucketingSorting.add((ReduceSinkOperator)input.getParentOperators().get(0)); ctx.setMultiFileSpray(multiFileSpray); ctx.setNumFiles(numFiles); ctx.setPartnCols(partnColsNoConvert); @@ -6453,7 +6458,7 @@ private Operator genFileSinkPlan(String dest, QB qb, Operator input) fileSinkDesc.setWriteType(wt); acidFileSinks.add(fileSinkDesc); } - + fileSinkDesc.setTemporary(destTableIsTemporary); /* Set List Bucketing context. */ @@ -6935,7 +6940,6 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, private ArrayList getSortCols(String dest, QB qb, Table tab, TableDesc table_desc, Operator input, boolean convert) throws SemanticException { - RowResolver inputRR = opParseCtx.get(input).getRowResolver(); List tabSortCols = tab.getSortCols(); List tabCols = tab.getCols(); @@ -6945,7 +6949,6 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, int pos = 0; for (FieldSchema tabCol : tabCols) { if (sortCol.getCol().equals(tabCol.getName())) { - ColumnInfo colInfo = inputRR.getColumnInfos().get(pos); posns.add(pos); break; } @@ -6958,7 +6961,6 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, private ArrayList getSortOrders(String dest, QB qb, Table tab, Operator input) throws SemanticException { - RowResolver inputRR = opParseCtx.get(input).getRowResolver(); List tabSortCols = tab.getSortCols(); List tabCols = tab.getCols(); @@ -6974,74 +6976,11 @@ private Operator genLimitMapRedPlan(String dest, QB qb, Operator input, return orders; } - @SuppressWarnings("nls") - private Operator genReduceSinkPlanForSortingBucketing(Table tab, Operator input, - ArrayList sortCols, - List sortOrders, - ArrayList partitionCols, - int numReducers) - throws SemanticException { - RowResolver inputRR = opParseCtx.get(input).getRowResolver(); - - // For the generation of the values expression just get the inputs - // signature and generate field expressions for those - Map colExprMap = new HashMap(); - ArrayList valueCols = new ArrayList(); - ArrayList outputColumns = new ArrayList(); - int i = 0; - for (ColumnInfo colInfo : inputRR.getColumnInfos()) { - String internalName = getColumnInternalName(i++); - outputColumns.add(internalName); - valueCols.add(new ExprNodeColumnDesc(colInfo)); - colExprMap.put(internalName, valueCols - .get(valueCols.size() - 1)); - } - - StringBuilder order = new StringBuilder(); - for (int sortOrder : sortOrders) { - order.append(sortOrder == BaseSemanticAnalyzer.HIVE_COLUMN_ORDER_ASC ? '+' : '-'); - } - - AcidUtils.Operation acidOp = (isAcidTable(tab) ? getAcidType() : AcidUtils.Operation.NOT_ACID); - - Operator interim = putOpInsertMap(OperatorFactory.getAndMakeChild(PlanUtils - .getReduceSinkDesc(sortCols, valueCols, outputColumns, false, -1, - partitionCols, order.toString(), numReducers, acidOp), - new RowSchema(inputRR.getColumnInfos()), input), inputRR); - interim.setColumnExprMap(colExprMap); - reduceSinkOperatorsAddedByEnforceBucketingSorting.add((ReduceSinkOperator) interim); - - // Add the extract operator to get the value fields - RowResolver out_rwsch = new RowResolver(); - RowResolver interim_rwsch = inputRR; - Integer pos = Integer.valueOf(0); - for (ColumnInfo colInfo : interim_rwsch.getColumnInfos()) { - String[] info = interim_rwsch.reverseLookup(colInfo.getInternalName()); - out_rwsch.put(info[0], info[1], new ColumnInfo( - getColumnInternalName(pos), colInfo.getType(), info[0], - colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol())); - pos = Integer.valueOf(pos.intValue() + 1); - } - - Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild( - new ExtractDesc(new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, - Utilities.ReduceField.VALUE.toString(), "", false)), new RowSchema( - out_rwsch.getColumnInfos()), interim), out_rwsch); - - if (LOG.isDebugEnabled()) { - LOG.debug("Created ReduceSink Plan for table: " + tab.getTableName() + - " row schema: " + out_rwsch.toString()); - } - - return output; - - } - private Operator genReduceSinkPlan(String dest, QB qb, Operator input, int numReducers) throws SemanticException { - + RowResolver inputRR = opParseCtx.get(input).getRowResolver(); - + // First generate the expression for the partition and sort keys // The cluster by clause / distribute by clause has the aliases for // partition function @@ -7099,16 +7038,16 @@ private Operator genReduceSinkPlan(String dest, QB qb, Operator input, sortCols.add(exprNode); } } - return genReduceSinkPlan(input, partCols, sortCols, order.toString(), numReducers); + return genReduceSinkPlan(input, partCols, sortCols, order.toString(), numReducers, Operation.NOT_ACID); } - + @SuppressWarnings("nls") private Operator genReduceSinkPlan(Operator input, - ArrayList partitionCols, ArrayList sortCols, - String sortOrder, int numReducers) throws SemanticException { + ArrayList partitionCols, ArrayList sortCols, + String sortOrder, int numReducers, AcidUtils.Operation acidOp) throws SemanticException { RowResolver inputRR = opParseCtx.get(input).getRowResolver(); - + Operator dummy = Operator.createDummy(); dummy.setParentOperators(Arrays.asList(input)); @@ -7171,9 +7110,8 @@ private Operator genReduceSinkPlan(Operator input, dummy.setParentOperators(null); - // TODO Not 100% sure NOT_ACID is always right here. ReduceSinkDesc rsdesc = PlanUtils.getReduceSinkDesc(sortCols, valueCols, outputColumns, - false, -1, partitionCols, sortOrder, numReducers, AcidUtils.Operation.NOT_ACID); + false, -1, partitionCols, sortOrder, numReducers, acidOp); Operator interim = putOpInsertMap(OperatorFactory.getAndMakeChild(rsdesc, new RowSchema(rsRR.getColumnInfos()), input), rsRR); @@ -11899,13 +11837,8 @@ Operator genPTFPlan(PTFInvocationSpec ptfQSpec, Operator input) throws SemanticE void buildPTFReduceSinkDetails(PartitionedTableFunctionDef tabDef, RowResolver inputRR, ArrayList partCols, - ArrayList valueCols, ArrayList orderCols, - Map colExprMap, - List outputColumnNames, - StringBuilder orderString, - RowResolver rsOpRR, - RowResolver extractRR) throws SemanticException { + StringBuilder orderString) throws SemanticException { List partColList = tabDef.getPartition().getExpressions(); @@ -11933,68 +11866,6 @@ void buildPTFReduceSinkDetails(PartitionedTableFunctionDef tabDef, } orderCols.add(colDef.getExprNode()); } - - ArrayList colInfoList = inputRR.getColumnInfos(); - /* - * construct the ReduceSinkRR - */ - int pos = 0; - for (ColumnInfo colInfo : colInfoList) { - ExprNodeDesc valueColExpr = new ExprNodeColumnDesc(colInfo); - valueCols.add(valueColExpr); - String internalName = SemanticAnalyzer.getColumnInternalName(pos++); - outputColumnNames.add(internalName); - colExprMap.put(internalName, valueColExpr); - - String[] alias = inputRR.reverseLookup(colInfo.getInternalName()); - ColumnInfo newColInfo = new ColumnInfo( - internalName, colInfo.getType(), alias[0], - colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol()); - rsOpRR.put(alias[0], alias[1], newColInfo); - } - - /* - * construct the ExtractRR - */ - LinkedHashMap colsAddedByHaving = - new LinkedHashMap(); - pos = 0; - for (ColumnInfo colInfo : colInfoList) { - String[] alias = inputRR.reverseLookup(colInfo.getInternalName()); - /* - * if we have already encountered this colInfo internalName. - * We encounter it again because it must be put for the Having clause. - * We will add these entries in the end; in a loop on colsAddedByHaving. See below. - */ - if ( colsAddedByHaving.containsKey(alias)) { - continue; - } - ASTNode astNode = PTFTranslator.getASTNode(colInfo, inputRR); - ColumnInfo eColInfo = new ColumnInfo( - SemanticAnalyzer.getColumnInternalName(pos++), colInfo.getType(), alias[0], - colInfo.getIsVirtualCol(), colInfo.isHiddenVirtualCol()); - - if ( astNode == null ) { - extractRR.put(alias[0], alias[1], eColInfo); - } - else { - /* - * in case having clause refers to this column may have been added twice; - * once with the ASTNode.toStringTree as the alias - * and then with the real alias. - */ - extractRR.putExpression(astNode, eColInfo); - if ( !astNode.toStringTree().toLowerCase().equals(alias[1]) ) { - colsAddedByHaving.put(alias, eColInfo); - } - } - } - - for(Map.Entry columnAddedByHaving : colsAddedByHaving.entrySet() ) { - String[] alias = columnAddedByHaving.getKey(); - ColumnInfo eColInfo = columnAddedByHaving.getValue(); - extractRR.put(alias[0], alias[1], eColInfo); - } } private Operator genPTFPlanForComponentQuery(PTFInvocationSpec ptfQSpec, Operator input) @@ -12005,27 +11876,6 @@ private Operator genPTFPlanForComponentQuery(PTFInvocationSpec ptfQSpec, Operato RowResolver rr = opParseCtx.get(input).getRowResolver(); PTFDesc ptfDesc = translatePTFInvocationSpec(ptfQSpec, rr); - RowResolver rsOpRR = new RowResolver(); - /* - * Build an RR for the Extract Op from the ReduceSink Op's RR. - * Why? - * We need to remove the Virtual Columns present in the RS's RR. The OI - * that gets passed to Extract at runtime doesn't contain the Virtual Columns. - * So internal names get changed. Consider testCase testJoinWithLeadLag, - * which is a self join on part and also has a Windowing expression. - * The RR of the RS op at translation time looks something like this: - * (_co1,_col2,..,_col7, _col8(vc=true),_col9(vc=true), - * _col10,_col11,.._col15(vc=true),_col16(vc=true),..) - * At runtime the Virtual columns are removed and all the columns after _col7 - * are shifted 1 or 2 positions. - * So in child Operators ColumnExprNodeDesc's are no longer referring to the right columns. - * - * So we build a new RR for the Extract Op, with the Virtual Columns removed. - * We hand this to the PTFTranslator as the - * starting RR to use to translate a PTF Chain. - */ - RowResolver extractOpRR = new RowResolver(); - /* * 2. build Map-side Op Graph. Graph template is either: * Input -> PTF_map -> ReduceSink @@ -12056,10 +11906,7 @@ private Operator genPTFPlanForComponentQuery(PTFInvocationSpec ptfQSpec, Operato */ ArrayList partCols = new ArrayList(); - ArrayList valueCols = new ArrayList(); ArrayList orderCols = new ArrayList(); - Map colExprMap = new HashMap(); - List outputColumnNames = new ArrayList(); StringBuilder orderString = new StringBuilder(); /* @@ -12068,45 +11915,20 @@ private Operator genPTFPlanForComponentQuery(PTFInvocationSpec ptfQSpec, Operato * If the parent of ReduceSinkOperator is PTFOperator, use it's * output RR. */ - buildPTFReduceSinkDetails(tabDef, - rr, - partCols, - valueCols, - orderCols, - colExprMap, - outputColumnNames, - orderString, - rsOpRR, - extractOpRR); - - input = putOpInsertMap(OperatorFactory.getAndMakeChild(PlanUtils - .getReduceSinkDesc(orderCols, - valueCols, outputColumnNames, false, - -1, partCols, orderString.toString(), -1, AcidUtils.Operation.NOT_ACID), - new RowSchema(rsOpRR.getColumnInfos()), input), rsOpRR); - input.setColumnExprMap(colExprMap); + buildPTFReduceSinkDetails(tabDef, rr, partCols, orderCols, orderString); + input = genReduceSinkPlan(input, partCols, orderCols, orderString.toString(), -1, Operation.NOT_ACID); } /* * 3. build Reduce-side Op Graph */ { - /* - * b. Construct Extract Operator. - */ - input = putOpInsertMap(OperatorFactory.getAndMakeChild( - new ExtractDesc( - new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, - Utilities.ReduceField.VALUE - .toString(), "", false)), - new RowSchema(extractOpRR.getColumnInfos()), - input), extractOpRR); /* * c. Rebuilt the QueryDef. * Why? * - so that the ExprNodeDescriptors in the QueryDef are based on the - * Extract Operator's RowResolver + * Select Operator's RowResolver */ rr = opParseCtx.get(input).getRowResolver(); ptfDesc = translatePTFInvocationSpec(ptfQSpec, rr); @@ -12120,9 +11942,7 @@ private Operator genPTFPlanForComponentQuery(PTFInvocationSpec ptfQSpec, Operato input), ptfOpRR); } - return input; - } //--------------------------- Windowing handling: PTFInvocationSpec to PTFDesc -------------------- @@ -12150,7 +11970,7 @@ Operator genWindowingPlan(WindowingSpec wSpec, Operator input) throws SemanticEx private Operator genReduceSinkPlanForWindowing(WindowingSpec spec, RowResolver inputRR, Operator input) throws SemanticException{ - + ArrayList partCols = new ArrayList(); ArrayList orderCols = new ArrayList(); StringBuilder order = new StringBuilder(); @@ -12174,7 +11994,7 @@ private Operator genReduceSinkPlanForWindowing(WindowingSpec spec, } } - return genReduceSinkPlan(input, partCols, orderCols, order.toString(), -1); + return genReduceSinkPlan(input, partCols, orderCols, order.toString(), -1, Operation.NOT_ACID); } public static ArrayList parseSelect(String selectExprStr) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExtractDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExtractDesc.java deleted file mode 100644 index 6762155..0000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExtractDesc.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - - -/** - * ExtractDesc. - * - */ -@Explain(displayName = "Extract") -public class ExtractDesc extends AbstractOperatorDesc { - private static final long serialVersionUID = 1L; - private ExprNodeDesc col; - - public ExtractDesc() { - } - - public ExtractDesc(final ExprNodeDesc col) { - this.col = col; - } - - public ExprNodeDesc getCol() { - return col; - } - - public void setCol(final ExprNodeDesc col) { - this.col = col; - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java index fa6b548..cfcfe17 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/SelectDesc.java @@ -56,14 +56,6 @@ public SelectDesc( this.outputColumnNames = outputColumnNames; } - public SelectDesc( - final List colList, - final boolean selectStar, final boolean selStarNoCompute) { - this.colList = colList; - this.selectStar = selectStar; - this.selStarNoCompute = selStarNoCompute; - } - @Override public Object clone() { SelectDesc ret = new SelectDesc(); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java index 41862e6..730823f 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hive.ql.exec.mr.ExecDriver; import org.apache.hadoop.hive.ql.exec.mr.MapRedTask; import org.apache.hadoop.hive.ql.io.AcidUtils; -import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat; +import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -47,7 +47,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.ExtractDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; import org.apache.hadoop.hive.ql.plan.MapredWork; @@ -94,7 +93,7 @@ tmppath = new Path(tmpdir); fs = FileSystem.get(conf); - if (fs.exists(tmppath) && !fs.getFileStatus(tmppath).isDir()) { + if (fs.exists(tmppath) && !fs.getFileStatus(tmppath).isDirectory()) { throw new RuntimeException(tmpdir + " exists but is not a directory"); } @@ -137,7 +136,7 @@ for (String src : srctables) { db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true); db.createTable(src, cols, null, TextInputFormat.class, - IgnoreKeyTextOutputFormat.class); + HiveIgnoreKeyTextOutputFormat.class); db.loadTable(hadoopDataFile[i], src, false, false, true, false, false); i++; } @@ -161,20 +160,19 @@ public static void addMapWork(MapredWork mr, Table tbl, String alias, Operator op3 = OperatorFactory.get(new FileSinkDesc(new Path(tmpdir + File.separator + "mapredplan1.out"), Utilities.defaultTd, false)); - Operator op2 = OperatorFactory.get(new ExtractDesc( - getStringColumn(Utilities.ReduceField.VALUE.toString())), op3); + List cols = new ArrayList(); + cols.add(getStringColumn(Utilities.ReduceField.VALUE.toString()+"."+outputColumns.get(1))); + List colNames = new ArrayList(); + colNames.add(HiveConf.getColumnInternalName(2)); + Operator op2 = OperatorFactory.get(new SelectDesc(cols, colNames), op3); rWork.setReducer(op2); } @@ -292,8 +293,10 @@ private void populateMapRedPlan2(Table src) throws Exception { Operator op3 = OperatorFactory.get(getTestFilterDesc("0"), op4); - Operator op2 = OperatorFactory.get(new ExtractDesc( - getStringColumn(Utilities.ReduceField.VALUE.toString())), op3); + List cols = new ArrayList(); + cols.add(getStringColumn(Utilities.ReduceField.KEY + ".reducesinkkey" + 0)); + cols.add(getStringColumn(Utilities.ReduceField.VALUE.toString()+"."+outputColumns.get(1))); + Operator op2 = OperatorFactory.get(new SelectDesc(cols, outputColumns), op3); rWork.setReducer(op2); } @@ -376,10 +379,10 @@ private void populateMapRedPlan4(Table src) throws SemanticException { // reduce side work Operator op3 = OperatorFactory.get(new FileSinkDesc(new Path(tmpdir + File.separator + "mapredplan4.out"), Utilities.defaultTd, false)); - - Operator op2 = OperatorFactory.get(new ExtractDesc( - getStringColumn(Utilities.ReduceField.VALUE.toString())), op3); - + List cols = new ArrayList(); + cols.add(getStringColumn(Utilities.ReduceField.KEY + ".reducesinkkey" + 0)); + cols.add(getStringColumn(Utilities.ReduceField.VALUE.toString()+"."+outputColumns.get(1))); + Operator op2 = OperatorFactory.get(new SelectDesc(cols, outputColumns), op3); rWork.setReducer(op2); } @@ -416,9 +419,10 @@ private void populateMapRedPlan5(Table src) throws SemanticException { Operator op3 = OperatorFactory.get(new FileSinkDesc(new Path(tmpdir + File.separator + "mapredplan5.out"), Utilities.defaultTd, false)); - Operator op2 = OperatorFactory.get(new ExtractDesc( - getStringColumn(Utilities.ReduceField.VALUE.toString())), op3); - + List cols = new ArrayList(); + cols.add(getStringColumn(Utilities.ReduceField.KEY + ".reducesinkkey" + 0)); + cols.add(getStringColumn(Utilities.ReduceField.VALUE.toString()+"."+outputColumns.get(1))); + Operator op2 = OperatorFactory.get(new SelectDesc(cols, outputColumns), op3); rWork.setReducer(op2); } @@ -459,8 +463,10 @@ private void populateMapRedPlan6(Table src) throws Exception { Operator op2 = OperatorFactory.get(getTestFilterDesc("0"), op3); - Operator op5 = OperatorFactory.get(new ExtractDesc( - getStringColumn(Utilities.ReduceField.VALUE.toString())), op2); + List cols = new ArrayList(); + cols.add(getStringColumn(Utilities.ReduceField.KEY + ".reducesinkkey" + 0)); + cols.add(getStringColumn(Utilities.ReduceField.VALUE.toString()+"."+outputColumns.get(1))); + Operator op5 = OperatorFactory.get(new SelectDesc(cols, outputColumns), op2); rWork.setReducer(op5); } diff --git a/ql/src/test/results/clientpositive/bucket1.q.out b/ql/src/test/results/clientpositive/bucket1.q.out index 13ec735..8009514 100644 --- a/ql/src/test/results/clientpositive/bucket1.q.out +++ b/ql/src/test/results/clientpositive/bucket1.q.out @@ -110,39 +110,37 @@ STAGE PLANS: /src [src] Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 100 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 100 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.bucket1_1 - serialization.ddl struct bucket1_1 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.bucket1_1 + serialization.ddl struct bucket1_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket1_1 - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket1_1 + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/bucket2.q.out b/ql/src/test/results/clientpositive/bucket2.q.out index 32a77c3..75de8a9 100644 --- a/ql/src/test/results/clientpositive/bucket2.q.out +++ b/ql/src/test/results/clientpositive/bucket2.q.out @@ -110,39 +110,37 @@ STAGE PLANS: /src [src] Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.bucket2_1 - serialization.ddl struct bucket2_1 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.bucket2_1 + serialization.ddl struct bucket2_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket2_1 - TotalFiles: 2 - GatherStats: true - MultiFileSpray: true + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket2_1 + TotalFiles: 2 + GatherStats: true + MultiFileSpray: true Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/bucket3.q.out b/ql/src/test/results/clientpositive/bucket3.q.out index ff7173e..c459870 100644 --- a/ql/src/test/results/clientpositive/bucket3.q.out +++ b/ql/src/test/results/clientpositive/bucket3.q.out @@ -114,42 +114,40 @@ STAGE PLANS: /src [src] Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 2 - Static Partition Specification: ds=1/ - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 2 + Static Partition Specification: ds=1/ + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.bucket3_1 - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket3_1 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.bucket3_1 + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket3_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket3_1 - TotalFiles: 2 - GatherStats: true - MultiFileSpray: true + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket3_1 + TotalFiles: 2 + GatherStats: true + MultiFileSpray: true Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/bucket4.q.out b/ql/src/test/results/clientpositive/bucket4.q.out index b99d12f..9eeb13d 100644 --- a/ql/src/test/results/clientpositive/bucket4.q.out +++ b/ql/src/test/results/clientpositive/bucket4.q.out @@ -107,40 +107,38 @@ STAGE PLANS: /src [src] Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.bucket4_1 - serialization.ddl struct bucket4_1 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.bucket4_1 + serialization.ddl struct bucket4_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket4_1 - TotalFiles: 2 - GatherStats: true - MultiFileSpray: true + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket4_1 + TotalFiles: 2 + GatherStats: true + MultiFileSpray: true Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/bucket5.q.out b/ql/src/test/results/clientpositive/bucket5.q.out index 5992d6d..0c8418d 100644 --- a/ql/src/test/results/clientpositive/bucket5.q.out +++ b/ql/src/test/results/clientpositive/bucket5.q.out @@ -171,40 +171,38 @@ STAGE PLANS: /src [src] Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 1 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.bucketed_table - serialization.ddl struct bucketed_table { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.bucketed_table + serialization.ddl struct bucketed_table { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucketed_table - TotalFiles: 1 - GatherStats: true - MultiFileSpray: false + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucketed_table + TotalFiles: 1 + GatherStats: true + MultiFileSpray: false Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/bucket6.q.out b/ql/src/test/results/clientpositive/bucket6.q.out index 5b23d7d..72fec49 100644 --- a/ql/src/test/results/clientpositive/bucket6.q.out +++ b/ql/src/test/results/clientpositive/bucket6.q.out @@ -33,9 +33,11 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string) + value expressions: _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/bucketsortoptimize_insert_1.q.out b/ql/src/test/results/clientpositive/bucketsortoptimize_insert_1.q.out index 75de953..9faa0d0 100644 --- a/ql/src/test/results/clientpositive/bucketsortoptimize_insert_1.q.out +++ b/ql/src/test/results/clientpositive/bucketsortoptimize_insert_1.q.out @@ -343,9 +343,11 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/bucketsortoptimize_insert_2.q.out b/ql/src/test/results/clientpositive/bucketsortoptimize_insert_2.q.out index 599b8b9..eec099c 100644 --- a/ql/src/test/results/clientpositive/bucketsortoptimize_insert_2.q.out +++ b/ql/src/test/results/clientpositive/bucketsortoptimize_insert_2.q.out @@ -274,11 +274,13 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: @@ -340,11 +342,13 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: @@ -376,9 +380,11 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: @@ -989,11 +995,13 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: @@ -1061,11 +1069,13 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: @@ -1101,9 +1111,11 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: diff --git a/ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out b/ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out index 7456ab0..e778e35 100644 --- a/ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out +++ b/ql/src/test/results/clientpositive/bucketsortoptimize_insert_3.q.out @@ -184,20 +184,18 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), UDFToInteger(VALUE._col1) (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: int), UDFToInteger(_col1) (type: int) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table2 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table2 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out b/ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out index fd99597..1a644a9 100644 --- a/ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out +++ b/ql/src/test/results/clientpositive/bucketsortoptimize_insert_4.q.out @@ -71,21 +71,44 @@ FROM test_table1 a JOIN test_table2 b ON a.key = b.key WHERE a.ds = '1' and b.ds = '1' POSTHOOK: type: QUERY STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 + Stage-6 is a root stage , consists of Stage-7, Stage-8, Stage-1 + Stage-7 has a backup stage: Stage-1 + Stage-4 depends on stages: Stage-7 + Stage-0 depends on stages: Stage-1, Stage-4, Stage-5 Stage-2 depends on stages: Stage-0 + Stage-8 has a backup stage: Stage-1 + Stage-5 depends on stages: Stage-8 + Stage-1 STAGE PLANS: - Stage: Stage-1 + Stage: Stage-6 + Conditional Operator + + Stage: Stage-7 + Map Reduce Local Work + Alias -> Map Local Tables: + b + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + b + TableScan + alias: b + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + keys: + 0 key (type: int) + 1 key (type: int) + + Stage: Stage-4 Map Reduce Map Operator Tree: TableScan alias: a - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: key is not null (type: boolean) - Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE - Sorted Merge Bucket Map Join Operator + Map Join Operator condition map: Inner Join 0 to 1 keys: @@ -93,15 +116,26 @@ STAGE PLANS: 1 key (type: int) outputColumnNames: _col0, _col1, _col7 Select Operator - expressions: _col0 (type: int), _col0 (type: int), concat(_col1, _col7) (type: string) - outputColumnNames: _col0, _col1, _col2 - File Output Operator - compressed: false - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.test_table3 + expressions: _col0 (type: int), concat(_col1, _col7) (type: string) + outputColumnNames: _col1, _col2 + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + value expressions: _col2 (type: string) + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 Stage: Stage-0 Move Operator @@ -118,6 +152,95 @@ STAGE PLANS: Stage: Stage-2 Stats-Aggr Operator + Stage: Stage-8 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Filter Operator + predicate: key is not null (type: boolean) + HashTable Sink Operator + keys: + 0 key (type: int) + 1 key (type: int) + + Stage: Stage-5 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Filter Operator + predicate: key is not null (type: boolean) + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col7) (type: string) + outputColumnNames: _col1, _col2 + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + value expressions: _col2 (type: string) + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE + Sorted Merge Bucket Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 key (type: int) + 1 key (type: int) + outputColumnNames: _col0, _col1, _col7 + Select Operator + expressions: _col0 (type: int), concat(_col1, _col7) (type: string) + outputColumnNames: _col1, _col2 + Reduce Output Operator + key expressions: _col1 (type: int) + sort order: + + Map-reduce partition columns: _col1 (type: int) + value expressions: _col2 (type: string) + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.test_table3 + PREHOOK: query: INSERT OVERWRITE TABLE test_table3 PARTITION (ds = '1') SELECT a.key, a.key, concat(a.value, b.value) FROM test_table1 a JOIN test_table2 b @@ -266,11 +389,13 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col0 (type: int) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: @@ -329,11 +454,13 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col0 (type: int) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: @@ -362,9 +489,11 @@ STAGE PLANS: key expressions: _col1 (type: string) sort order: + Map-reduce partition columns: _col1 (type: string) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col0 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: diff --git a/ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out b/ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out index 8130ab9..e4f90e4 100644 --- a/ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out +++ b/ql/src/test/results/clientpositive/bucketsortoptimize_insert_5.q.out @@ -122,11 +122,13 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: @@ -188,11 +190,13 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: @@ -224,9 +228,11 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: @@ -377,11 +383,13 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: @@ -449,11 +457,13 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: @@ -489,9 +499,11 @@ STAGE PLANS: key expressions: _col0 (type: int) sort order: - Map-reduce partition columns: _col0 (type: int) - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 File Output Operator compressed: false table: diff --git a/ql/src/test/results/clientpositive/bucketsortoptimize_insert_6.q.out b/ql/src/test/results/clientpositive/bucketsortoptimize_insert_6.q.out index 627aba0..307c83b 100644 --- a/ql/src/test/results/clientpositive/bucketsortoptimize_insert_6.q.out +++ b/ql/src/test/results/clientpositive/bucketsortoptimize_insert_6.q.out @@ -394,11 +394,13 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: int) sort order: +- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false table: @@ -460,11 +462,13 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: int) sort order: +- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false table: @@ -496,9 +500,11 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: int) sort order: +- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false table: @@ -581,11 +587,13 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: int) sort order: +- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false table: @@ -647,11 +655,13 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: int) sort order: +- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false table: @@ -683,9 +693,11 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: int) sort order: +- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false table: @@ -1090,11 +1102,13 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: int) sort order: -- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false table: @@ -1156,11 +1170,13 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: int) sort order: -- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Local Work: Map Reduce Local Work Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false table: @@ -1192,9 +1208,11 @@ STAGE PLANS: key expressions: _col0 (type: int), _col1 (type: int) sort order: -- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 File Output Operator compressed: false table: diff --git a/ql/src/test/results/clientpositive/disable_merge_for_bucketing.q.out b/ql/src/test/results/clientpositive/disable_merge_for_bucketing.q.out index 9b058c8..a0edd3b 100644 --- a/ql/src/test/results/clientpositive/disable_merge_for_bucketing.q.out +++ b/ql/src/test/results/clientpositive/disable_merge_for_bucketing.q.out @@ -106,39 +106,37 @@ STAGE PLANS: /src [src] Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.bucket2_1 - serialization.ddl struct bucket2_1 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.bucket2_1 + serialization.ddl struct bucket2_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket2_1 - TotalFiles: 2 - GatherStats: true - MultiFileSpray: true + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket2_1 + TotalFiles: 2 + GatherStats: true + MultiFileSpray: true Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out b/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out index 32e0745..f49f590 100644 --- a/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out +++ b/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out @@ -206,7 +206,9 @@ STAGE PLANS: Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -291,7 +293,9 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -349,7 +353,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Execution mode: vectorized Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -406,7 +412,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Execution mode: vectorized Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -555,7 +563,9 @@ STAGE PLANS: Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -640,7 +650,9 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -698,7 +710,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Execution mode: vectorized Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -755,7 +769,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Execution mode: vectorized Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1381,7 +1397,9 @@ STAGE PLANS: Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1468,7 +1486,9 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1621,7 +1641,9 @@ STAGE PLANS: Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2002,10 +2024,12 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) Execution mode: vectorized Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: float), VALUE._col3 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2062,7 +2086,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Execution mode: vectorized Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out index 494bfa3..9e947bb 100644 --- a/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out +++ b/ql/src/test/results/clientpositive/dynpart_sort_optimization.q.out @@ -140,7 +140,9 @@ STAGE PLANS: Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -224,7 +226,9 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -281,7 +285,9 @@ STAGE PLANS: Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -337,7 +343,9 @@ STAGE PLANS: Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -463,7 +471,9 @@ STAGE PLANS: Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -547,7 +557,9 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -604,7 +616,9 @@ STAGE PLANS: Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -660,7 +674,9 @@ STAGE PLANS: Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1284,7 +1300,9 @@ STAGE PLANS: Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1370,7 +1388,9 @@ STAGE PLANS: Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1521,7 +1541,9 @@ STAGE PLANS: Statistics: Num rows: 2221 Data size: 53305 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 2221 Data size: 53305 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1902,9 +1924,11 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: float), VALUE._col3 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1960,7 +1984,9 @@ STAGE PLANS: Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out index b6e7b88..547bb0e 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out @@ -131,7 +131,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 24 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false @@ -403,7 +405,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -763,7 +767,9 @@ STAGE PLANS: /encryptedTable/key=86 [encryptedtable] Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 21 Data size: 2372 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out index fc6d2ae..a4fe0ad 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out @@ -134,7 +134,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 24 Basic stats: PARTIAL Column stats: NONE File Output Operator compressed: false @@ -405,7 +407,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 2 Data size: 400 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -668,7 +672,9 @@ STAGE PLANS: /encryptedTable/ds=yesterday [encryptedtable] Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 12 Data size: 2695 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/load_dyn_part2.q.out b/ql/src/test/results/clientpositive/load_dyn_part2.q.out index 26f318a..0c5fe6b 100644 --- a/ql/src/test/results/clientpositive/load_dyn_part2.q.out +++ b/ql/src/test/results/clientpositive/load_dyn_part2.q.out @@ -60,7 +60,9 @@ STAGE PLANS: Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/ptf.q.out b/ql/src/test/results/clientpositive/ptf.q.out index f678035..4bf5d79 100644 --- a/ql/src/test/results/clientpositive/ptf.q.out +++ b/ql/src/test/results/clientpositive/ptf.q.out @@ -41,9 +41,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -211,9 +213,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE @@ -333,9 +337,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col2, _col1, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -436,9 +442,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -571,9 +579,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -709,9 +719,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -870,9 +882,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1011,9 +1025,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1154,9 +1170,10 @@ STAGE PLANS: sort order: ++- Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1282,9 +1299,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1413,9 +1432,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1547,9 +1568,11 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1571,9 +1594,11 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1712,9 +1737,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1858,9 +1885,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2031,9 +2060,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2213,9 +2244,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + value expressions: _col1 (type: string), _col2 (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: double) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE @@ -2397,9 +2430,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2698,9 +2733,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2722,9 +2759,11 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2887,9 +2926,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2909,9 +2950,11 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2931,9 +2974,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3091,9 +3136,11 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3113,9 +3160,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3274,9 +3323,11 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3296,9 +3347,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3320,9 +3373,11 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3482,9 +3537,11 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3506,9 +3563,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3662,9 +3721,11 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3686,9 +3747,11 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/ptf_streaming.q.out b/ql/src/test/results/clientpositive/ptf_streaming.q.out index 9cf645d..92d293b 100644 --- a/ql/src/test/results/clientpositive/ptf_streaming.q.out +++ b/ql/src/test/results/clientpositive/ptf_streaming.q.out @@ -41,9 +41,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -211,9 +213,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE @@ -336,9 +340,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -479,9 +485,10 @@ STAGE PLANS: sort order: ++- Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -607,9 +614,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -741,9 +750,11 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -765,9 +776,11 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -901,9 +914,11 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -925,9 +940,11 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1061,9 +1078,11 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1085,9 +1104,11 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1225,9 +1246,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1421,9 +1444,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1445,9 +1470,11 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1610,9 +1637,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1632,9 +1661,11 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1654,9 +1685,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1812,9 +1845,11 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1836,9 +1871,11 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/smb_mapjoin_20.q.out b/ql/src/test/results/clientpositive/smb_mapjoin_20.q.out index 999dabd..0324aa4 100644 --- a/ql/src/test/results/clientpositive/smb_mapjoin_20.q.out +++ b/ql/src/test/results/clientpositive/smb_mapjoin_20.q.out @@ -57,17 +57,19 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), value (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: UDFToString(_col0) (type: string) sort order: + Map-reduce partition columns: UDFToString(_col0) (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -176,8 +178,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string), key (type: int), value (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: value (type: string), key (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -298,17 +300,19 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: (key + key) (type: int), value (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: (key + key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: UDFToString(_col0) (type: string) sort order: + Map-reduce partition columns: UDFToString(_col0) (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/smb_mapjoin_21.q.out b/ql/src/test/results/clientpositive/smb_mapjoin_21.q.out index 539b70e..319fef3 100644 --- a/ql/src/test/results/clientpositive/smb_mapjoin_21.q.out +++ b/ql/src/test/results/clientpositive/smb_mapjoin_21.q.out @@ -135,9 +135,11 @@ STAGE PLANS: sort order: - Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -214,9 +216,10 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -293,9 +296,11 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col0 (type: int) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -372,9 +377,11 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -452,7 +459,9 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: int), _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/spark/bucket2.q.out b/ql/src/test/results/clientpositive/spark/bucket2.q.out index 5eb28fa..89c3b4c 100644 --- a/ql/src/test/results/clientpositive/spark/bucket2.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket2.q.out @@ -116,39 +116,37 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.bucket2_1 - serialization.ddl struct bucket2_1 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.bucket2_1 + serialization.ddl struct bucket2_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket2_1 - TotalFiles: 2 - GatherStats: true - MultiFileSpray: true + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket2_1 + TotalFiles: 2 + GatherStats: true + MultiFileSpray: true Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/bucket3.q.out b/ql/src/test/results/clientpositive/spark/bucket3.q.out index 1b1010a..2fc4855 100644 --- a/ql/src/test/results/clientpositive/spark/bucket3.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket3.q.out @@ -120,42 +120,40 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 2 - Static Partition Specification: ds=1/ - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 2 + Static Partition Specification: ds=1/ + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.bucket3_1 - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket3_1 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.bucket3_1 + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket3_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket3_1 - TotalFiles: 2 - GatherStats: true - MultiFileSpray: true + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket3_1 + TotalFiles: 2 + GatherStats: true + MultiFileSpray: true Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/bucket4.q.out b/ql/src/test/results/clientpositive/spark/bucket4.q.out index 7dd49ac..44e0f9f 100644 --- a/ql/src/test/results/clientpositive/spark/bucket4.q.out +++ b/ql/src/test/results/clientpositive/spark/bucket4.q.out @@ -113,40 +113,38 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.bucket4_1 - serialization.ddl struct bucket4_1 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.bucket4_1 + serialization.ddl struct bucket4_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket4_1 - TotalFiles: 2 - GatherStats: true - MultiFileSpray: true + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket4_1 + TotalFiles: 2 + GatherStats: true + MultiFileSpray: true Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out b/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out index 365306e..bcef03c 100644 --- a/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out @@ -153,12 +153,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -323,12 +325,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -517,12 +521,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 92 Data size: 809 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 92 Data size: 809 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -723,12 +729,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -911,12 +919,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1099,12 +1109,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out b/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out index 3846de7..f3c8f04 100644 --- a/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out @@ -121,20 +121,22 @@ STAGE PLANS: 0 Map 1 Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: _col0 (type: int), _col0 (type: int), concat(_col1, _col7) (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: _col0 (type: int), concat(_col1, _col7) (type: string) + outputColumnNames: _col1, _col2 Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col1 (type: int) sort order: + Map-reduce partition columns: _col1 (type: int) Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -314,12 +316,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col1 (type: string) Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col0 (type: int) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out b/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out index 5b559c4..25f61b8 100644 --- a/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out @@ -131,12 +131,14 @@ STAGE PLANS: sort order: +- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -308,12 +310,14 @@ STAGE PLANS: sort order: +- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -485,12 +489,14 @@ STAGE PLANS: sort order: +- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -597,12 +603,14 @@ STAGE PLANS: sort order: +- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -715,12 +723,14 @@ STAGE PLANS: sort order: +- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -910,12 +920,14 @@ STAGE PLANS: sort order: +- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1115,12 +1127,14 @@ STAGE PLANS: sort order: -- Map-reduce partition columns: _col0 (type: int), _col1 (type: int) Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col2 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out b/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out index cefc6aa..7c087ec 100644 --- a/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out @@ -131,12 +131,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -311,12 +313,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -497,12 +501,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 15 Data size: 134 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 15 Data size: 134 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out b/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out index ca44d7c..dade75c 100644 --- a/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out @@ -129,12 +129,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col1 (type: int), _col2 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -300,12 +302,14 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string) + value expressions: _col1 (type: int), _col2 (type: string) Local Work: Map Reduce Local Work Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out b/ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out index 3864c44..d39c5be 100644 --- a/ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out +++ b/ql/src/test/results/clientpositive/spark/disable_merge_for_bucketing.q.out @@ -112,39 +112,37 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.bucket2_1 - serialization.ddl struct bucket2_1 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.bucket2_1 + serialization.ddl struct bucket2_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket2_1 - TotalFiles: 2 - GatherStats: true - MultiFileSpray: true + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket2_1 + TotalFiles: 2 + GatherStats: true + MultiFileSpray: true Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/load_dyn_part2.q.out b/ql/src/test/results/clientpositive/spark/load_dyn_part2.q.out index a8cef34..9177290 100644 --- a/ql/src/test/results/clientpositive/spark/load_dyn_part2.q.out +++ b/ql/src/test/results/clientpositive/spark/load_dyn_part2.q.out @@ -66,7 +66,9 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/spark/ptf.q.out b/ql/src/test/results/clientpositive/spark/ptf.q.out index ad7d9d5..e38d977 100644 --- a/ql/src/test/results/clientpositive/spark/ptf.q.out +++ b/ql/src/test/results/clientpositive/spark/ptf.q.out @@ -46,10 +46,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -204,10 +206,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE @@ -322,10 +326,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col2, _col1, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -431,10 +437,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -562,10 +570,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -696,10 +706,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -843,7 +855,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Map 4 Map Operator Tree: TableScan @@ -859,7 +871,9 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -995,7 +1009,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reducer 2 Reduce Operator Tree: Join Operator @@ -1019,7 +1033,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 4 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1123,10 +1139,11 @@ STAGE PLANS: sort order: ++- Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1247,10 +1264,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1374,10 +1393,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1504,10 +1525,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1518,10 +1541,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1655,10 +1680,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1797,7 +1824,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) Map 5 Map Operator Tree: TableScan @@ -1813,7 +1840,9 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1958,10 +1987,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2126,10 +2157,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + value expressions: _col1 (type: string), _col2 (type: double) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: double) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE @@ -2307,7 +2340,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 3 Reduce Operator Tree: Select Operator @@ -2364,7 +2397,9 @@ STAGE PLANS: name: default.part_5 Reducer 6 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2376,7 +2411,9 @@ STAGE PLANS: value expressions: _col5 (type: int), _col7 (type: double) Reducer 7 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2590,10 +2627,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2604,10 +2643,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2765,10 +2806,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2777,10 +2820,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2789,10 +2834,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reducer 4 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2945,10 +2992,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2957,10 +3006,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3114,10 +3165,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3126,10 +3179,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3140,10 +3195,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 4 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3298,10 +3355,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3312,10 +3371,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3464,10 +3525,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3478,10 +3541,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/spark/ptf_streaming.q.out b/ql/src/test/results/clientpositive/spark/ptf_streaming.q.out index 9f98933..1e3ea44 100644 --- a/ql/src/test/results/clientpositive/spark/ptf_streaming.q.out +++ b/ql/src/test/results/clientpositive/spark/ptf_streaming.q.out @@ -46,10 +46,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -204,10 +206,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE @@ -325,7 +329,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Map 4 Map Operator Tree: TableScan @@ -341,7 +345,9 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -466,10 +472,11 @@ STAGE PLANS: sort order: ++- Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -590,10 +597,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -720,10 +729,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -734,10 +745,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -866,10 +879,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -880,10 +895,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1012,10 +1029,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1026,10 +1045,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1162,7 +1183,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) Map 5 Map Operator Tree: TableScan @@ -1178,7 +1199,9 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1346,10 +1369,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1360,10 +1385,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1521,10 +1548,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1533,10 +1562,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1545,10 +1576,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reducer 4 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1699,10 +1732,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1713,10 +1748,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_20.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_20.q.out index 9815d24..e4a9ba1 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_20.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_20.q.out @@ -62,18 +62,20 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: key (type: int), value (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: key (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: UDFToString(_col0) (type: string) sort order: + Map-reduce partition columns: UDFToString(_col0) (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -185,8 +187,8 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: value (type: string), key (type: int), value (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: value (type: string), key (type: int) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -312,18 +314,20 @@ STAGE PLANS: alias: a Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator - expressions: (key + key) (type: int), value (type: string), value (type: string) - outputColumnNames: _col0, _col1, _col2 + expressions: (key + key) (type: int), value (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: UDFToString(_col0) (type: string) sort order: + Map-reduce partition columns: UDFToString(_col0) (type: string) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string) + value expressions: _col0 (type: int), _col1 (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_21.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_21.q.out index 44bae6e..7828178 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_21.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_21.q.out @@ -143,10 +143,12 @@ STAGE PLANS: sort order: - Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -228,10 +230,11 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -313,10 +316,12 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col0 (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -398,10 +403,12 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: int) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: int), _col1 (type: string) + value expressions: _col1 (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -485,7 +492,9 @@ STAGE PLANS: value expressions: _col0 (type: int), _col1 (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/spark/stats10.q.out b/ql/src/test/results/clientpositive/spark/stats10.q.out index 4bec7fc..c840ab7 100644 --- a/ql/src/test/results/clientpositive/spark/stats10.q.out +++ b/ql/src/test/results/clientpositive/spark/stats10.q.out @@ -42,20 +42,18 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket3_1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket3_1 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out index 020fdff..ab0fd06 100644 --- a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out @@ -244,7 +244,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -299,7 +299,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -669,12 +671,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE @@ -843,7 +847,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -898,7 +902,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col2, _col1, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1101,7 +1107,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -1156,7 +1162,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1388,7 +1396,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -1443,7 +1451,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1685,7 +1695,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -1740,7 +1750,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1952,7 +1964,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -2074,7 +2086,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -2329,7 +2343,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -2420,7 +2434,9 @@ STAGE PLANS: Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -2583,7 +2599,6 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -2637,7 +2652,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -2857,7 +2874,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -2911,7 +2928,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3134,7 +3153,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -3189,7 +3208,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3422,7 +3443,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -3477,7 +3498,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3489,12 +3512,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3747,7 +3772,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -3802,7 +3827,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -4132,7 +4159,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -4254,7 +4281,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -4457,7 +4486,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -4512,7 +4541,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -4807,13 +4838,15 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + value expressions: _col1 (type: string), _col2 (type: double) auto parallelism: false Execution mode: vectorized Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: double) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE @@ -5203,7 +5236,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -5353,7 +5386,9 @@ STAGE PLANS: Reducer 6 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -5368,7 +5403,9 @@ STAGE PLANS: Reducer 7 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -5718,7 +5755,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -5773,7 +5810,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -5785,12 +5824,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6074,7 +6115,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -6129,7 +6170,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6139,12 +6182,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6154,12 +6199,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6426,7 +6473,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -6481,7 +6528,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6491,12 +6540,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6775,7 +6826,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -6830,7 +6881,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6840,12 +6893,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6857,12 +6912,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7172,7 +7229,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -7227,7 +7284,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7239,12 +7298,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7520,7 +7581,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -7575,7 +7636,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7587,12 +7650,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/stats10.q.out b/ql/src/test/results/clientpositive/stats10.q.out index 804e1b8..dc8aa7f 100644 --- a/ql/src/test/results/clientpositive/stats10.q.out +++ b/ql/src/test/results/clientpositive/stats10.q.out @@ -36,20 +36,18 @@ STAGE PLANS: Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket3_1 + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket3_1 Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/tez/bucket2.q.out b/ql/src/test/results/clientpositive/tez/bucket2.q.out index 9536ed0..55aa220 100644 --- a/ql/src/test/results/clientpositive/tez/bucket2.q.out +++ b/ql/src/test/results/clientpositive/tez/bucket2.q.out @@ -117,39 +117,37 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.bucket2_1 - serialization.ddl struct bucket2_1 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.bucket2_1 + serialization.ddl struct bucket2_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket2_1 - TotalFiles: 2 - GatherStats: true - MultiFileSpray: true + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket2_1 + TotalFiles: 2 + GatherStats: true + MultiFileSpray: true Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/tez/bucket3.q.out b/ql/src/test/results/clientpositive/tez/bucket3.q.out index 3427474..bbd9c67 100644 --- a/ql/src/test/results/clientpositive/tez/bucket3.q.out +++ b/ql/src/test/results/clientpositive/tez/bucket3.q.out @@ -121,42 +121,40 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 2 - Static Partition Specification: ds=1/ - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 2 + Static Partition Specification: ds=1/ + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.bucket3_1 - partition_columns ds - partition_columns.types string - serialization.ddl struct bucket3_1 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.bucket3_1 + partition_columns ds + partition_columns.types string + serialization.ddl struct bucket3_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket3_1 - TotalFiles: 2 - GatherStats: true - MultiFileSpray: true + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket3_1 + TotalFiles: 2 + GatherStats: true + MultiFileSpray: true Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/tez/bucket4.q.out b/ql/src/test/results/clientpositive/tez/bucket4.q.out index ca3584e..ed8671c 100644 --- a/ql/src/test/results/clientpositive/tez/bucket4.q.out +++ b/ql/src/test/results/clientpositive/tez/bucket4.q.out @@ -114,40 +114,38 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - SORTBUCKETCOLSPREFIX TRUE - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + SORTBUCKETCOLSPREFIX TRUE + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.bucket4_1 - serialization.ddl struct bucket4_1 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.bucket4_1 + serialization.ddl struct bucket4_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket4_1 - TotalFiles: 2 - GatherStats: true - MultiFileSpray: true + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket4_1 + TotalFiles: 2 + GatherStats: true + MultiFileSpray: true Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/tez/disable_merge_for_bucketing.q.out b/ql/src/test/results/clientpositive/tez/disable_merge_for_bucketing.q.out index dcffd87..84e0d2c 100644 --- a/ql/src/test/results/clientpositive/tez/disable_merge_for_bucketing.q.out +++ b/ql/src/test/results/clientpositive/tez/disable_merge_for_bucketing.q.out @@ -113,39 +113,37 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - GlobalTableId: 1 + File Output Operator + compressed: false + GlobalTableId: 1 #### A masked pattern was here #### - NumFilesPerFileSink: 2 - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE + NumFilesPerFileSink: 2 + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### - table: - input format: org.apache.hadoop.mapred.TextInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat - properties: - bucket_count 2 - bucket_field_name key - columns key,value - columns.comments - columns.types int:string + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count 2 + bucket_field_name key + columns key,value + columns.comments + columns.types int:string #### A masked pattern was here #### - name default.bucket2_1 - serialization.ddl struct bucket2_1 { i32 key, string value} - serialization.format 1 - serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name default.bucket2_1 + serialization.ddl struct bucket2_1 { i32 key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe #### A masked pattern was here #### - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.bucket2_1 - TotalFiles: 2 - GatherStats: true - MultiFileSpray: true + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.bucket2_1 + TotalFiles: 2 + GatherStats: true + MultiFileSpray: true Stage: Stage-2 Dependency Collection diff --git a/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out b/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out index ad6dbf7..6989b92 100644 --- a/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out +++ b/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out @@ -204,7 +204,9 @@ STAGE PLANS: Execution mode: vectorized Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -291,7 +293,9 @@ STAGE PLANS: Execution mode: vectorized Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -360,7 +364,9 @@ STAGE PLANS: Execution mode: vectorized Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -428,7 +434,9 @@ STAGE PLANS: Execution mode: vectorized Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -579,7 +587,9 @@ STAGE PLANS: Execution mode: vectorized Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -666,7 +676,9 @@ STAGE PLANS: Execution mode: vectorized Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -735,7 +747,9 @@ STAGE PLANS: Execution mode: vectorized Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -803,7 +817,9 @@ STAGE PLANS: Execution mode: vectorized Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1442,7 +1458,9 @@ STAGE PLANS: Execution mode: vectorized Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1531,7 +1549,9 @@ STAGE PLANS: Execution mode: vectorized Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 10 Data size: 2960 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1697,7 +1717,9 @@ STAGE PLANS: Execution mode: vectorized Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 524 Data size: 155436 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2088,11 +2110,13 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) Execution mode: vectorized Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: float), VALUE._col3 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2160,7 +2184,9 @@ STAGE PLANS: Execution mode: vectorized Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 1048 Data size: 310873 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out b/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out index ff44a57..8a16645 100644 --- a/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out +++ b/ql/src/test/results/clientpositive/tez/dynpart_sort_optimization.q.out @@ -147,7 +147,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -231,7 +233,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -298,7 +302,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -364,7 +370,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -500,7 +508,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -584,7 +594,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -651,7 +663,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -717,7 +731,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1351,7 +1367,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1437,7 +1455,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 10 Data size: 240 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1598,7 +1618,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 2221 Data size: 53305 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1988,10 +2010,12 @@ STAGE PLANS: sort order: + Map-reduce partition columns: _col0 (type: smallint) Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint) + value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col4 (type: tinyint) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), KEY.reducesinkkey0 (type: float), VALUE._col3 (type: tinyint) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2057,7 +2081,9 @@ STAGE PLANS: value expressions: _col0 (type: smallint), _col1 (type: int), _col2 (type: bigint), _col3 (type: float), _col4 (type: tinyint), '_bucket_number' (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: int), VALUE._col2 (type: bigint), VALUE._col3 (type: float), VALUE._col4 (type: tinyint), VALUE._bucket_number (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _bucket_number Statistics: Num rows: 4442 Data size: 106611 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/tez/load_dyn_part2.q.out b/ql/src/test/results/clientpositive/tez/load_dyn_part2.q.out index d1c4a10..1c0472f 100644 --- a/ql/src/test/results/clientpositive/tez/load_dyn_part2.q.out +++ b/ql/src/test/results/clientpositive/tez/load_dyn_part2.q.out @@ -67,7 +67,9 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false diff --git a/ql/src/test/results/clientpositive/tez/ptf.q.out b/ql/src/test/results/clientpositive/tez/ptf.q.out index 6f9dd91..23bd201 100644 --- a/ql/src/test/results/clientpositive/tez/ptf.q.out +++ b/ql/src/test/results/clientpositive/tez/ptf.q.out @@ -46,10 +46,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -204,10 +206,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE @@ -322,10 +326,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col2, _col1, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -431,10 +437,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -562,10 +570,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -696,10 +706,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -843,7 +855,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Map 4 Map Operator Tree: TableScan @@ -859,7 +871,9 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -995,7 +1009,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Reducer 2 Reduce Operator Tree: Merge Join Operator @@ -1019,7 +1033,9 @@ STAGE PLANS: serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Reducer 4 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1123,10 +1139,11 @@ STAGE PLANS: sort order: ++- Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1247,10 +1264,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1374,10 +1393,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1504,10 +1525,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1518,10 +1541,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1655,10 +1680,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1797,7 +1824,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) Map 5 Map Operator Tree: TableScan @@ -1813,7 +1840,9 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1958,10 +1987,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2126,10 +2157,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + value expressions: _col1 (type: string), _col2 (type: double) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: double) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE @@ -2307,10 +2340,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2587,10 +2622,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2601,10 +2638,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2762,10 +2801,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2774,10 +2815,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2786,10 +2829,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reducer 4 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2942,10 +2987,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -2954,10 +3001,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3111,10 +3160,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3123,10 +3174,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3137,10 +3190,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 4 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3295,10 +3350,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3309,10 +3366,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3461,10 +3520,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -3475,10 +3536,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out b/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out index a935ef6..b24932c 100644 --- a/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out +++ b/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out @@ -46,10 +46,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -204,10 +206,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE @@ -325,7 +329,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) Map 4 Map Operator Tree: TableScan @@ -341,7 +345,9 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -466,10 +472,11 @@ STAGE PLANS: sort order: ++- Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -590,10 +597,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -720,10 +729,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -734,10 +745,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -866,10 +879,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -880,10 +895,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1012,10 +1029,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1026,10 +1045,12 @@ STAGE PLANS: sort order: +++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1162,7 +1183,7 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) Map 5 Map Operator Tree: TableScan @@ -1178,7 +1199,9 @@ STAGE PLANS: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1346,10 +1369,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1360,10 +1385,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1521,10 +1548,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1533,10 +1562,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1545,10 +1576,12 @@ STAGE PLANS: sort order: ++ Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) Reducer 4 Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1699,10 +1732,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE @@ -1713,10 +1748,12 @@ STAGE PLANS: sort order: ++++ Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) Reducer 3 Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/tez/vector_bucket.q.out b/ql/src/test/results/clientpositive/tez/vector_bucket.q.out index e4deb04..410212a 100644 --- a/ql/src/test/results/clientpositive/tez/vector_bucket.q.out +++ b/ql/src/test/results/clientpositive/tez/vector_bucket.q.out @@ -41,20 +41,18 @@ STAGE PLANS: value expressions: _col0 (type: string), _col1 (type: string) Reducer 2 Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.non_orc_table + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.non_orc_table Execution mode: vectorized Stage: Stage-2 diff --git a/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out index a814849..516ab02 100644 --- a/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out @@ -244,7 +244,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -299,7 +299,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -670,12 +672,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE @@ -844,7 +848,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -899,7 +903,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col2, _col1, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1102,7 +1108,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -1157,7 +1163,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1389,7 +1397,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -1444,7 +1452,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1686,7 +1696,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -1741,7 +1751,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1953,7 +1965,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -2075,7 +2087,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -2331,7 +2345,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -2423,7 +2437,9 @@ STAGE PLANS: Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -2586,7 +2602,6 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -2640,7 +2655,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -2860,7 +2877,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -2914,7 +2931,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3137,7 +3156,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -3192,7 +3211,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3425,7 +3446,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -3480,7 +3501,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3492,12 +3515,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3750,7 +3775,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -3805,7 +3830,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -4135,7 +4162,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -4257,7 +4284,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -4461,7 +4490,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -4516,7 +4545,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -4811,13 +4842,15 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + value expressions: _col1 (type: string), _col2 (type: double) auto parallelism: true Execution mode: vectorized Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: double) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE @@ -5207,7 +5240,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -5262,7 +5295,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -5718,7 +5753,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -5773,7 +5808,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -5785,12 +5822,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6074,7 +6113,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -6129,7 +6168,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6139,12 +6180,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6154,12 +6197,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: true Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6426,7 +6471,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -6481,7 +6526,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6491,12 +6538,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6775,7 +6824,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -6830,7 +6879,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6840,12 +6891,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6857,12 +6910,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: true Reducer 4 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7172,7 +7227,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -7227,7 +7282,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7239,12 +7296,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7520,7 +7579,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: true Path -> Alias: #### A masked pattern was here #### @@ -7575,7 +7634,9 @@ STAGE PLANS: Reducer 2 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7587,12 +7648,14 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: true Reducer 3 Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE diff --git a/ql/src/test/results/clientpositive/vector_bucket.q.out b/ql/src/test/results/clientpositive/vector_bucket.q.out index 952c033..1ac37a7 100644 --- a/ql/src/test/results/clientpositive/vector_bucket.q.out +++ b/ql/src/test/results/clientpositive/vector_bucket.q.out @@ -34,20 +34,18 @@ STAGE PLANS: Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE value expressions: _col0 (type: string), _col1 (type: string) Reduce Operator Tree: - Extract + Select Operator + expressions: UDFToInteger(VALUE._col0) (type: int), VALUE._col1 (type: string) + outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE - Select Operator - expressions: UDFToInteger(_col0) (type: int), _col1 (type: string) - outputColumnNames: _col0, _col1 + File Output Operator + compressed: false Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 0 Data size: 26 Basic stats: PARTIAL Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - name: default.non_orc_table + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.non_orc_table Stage: Stage-0 Move Operator diff --git a/ql/src/test/results/clientpositive/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/vectorized_ptf.q.out index 1e3c43c..2d561cb 100644 --- a/ql/src/test/results/clientpositive/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/vectorized_ptf.q.out @@ -239,7 +239,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -293,7 +293,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -675,7 +677,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -704,7 +706,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE @@ -915,7 +919,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -969,7 +973,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col2, _col1, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1167,7 +1173,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -1221,7 +1227,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1495,7 +1503,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -1549,7 +1557,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -1833,7 +1843,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -1887,7 +1897,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -2187,7 +2199,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -2241,7 +2253,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -2531,7 +2545,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) + value expressions: p_partkey (type: int), p_brand (type: string), p_type (type: string), p_size (type: int), p_container (type: string), p_retailprice (type: double), p_comment (type: string) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -2585,7 +2599,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string), VALUE._col3 (type: int), VALUE._col4 (type: string), VALUE._col5 (type: double), VALUE._col6 (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -2886,7 +2902,6 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -2939,7 +2954,9 @@ STAGE PLANS: /part_orc [part_orc] Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey2 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3201,7 +3218,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -3254,7 +3271,9 @@ STAGE PLANS: /part_orc [part_orc] Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3519,7 +3538,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -3573,7 +3592,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3848,7 +3869,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -3902,7 +3923,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -3937,7 +3960,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _col7 (type: double) + value expressions: _col5 (type: int), _col7 (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -3966,7 +3989,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -4261,7 +4286,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -4315,7 +4340,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -4687,7 +4714,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_partkey (type: int), p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_partkey (type: int), p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -4741,7 +4768,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col0 (type: int), KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -5093,7 +5122,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -5147,7 +5176,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -5505,7 +5536,7 @@ STAGE PLANS: Map-reduce partition columns: _col0 (type: string) Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double) + value expressions: _col1 (type: string), _col2 (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -5534,7 +5565,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: double) + outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE @@ -5965,7 +5998,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int), p_retailprice (type: double) + value expressions: p_size (type: int), p_retailprice (type: double) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -6019,7 +6052,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int), VALUE._col5 (type: double) + outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6608,7 +6643,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -6662,7 +6697,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -6697,7 +6734,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -6726,7 +6763,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7052,7 +7091,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_name (type: string), p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -7106,7 +7145,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7139,7 +7180,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -7168,7 +7209,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7201,7 +7244,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -7230,7 +7273,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7539,7 +7584,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -7593,7 +7638,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7626,7 +7673,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -7655,7 +7702,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -7976,7 +8025,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -8030,7 +8079,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -8063,7 +8114,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -8092,7 +8143,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -8127,7 +8180,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -8156,7 +8209,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -8508,7 +8563,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -8562,7 +8617,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -8597,7 +8654,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col1 (type: string), _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -8626,7 +8683,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col4 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -8944,7 +9003,7 @@ STAGE PLANS: Map-reduce partition columns: p_mfgr (type: string), p_name (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: p_name (type: string), p_mfgr (type: string), p_size (type: int) + value expressions: p_size (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -8998,7 +9057,9 @@ STAGE PLANS: Execution mode: vectorized Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE @@ -9033,7 +9094,7 @@ STAGE PLANS: Map-reduce partition columns: _col2 (type: string), _col1 (type: string) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE tag: -1 - value expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int) + value expressions: _col5 (type: int) auto parallelism: false Path -> Alias: #### A masked pattern was here #### @@ -9062,7 +9123,9 @@ STAGE PLANS: #### A masked pattern was here #### Needs Tagging: false Reduce Operator Tree: - Extract + Select Operator + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE