diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 78ef0a0..94e665b 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -140,6 +140,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\ orc_vectorization_ppd.q,\ parallel.q,\ ptf.q,\ + ptf_matchpath.q,\ ptf_streaming.q,\ sample1.q,\ selectDistinctStar.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index daf6cb8..149f911 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -33,7 +33,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -478,11 +477,11 @@ else if (ent.getValue() instanceof Map) { json.put(ent.getKey().toString(), ent.getValue().toString()); } } - else if (ent.getValue() instanceof Serializable) { + else if (ent.getValue() != null) { if (out != null) { out.println(); } - JSONObject jsonOut = outputPlan((Serializable) ent.getValue(), out, + JSONObject jsonOut = outputPlan(ent.getValue(), out, extended, jsonOutput, jsonOutput ? 0 : indent + 2); if (jsonOutput) { json.put(ent.getKey().toString(), jsonOut); @@ -518,11 +517,11 @@ private JSONArray outputList(List l, PrintStream out, boolean hasHeader, } nl = true; } - else if (o instanceof Serializable) { + else { if (first_el && (out != null) && hasHeader) { out.println(); } - JSONObject jsonOut = outputPlan((Serializable) o, out, extended, + JSONObject jsonOut = outputPlan(o, out, extended, jsonOutput, jsonOutput ? 0 : (hasHeader ? indent + 2 : indent)); if (jsonOutput) { outputArray.put(jsonOut); @@ -553,12 +552,12 @@ private boolean isPrintable(Object val) { return false; } - private JSONObject outputPlan(Serializable work, + private JSONObject outputPlan(Object work, PrintStream out, boolean extended, boolean jsonOutput, int indent) throws Exception { return outputPlan(work, out, extended, jsonOutput, indent, ""); } - private JSONObject outputPlan(Serializable work, PrintStream out, + private JSONObject outputPlan(Object work, PrintStream out, boolean extended, boolean jsonOutput, int indent, String appendToHeader) throws Exception { // Check if work has an explain annotation Annotation note = AnnotationUtils.getAnnotation(work.getClass(), Explain.class); @@ -678,7 +677,7 @@ private JSONObject outputPlan(Serializable work, PrintStream out, } // Try this as a map - try { + if (val instanceof Map) { // Go through the map and print out the stuff Map mp = (Map) val; @@ -692,22 +691,10 @@ private JSONObject outputPlan(Serializable work, PrintStream out, } continue; } - catch (ClassCastException ce) { - // Ignore - all this means is that this is not a map - } // Try this as a list - try { - List l; - - try { - l = (List) val; - } catch (ClassCastException e) { - Set s = (Set) val; - l = new LinkedList(); - l.addAll(s); - } - + if (val instanceof List || val instanceof Set) { + List l = val instanceof List ? (List)val : new ArrayList((Set)val); if (out != null && !skipHeader && l != null && !l.isEmpty()) { out.print(header); } @@ -720,18 +707,13 @@ private JSONObject outputPlan(Serializable work, PrintStream out, continue; } - catch (ClassCastException ce) { - // Ignore - } // Finally check if it is serializable try { - Serializable s = (Serializable) val; - if (!skipHeader && out != null) { out.println(header); } - JSONObject jsonOut = outputPlan(s, out, extended, jsonOutput, ind); + JSONObject jsonOut = outputPlan(val, out, extended, jsonOutput, ind); if (jsonOutput) { if (!skipHeader) { json.put(header, jsonOut); @@ -779,7 +761,7 @@ private boolean shouldPrint(Explain exp, Object val) { return true; } - private JSONObject outputPlan(Task task, + private JSONObject outputPlan(Task task, PrintStream out, JSONObject parentJSON, boolean extended, boolean jsonOutput, int indent) throws Exception { @@ -805,7 +787,7 @@ private JSONObject outputPlan(Task task, return null; } - private JSONObject outputDependencies(Task task, + private JSONObject outputDependencies(Task task, PrintStream out, JSONObject parentJson, boolean jsonOutput, boolean taskType, int indent) throws Exception { @@ -830,7 +812,7 @@ private JSONObject outputDependencies(Task task, else { StringBuffer s = new StringBuffer(); first = true; - for (Task parent : task.getParentTasks()) { + for (Task parent : task.getParentTasks()) { if (!first) { s.append(", "); } @@ -847,7 +829,7 @@ private JSONObject outputDependencies(Task task, } } - Task currBackupTask = task.getBackupTask(); + Task currBackupTask = task.getBackupTask(); if (currBackupTask != null) { if (out != null) { out.print(" has a backup stage: "); @@ -862,7 +844,7 @@ private JSONObject outputDependencies(Task task, && ((ConditionalTask) task).getListTasks() != null) { StringBuffer s = new StringBuffer(); first = true; - for (Task con : ((ConditionalTask) task).getListTasks()) { + for (Task con : ((ConditionalTask) task).getListTasks()) { if (!first) { s.append(", "); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java index f49b7cd..d3e9992 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; +import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -68,6 +69,9 @@ import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFInputDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.ShapeDetails; import org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef; import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef; import org.apache.hadoop.hive.ql.udf.ptf.Noop; @@ -260,34 +264,34 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, PTFDesc conf = op.getConf(); //Since we cannot know what columns will be needed by a PTF chain, //we do not prune columns on PTFOperator for PTF chains. - if (!conf.forWindowing() && !Noop.class.isInstance(conf.getFuncDef().getTFunction())) { + PartitionedTableFunctionDef funcDef = conf.getFuncDef(); + if (!conf.forWindowing() && !Noop.class.isInstance(funcDef.getTFunction())) { return super.process(nd, stack, cppCtx, nodeOutputs); } - - List prunedCols = cppCtx.getPrunedColList(op.getChildOperators().get(0)); - - WindowTableFunctionDef def = null; - if (conf.forWindowing()) { - def = (WindowTableFunctionDef) conf.getFuncDef(); + + //we create a copy of prunedCols to create a list of pruned columns for PTFOperator + List prunedCols = + new ArrayList(cppCtx.getPrunedColList(op.getChildOperators().get(0))); + if (funcDef instanceof WindowTableFunctionDef) { + WindowTableFunctionDef def = (WindowTableFunctionDef) funcDef; prunedCols = Utilities.mergeUniqElems(getWindowFunctionColumns(def), prunedCols); - prunedCols = prunedColumnsList(prunedCols, def); } + + List newRS = prunedColumnsList(prunedCols, op.getSchema(), funcDef); + + op.getSchema().setSignature(new ArrayList(newRS)); - RowSchema oldRS = op.getSchema(); - ArrayList sig = buildPrunedRR(prunedCols, oldRS); - op.getSchema().setSignature(sig); - - prunedCols = def == null ? prunedCols : prunedInputList(prunedCols, def); - cppCtx.getPrunedColLists().put(op, prunedCols); + ShapeDetails outputShape = funcDef.getStartOfChain().getInput().getOutputShape(); + cppCtx.getPrunedColLists().put(op, outputShape.getColumnNames()); return null; } - private static ArrayList buildPrunedRR(List prunedCols, - RowSchema oldRS) throws SemanticException{ + private List buildPrunedRS(List prunedCols, RowSchema oldRS) + throws SemanticException { ArrayList sig = new ArrayList(); HashSet prunedColsSet = new HashSet(prunedCols); - for(ColumnInfo cInfo : oldRS.getSignature()) { - if ( prunedColsSet.contains(cInfo.getInternalName())) { + for (ColumnInfo cInfo : oldRS.getSignature()) { + if (prunedColsSet.contains(cInfo.getInternalName())) { sig.add(cInfo); } } @@ -305,48 +309,74 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, return columns; } + private RowResolver buildPrunedRR(List prunedCols, RowSchema oldRS) + throws SemanticException { + RowResolver resolver = new RowResolver(); + HashSet prunedColsSet = new HashSet(prunedCols); + for (ColumnInfo cInfo : oldRS.getSignature()) { + if (prunedColsSet.contains(cInfo.getInternalName())) { + resolver.put(cInfo.getTabAlias(), cInfo.getAlias(), cInfo); + } + } + return resolver; + } + /* * add any input columns referenced in WindowFn args or expressions. */ - private ArrayList prunedColumnsList(List prunedCols, - WindowTableFunctionDef tDef) { - //we create a copy of prunedCols to create a list of pruned columns for PTFOperator - ArrayList mergedColList = new ArrayList(prunedCols); - if ( tDef.getWindowFunctions() != null ) { - for(WindowFunctionDef wDef : tDef.getWindowFunctions() ) { - if ( wDef.getArgs() == null) { - continue; + private List prunedColumnsList(List prunedCols, RowSchema oldRS, + PartitionedTableFunctionDef pDef) throws SemanticException { + pDef.getOutputShape().setRr(null); + pDef.getOutputShape().setColumnNames(null); + if (pDef instanceof WindowTableFunctionDef) { + WindowTableFunctionDef tDef = (WindowTableFunctionDef) pDef; + if (tDef.getWindowFunctions() != null) { + for (WindowFunctionDef wDef : tDef.getWindowFunctions()) { + if (wDef.getArgs() == null) { + continue; + } + for (PTFExpressionDef arg : wDef.getArgs()) { + ExprNodeDesc exprNode = arg.getExprNode(); + Utilities.mergeUniqElems(prunedCols, exprNode.getCols()); + } + } + } + if (tDef.getPartition() != null) { + for (PTFExpressionDef col : tDef.getPartition().getExpressions()) { + ExprNodeDesc exprNode = col.getExprNode(); + Utilities.mergeUniqElems(prunedCols, exprNode.getCols()); } - for(PTFExpressionDef arg : wDef.getArgs()) { - ExprNodeDesc exprNode = arg.getExprNode(); - Utilities.mergeUniqElems(mergedColList, exprNode.getCols()); + } + if (tDef.getOrder() != null) { + for (PTFExpressionDef col : tDef.getOrder().getExpressions()) { + ExprNodeDesc exprNode = col.getExprNode(); + Utilities.mergeUniqElems(prunedCols, exprNode.getCols()); } } + } else { + pDef.getOutputShape().setRr(buildPrunedRR(prunedCols, oldRS)); } - if(tDef.getPartition() != null){ - for(PTFExpressionDef col : tDef.getPartition().getExpressions()){ - ExprNodeDesc exprNode = col.getExprNode(); - Utilities.mergeUniqElems(mergedColList, exprNode.getCols()); - } - } - if(tDef.getOrder() != null){ - for(PTFExpressionDef col : tDef.getOrder().getExpressions()){ - ExprNodeDesc exprNode = col.getExprNode(); - Utilities.mergeUniqElems(mergedColList, exprNode.getCols()); - } - } - return mergedColList; + + PTFInputDef input = pDef.getInput(); + if (input instanceof PartitionedTableFunctionDef) { + return prunedColumnsList(prunedCols, oldRS, (PartitionedTableFunctionDef)input); + } + + ArrayList inputColumns = prunedInputList(prunedCols, input); + input.getOutputShape().setRr(buildPrunedRR(inputColumns, oldRS)); + input.getOutputShape().setColumnNames(inputColumns); + + return buildPrunedRS(prunedCols, oldRS); } /* * from the prunedCols list filter out columns that refer to WindowFns or WindowExprs * the returned list is set as the prunedList needed by the PTFOp. */ - private ArrayList prunedInputList(List prunedCols, - WindowTableFunctionDef tDef) { + private ArrayList prunedInputList(List prunedCols, PTFInputDef tDef) { ArrayList prunedInputCols = new ArrayList(); - StructObjectInspector OI = tDef.getInput().getOutputShape().getOI(); + StructObjectInspector OI = tDef.getOutputShape().getOI(); for(StructField f : OI.getAllStructFieldRefs()) { String fName = f.getFieldName(); if ( prunedCols.contains(fName)) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java index fad582a..1aec307 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java @@ -90,7 +90,7 @@ import org.apache.hadoop.hive.ql.plan.FetchWork; import org.apache.hadoop.hive.ql.plan.FileMergeDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; -import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; +import org.apache.hadoop.hive.ql.plan.FilterDesc.SampleDesc; import org.apache.hadoop.hive.ql.plan.LoadFileDesc; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.ql.plan.MapredLocalWork; @@ -586,7 +586,7 @@ public static void setMapWork(MapWork plan, ParseContext parseCtx, Set partToPruner = parseCtx.getOpToPartToSkewedPruner().get(topOp); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java index b0f4b47..37f9473 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SamplePruner.java @@ -48,7 +48,7 @@ import org.apache.hadoop.hive.ql.parse.ParseContext; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.FilterDesc; -import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; +import org.apache.hadoop.hive.ql.plan.FilterDesc.SampleDesc; /** * The transformation step that does sample pruning. @@ -61,17 +61,17 @@ * */ public static class SamplePrunerCtx implements NodeProcessorCtx { - HashMap opToSamplePruner; + HashMap opToSamplePruner; public SamplePrunerCtx( - HashMap opToSamplePruner) { + HashMap opToSamplePruner) { this.opToSamplePruner = opToSamplePruner; } /** * @return the opToSamplePruner */ - public HashMap getOpToSamplePruner() { + public HashMap getOpToSamplePruner() { return opToSamplePruner; } @@ -80,7 +80,7 @@ public SamplePrunerCtx( * the opToSamplePruner to set */ public void setOpToSamplePruner( - HashMap opToSamplePruner) { + HashMap opToSamplePruner) { this.opToSamplePruner = opToSamplePruner; } } @@ -135,7 +135,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Object... nodeOutputs) throws SemanticException { FilterOperator filOp = (FilterOperator) nd; FilterDesc filOpDesc = filOp.getConf(); - sampleDesc sampleDescr = filOpDesc.getSampleDescr(); + SampleDesc sampleDescr = filOpDesc.getSampleDescr(); if ((sampleDescr == null) || !sampleDescr.getInputPruning()) { return null; @@ -182,7 +182,7 @@ public static NodeProcessor getDefaultProc() { * @throws SemanticException */ @SuppressWarnings("nls") - public static Path[] prune(Partition part, sampleDesc sampleDescr) + public static Path[] prune(Partition part, SampleDesc sampleDescr) throws SemanticException { int num = sampleDescr.getNumerator(); int den = sampleDescr.getDenominator(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java index 6c1ab07..3c6d88a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java @@ -43,7 +43,7 @@ import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner; import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; +import org.apache.hadoop.hive.ql.plan.FilterDesc.SampleDesc; import org.apache.hadoop.hive.ql.plan.LoadFileDesc; import org.apache.hadoop.hive.ql.plan.LoadTableDesc; import org.apache.hadoop.hive.ql.plan.MapJoinDesc; @@ -64,7 +64,7 @@ private QB qb; private HashMap opToPartPruner; private HashMap opToPartList; - private HashMap opToSamplePruner; + private HashMap opToSamplePruner; private Map> opToPartToSkewedPruner; private HashMap> topOps; private Set joinOps; @@ -152,7 +152,7 @@ public ParseContext( Context ctx, HashMap idToTableNameMap, int destTableId, UnionProcContext uCtx, List> listMapJoinOpsNoReducer, Map prunedPartitions, - HashMap opToSamplePruner, + HashMap opToSamplePruner, GlobalLimitCtx globalLimitCtx, HashMap nameToSplitSample, HashSet semanticInputs, List> rootTasks, @@ -373,7 +373,7 @@ public void setListMapJoinOpsNoReducer( /** * @return the opToSamplePruner */ - public HashMap getOpToSamplePruner() { + public HashMap getOpToSamplePruner() { return opToSamplePruner; } @@ -382,7 +382,7 @@ public void setListMapJoinOpsNoReducer( * the opToSamplePruner to set */ public void setOpToSamplePruner( - HashMap opToSamplePruner) { + HashMap opToSamplePruner) { this.opToSamplePruner = opToSamplePruner; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index c69ed69..28a6c7c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -152,7 +152,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; -import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; +import org.apache.hadoop.hive.ql.plan.FilterDesc.SampleDesc; import org.apache.hadoop.hive.ql.plan.ForwardDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.HiveOperation; @@ -244,7 +244,7 @@ private int destTableId; private UnionProcContext uCtx; List> listMapJoinOpsNoReducer; - private HashMap opToSamplePruner; + private HashMap opToSamplePruner; private final Map> opToPartToSkewedPruner; /** * a map for the split sampling, from alias to an instance of SplitSample @@ -300,7 +300,7 @@ public SemanticAnalyzer(HiveConf conf) throws SemanticException { super(conf); opToPartPruner = new HashMap(); opToPartList = new HashMap(); - opToSamplePruner = new HashMap(); + opToSamplePruner = new HashMap(); nameToSplitSample = new HashMap(); // Must be deterministic order maps - see HIVE-8707 topOps = new LinkedHashMap>(); @@ -9195,7 +9195,7 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { ExprNodeDesc samplePredicate = genSamplePredicate(ts, tabBucketCols, colsEqual, alias, rwsch, qb.getMetaData(), null); tableOp = OperatorFactory.getAndMakeChild(new FilterDesc( - samplePredicate, true, new sampleDesc(ts.getNumerator(), ts + samplePredicate, true, new SampleDesc(ts.getNumerator(), ts .getDenominator(), tabBucketCols, true)), new RowSchema(rwsch.getColumnInfos()), top); } else { @@ -9236,7 +9236,7 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { .getBucketCols(), true, alias, rwsch, qb.getMetaData(), null); tableOp = OperatorFactory .getAndMakeChild(new FilterDesc(samplePred, true, - new sampleDesc(tsSample.getNumerator(), tsSample + new SampleDesc(tsSample.getNumerator(), tsSample .getDenominator(), tab.getBucketCols(), true)), new RowSchema(rwsch.getColumnInfos()), top); LOG.info("No need for sample filter"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java index 5856743..22fd29e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/FilterDesc.java @@ -31,7 +31,7 @@ /** * sampleDesc is used to keep track of the sampling descriptor. */ - public static class sampleDesc implements Cloneable { + public static class SampleDesc implements Cloneable { // The numerator of the TABLESAMPLE clause private int numerator; @@ -41,11 +41,11 @@ // Input files can be pruned private boolean inputPruning; - public sampleDesc() { + public SampleDesc() { } - public sampleDesc(int numerator, int denominator, - List tabBucketCols, boolean inputPruning) { + public SampleDesc(int numerator, int denominator, + List tabBucketCols, boolean inputPruning) { this.numerator = numerator; this.denominator = denominator; this.inputPruning = inputPruning; @@ -65,15 +65,19 @@ public boolean getInputPruning() { @Override public Object clone() { - sampleDesc desc = new sampleDesc(numerator, denominator, null, inputPruning); + SampleDesc desc = new SampleDesc(numerator, denominator, null, inputPruning); return desc; } + + public String toString() { + return inputPruning ? "BUCKET " + numerator + " OUT OF " + denominator: null; + } } private static final long serialVersionUID = 1L; private org.apache.hadoop.hive.ql.plan.ExprNodeDesc predicate; private boolean isSamplingPred; - private transient sampleDesc sampleDescr; + private transient SampleDesc sampleDescr; //Is this a filter that should perform a comparison for sorted searches private boolean isSortedFilter; @@ -90,7 +94,7 @@ public FilterDesc( public FilterDesc( final org.apache.hadoop.hive.ql.plan.ExprNodeDesc predicate, - boolean isSamplingPred, final sampleDesc sampleDescr) { + boolean isSamplingPred, final SampleDesc sampleDescr) { this.predicate = predicate; this.isSamplingPred = isSamplingPred; this.sampleDescr = sampleDescr; @@ -121,15 +125,19 @@ public void setIsSamplingPred(final boolean isSamplingPred) { this.isSamplingPred = isSamplingPred; } - @Explain(displayName = "sampleDesc", normalExplain = false) - public sampleDesc getSampleDescr() { + public SampleDesc getSampleDescr() { return sampleDescr; } - public void setSampleDescr(final sampleDesc sampleDescr) { + public void setSampleDescr(final SampleDesc sampleDescr) { this.sampleDescr = sampleDescr; } + @Explain(displayName = "sampleDesc", normalExplain = false) + public String getSampleDescExpr() { + return sampleDescr == null ? null : sampleDescr.toString(); + } + public boolean isSortedFilter() { return isSortedFilter; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java index 3ac3245..2f31eed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java @@ -22,14 +22,16 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.PTFUtils; import org.apache.hadoop.hive.ql.parse.LeadLagInfo; -import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order; -import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputType; +import org.apache.hadoop.hive.ql.plan.ptf.PTFInputDef; import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + @Explain(displayName = "PTF Operator") public class PTFDesc extends AbstractOperatorDesc { private static final long serialVersionUID = 1L; @@ -62,6 +64,19 @@ public PartitionedTableFunctionDef getStartOfChain() { return funcDef == null ? null : funcDef.getStartOfChain(); } + @Explain(displayName = "Function definitions") + public List getFuncDefExplain() { + if (funcDef == null) { + return null; + } + List inputs = new ArrayList(); + for (PTFInputDef current = funcDef; current != null; current = current.getInput()) { + inputs.add(current); + } + Collections.reverse(inputs); + return inputs; + } + public LeadLagInfo getLlInfo() { return llInfo; } @@ -70,10 +85,19 @@ public void setLlInfo(LeadLagInfo llInfo) { this.llInfo = llInfo; } + @Explain(displayName = "Lead/Lag information") + public String getLlInfoExplain() { + if (llInfo != null && llInfo.getLeadLagExprs() != null) { + return PlanUtils.getExprListString(llInfo.getLeadLagExprs()); + } + return null; + } + public boolean forWindowing() { - return funcDef != null && (funcDef instanceof WindowTableFunctionDef); + return funcDef instanceof WindowTableFunctionDef; } + @Explain(displayName = "Map-side function", displayOnlyOnTrue = true) public boolean isMapSide() { return isMapSide; } @@ -89,5 +113,4 @@ public Configuration getCfg() { public void setCfg(Configuration cfg) { this.cfg = cfg; } - } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index cea9fe2..36e3512 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -924,7 +924,7 @@ public static ReadEntity addInput(Set inputs, ReadEntity newInput) { return null; } - public static String getExprListString(Collection exprs) { + public static String getExprListString(Collection exprs) { StringBuffer sb = new StringBuffer(); boolean first = true; for (ExprNodeDesc expr: exprs) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java index 07590c0..f692fa2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java @@ -32,4 +32,10 @@ public void setDirection(Direction direction) { } public abstract int getAmt(); + + @Override + public String toString() { + return direction == null ? "" : + direction + "(" + (getAmt() == Integer.MAX_VALUE ? "MAX" : getAmt()) + ")"; + } } \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java index 5d200fb..a0370bf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.PTFUtils; +import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -58,6 +59,11 @@ public void setExprNode(ExprNodeDesc exprNode) { this.exprNode = exprNode; } + @Explain(displayName = "expr") + public String getExprNodeExplain() { + return exprNode == null ? null : exprNode.getExprString(); + } + public ExprNodeEvaluator getExprEvaluator() { return exprEvaluator; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java index 19ed2f2..95296c0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java @@ -19,6 +19,10 @@ package org.apache.hadoop.hive.ql.plan.ptf; +import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.util.StringUtils; + public abstract class PTFInputDef { private String expressionTreeString; private ShapeDetails outputShape; @@ -36,9 +40,17 @@ public ShapeDetails getOutputShape() { return outputShape; } + @Explain(displayName = "output shape") + public String getOutputShapeExplain() { + RowSchema schema = outputShape.getRr().getRowSchema(); + return StringUtils.join(", ", schema.getSignature()); + } + public void setOutputShape(ShapeDetails outputShape) { this.outputShape = outputShape; } + + @Explain(displayName = "input alias") public String getAlias() { return alias; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java index 11ef932..227b117 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java @@ -19,11 +19,14 @@ package org.apache.hadoop.hive.ql.plan.ptf; import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputType; +import org.apache.hadoop.hive.ql.plan.Explain; +@Explain(displayName = "Input definition") public class PTFQueryInputDef extends PTFInputDef { private String destination; private PTFQueryInputType type; + @Explain(displayName = "destination") public String getDestination() { return destination; } @@ -40,6 +43,11 @@ public void setType(PTFQueryInputType type) { this.type = type; } + @Explain(displayName = "type") + public String getTypeExplain() { + return type.name(); + } + @Override public PTFInputDef getInput() { return null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java index 327304c..967caaa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java @@ -21,8 +21,11 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec; +import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator; +@Explain(displayName = "Partition table definition") public class PartitionedTableFunctionDef extends PTFInputDef { private String name; private String resolverClassName; @@ -35,6 +38,7 @@ private TableFunctionEvaluator tFunction; boolean transformsRawInput; + @Explain(displayName = "name") public String getName() { return name; } @@ -47,6 +51,11 @@ public ShapeDetails getRawInputShape() { return rawInputShape; } + @Explain(displayName = "raw input shape") + public ShapeDetails getRawInputShapeExplain() { + return rawInputShape; + } + public void setRawInputShape(ShapeDetails rawInputShape) { this.rawInputShape = rawInputShape; } @@ -72,6 +81,21 @@ public PartitionDef getPartition() { return partition; } + @Explain(displayName = "partition by") + public String getPartitionExplain() { + if (partition == null || partition.getExpressions() == null) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (PTFExpressionDef expression : partition.getExpressions()) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(expression.getExprNode().getExprString()); + } + return builder.toString(); + } + public void setPartition(PartitionDef partition) { this.partition = partition; } @@ -84,9 +108,28 @@ public void setOrder(OrderDef order) { this.order = order; } + @Explain(displayName = "order by") + public String getOrderExplain() { + if (order == null || order.getExpressions() == null) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (OrderExpressionDef expression : order.getExpressions()) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(expression.getExprNode().getExprString()); + if (expression.getOrder() == PTFInvocationSpec.Order.DESC) { + builder.append("(DESC)"); + } + } + return builder.toString(); + } + public TableFunctionEvaluator getTFunction() { return tFunction; } + public void setTFunction(TableFunctionEvaluator tFunction) { this.tFunction = tFunction; } @@ -99,6 +142,21 @@ public void setArgs(List args) { this.args = args; } + @Explain(displayName = "arguments") + public String getArgsExplain() { + if (args == null) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (PTFExpressionDef expression : args) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(expression.getExprNode().getExprString()); + } + return builder.toString(); + } + public void addArg(PTFExpressionDef arg) { args = args == null ? new ArrayList() : args; args.add(arg); @@ -111,6 +169,7 @@ public PartitionedTableFunctionDef getStartOfChain() { return this; } + @Explain(displayName = "transforms raw input", displayOnlyOnTrue=true) public boolean isTransformsRawInput() { return transformsRawInput; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java index b96e9d6..de18575 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java @@ -19,15 +19,12 @@ package org.apache.hadoop.hive.ql.plan.ptf; -public class WindowExpressionDef extends PTFExpressionDef { - private String alias; - - public WindowExpressionDef() {} +import org.apache.hadoop.hive.ql.plan.Explain; - public WindowExpressionDef(PTFExpressionDef eDef) { - super(eDef); - } +public abstract class WindowExpressionDef extends PTFExpressionDef { + private String alias; + @Explain(displayName = "alias") public String getAlias() { return alias; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java index 949ed10..e08bdd5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java @@ -38,4 +38,9 @@ public BoundaryDef getEnd() { public void setEnd(BoundaryDef end) { this.end = end; } + + @Override + public String toString() { + return start + "~" + end; + } } \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java index e4ea358..ed6c671 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java @@ -21,8 +21,10 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +@Explain(displayName = "window function definition") public class WindowFunctionDef extends WindowExpressionDef { String name; boolean isStar; @@ -32,6 +34,7 @@ GenericUDAFEvaluator wFnEval; boolean pivotResult; + @Explain(displayName = "name") public String getName() { return name; } @@ -40,6 +43,7 @@ public void setName(String name) { this.name = name; } + @Explain(displayName = "isStar", displayOnlyOnTrue = true) public boolean isStar() { return isStar; } @@ -48,6 +52,7 @@ public void setStar(boolean isStar) { this.isStar = isStar; } + @Explain(displayName = "isDistinct", displayOnlyOnTrue = true) public boolean isDistinct() { return isDistinct; } @@ -69,6 +74,21 @@ public void addArg(PTFExpressionDef arg) { args.add(arg); } + @Explain(displayName = "arguments") + public String getArgsExplain() { + if (args == null) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (PTFExpressionDef expression : args) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(expression.getExprNode().getExprString()); + } + return builder.toString(); + } + public WindowFrameDef getWindowFrame() { return windowFrame; } @@ -77,6 +97,11 @@ public void setWindowFrame(WindowFrameDef windowFrame) { this.windowFrame = windowFrame; } + @Explain(displayName = "window frame") + public String getWindowFrameExplain() { + return windowFrame == null ? null : windowFrame.toString(); + } + public GenericUDAFEvaluator getWFnEval() { return wFnEval; } @@ -85,6 +110,12 @@ public void setWFnEval(GenericUDAFEvaluator wFnEval) { this.wFnEval = wFnEval; } + @Explain(displayName = "window function") + public String getWFnEvalExplain() { + return wFnEval == null ? null : wFnEval.getClass().getSimpleName(); + } + + @Explain(displayName = "isPivotResult", displayOnlyOnTrue = true) public boolean isPivotResult() { return pivotResult; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java index 083aaf2..97ba17e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java @@ -18,21 +18,26 @@ package org.apache.hadoop.hive.ql.plan.ptf; -import java.util.List; +import org.apache.hadoop.hive.ql.plan.Explain; +import java.util.List; +@Explain(displayName = "Windowing table definition") public class WindowTableFunctionDef extends PartitionedTableFunctionDef { List windowFunctions; int rankLimit = -1; int rankLimitFunction; + @Explain(displayName = "window functions") public List getWindowFunctions() { return windowFunctions; } + public void setWindowFunctions(List windowFunctions) { this.windowFunctions = windowFunctions; } + public int getRankLimit() { return rankLimit; } diff --git a/ql/src/test/queries/clientpositive/ptf_matchpath.q b/ql/src/test/queries/clientpositive/ptf_matchpath.q index 80dbe29..6487135 100644 --- a/ql/src/test/queries/clientpositive/ptf_matchpath.q +++ b/ql/src/test/queries/clientpositive/ptf_matchpath.q @@ -15,6 +15,17 @@ LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt' OVERWRITE INTO TABLE -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ); + select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -26,6 +37,17 @@ from matchpath(on ); -- 2. Matchpath on 1 partition +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142; + select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -37,6 +59,17 @@ from matchpath(on where fl_num = 1142; -- 3. empty partition. +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ); + + select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny diff --git a/ql/src/test/results/clientpositive/correlationoptimizer12.q.out b/ql/src/test/results/clientpositive/correlationoptimizer12.q.out index 1333a8a..acdf8c6 100644 --- a/ql/src/test/results/clientpositive/correlationoptimizer12.q.out +++ b/ql/src/test/results/clientpositive/correlationoptimizer12.q.out @@ -43,6 +43,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -114,6 +132,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) diff --git a/ql/src/test/results/clientpositive/ctas_colname.q.out b/ql/src/test/results/clientpositive/ctas_colname.q.out index 4eb827c..a83af21 100644 --- a/ql/src/test/results/clientpositive/ctas_colname.q.out +++ b/ql/src/test/results/clientpositive/ctas_colname.q.out @@ -189,6 +189,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _wcol0 (type: int) @@ -336,6 +355,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col0, 1 + name: lead + window function: GenericUDAFLeadEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _wcol0 (type: string) diff --git a/ql/src/test/results/clientpositive/groupby_grouping_window.q.out b/ql/src/test/results/clientpositive/groupby_grouping_window.q.out index c0e6e5f..b2434c5 100644 --- a/ql/src/test/results/clientpositive/groupby_grouping_window.q.out +++ b/ql/src/test/results/clientpositive/groupby_grouping_window.q.out @@ -93,6 +93,25 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3 Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col2: int, _col3: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col3 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _wcol0 (type: int) diff --git a/ql/src/test/results/clientpositive/groupby_resolution.q.out b/ql/src/test/results/clientpositive/groupby_resolution.q.out index e5d22a3..c371f5c 100644 --- a/ql/src/test/results/clientpositive/groupby_resolution.q.out +++ b/ql/src/test/results/clientpositive/groupby_resolution.q.out @@ -677,6 +677,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: bigint + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: 0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _wcol0 (type: int) diff --git a/ql/src/test/results/clientpositive/ptf.q.out b/ql/src/test/results/clientpositive/ptf.q.out index b82defd..4238cdf 100644 --- a/ql/src/test/results/clientpositive/ptf.q.out +++ b/ql/src/test/results/clientpositive/ptf.q.out @@ -48,6 +48,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -72,6 +84,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -220,6 +264,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -244,6 +300,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -340,10 +415,22 @@ STAGE PLANS: value expressions: p_size (type: int) Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) - outputColumnNames: _col2, _col1, _col5 + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -449,6 +536,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -473,6 +572,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -586,6 +717,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -610,6 +753,39 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -726,6 +902,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -781,6 +969,39 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -889,6 +1110,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1028,6 +1261,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1160,7 +1405,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -1172,6 +1431,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1195,6 +1467,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -1289,7 +1580,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -1302,6 +1607,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1326,6 +1644,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1435,6 +1785,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1459,6 +1821,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1571,9 +1965,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -1597,6 +2024,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1621,6 +2068,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1740,6 +2219,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1764,6 +2255,30 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -1888,6 +2403,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1952,6 +2479,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -2063,6 +2635,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2220,6 +2804,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 + raw input shape: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2244,6 +2840,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -2406,6 +3020,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2436,6 +3062,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -2479,6 +3137,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _wcol0 (type: bigint) @@ -2507,6 +3183,45 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + raw input shape: + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -2713,9 +3428,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -2739,6 +3494,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2763,6 +3538,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2906,6 +3713,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2930,6 +3756,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2954,6 +3792,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2978,6 +3828,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3116,6 +3998,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3140,6 +4041,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3164,6 +4084,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3303,6 +4255,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3327,9 +4298,35 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -3353,6 +4350,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3377,6 +4387,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3517,9 +4559,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -3543,6 +4618,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3567,6 +4662,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -3701,9 +4828,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -3727,6 +4887,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3751,6 +4924,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/ptf_matchpath.q.out b/ql/src/test/results/clientpositive/ptf_matchpath.q.out index e0cea0d..aaa66cf 100644 --- a/ql/src/test/results/clientpositive/ptf_matchpath.q.out +++ b/ql/src/test/results/clientpositive/ptf_matchpath.q.out @@ -37,6 +37,7 @@ POSTHOOK: Output: default@flights_tiny PREHOOK: query: -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -47,11 +48,10 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -62,6 +62,83 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: ++++ + Map-reduce partition columns: fl_num (type: string) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), arr_delay (type: float), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), VALUE._col2 (type: float), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: _col6 + raw input shape: + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### Baltimore 1142 2010 10 20 6 20 @@ -81,6 +158,7 @@ Chicago 897 2010 10 21 3 21 Chicago 897 2010 10 22 2 22 Washington 7291 2010 10 27 2 27 PREHOOK: query: -- 2. Matchpath on 1 partition +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -91,9 +169,8 @@ from matchpath(on ) where fl_num = 1142 PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- 2. Matchpath on 1 partition +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -104,6 +181,86 @@ from matchpath(on ) where fl_num = 1142 POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), arr_delay (type: float), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col2 (type: float), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + raw input shape: + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = 1142) (type: boolean) + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### Baltimore 1142 2010 10 20 6 20 @@ -112,6 +269,7 @@ Baltimore 1142 2010 10 22 4 22 Baltimore 1142 2010 10 25 3 25 Baltimore 1142 2010 10 26 2 26 PREHOOK: query: -- 3. empty partition. +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny @@ -121,9 +279,8 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- 3. empty partition. +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny @@ -133,5 +290,87 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 24 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = -1142) (type: boolean) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), '-1142' (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col5 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col2 (type: float), '-1142' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string + type: SUBQUERY + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + raw input shape: + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '-1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/ptf_streaming.q.out b/ql/src/test/results/clientpositive/ptf_streaming.q.out index 6552c3a6..b3605c8 100644 --- a/ql/src/test/results/clientpositive/ptf_streaming.q.out +++ b/ql/src/test/results/clientpositive/ptf_streaming.q.out @@ -48,6 +48,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -72,6 +84,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -220,6 +264,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -244,6 +300,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -347,6 +422,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -475,7 +562,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -487,6 +588,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -510,6 +624,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -604,7 +737,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -617,6 +764,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -641,6 +801,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -753,9 +945,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -779,6 +1004,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -803,6 +1048,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -917,9 +1194,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -943,6 +1253,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -967,6 +1297,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1081,9 +1443,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -1107,6 +1502,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1131,6 +1546,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1249,6 +1696,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1313,6 +1772,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1447,9 +1951,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -1473,6 +2017,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1497,6 +2061,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1640,6 +2236,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1664,6 +2279,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1688,6 +2315,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1712,6 +2351,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1848,9 +2519,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -1874,6 +2578,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1898,6 +2615,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/quotedid_basic.q.out b/ql/src/test/results/clientpositive/quotedid_basic.q.out index 6c7a017..2dbe996 100644 --- a/ql/src/test/results/clientpositive/quotedid_basic.q.out +++ b/ql/src/test/results/clientpositive/quotedid_basic.q.out @@ -198,6 +198,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _wcol0 (type: int) @@ -283,6 +302,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _wcol0 (type: int) diff --git a/ql/src/test/results/clientpositive/sample10.q.out b/ql/src/test/results/clientpositive/sample10.q.out index 6495fb8..a6bc91a 100644 --- a/ql/src/test/results/clientpositive/sample10.q.out +++ b/ql/src/test/results/clientpositive/sample10.q.out @@ -97,6 +97,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 4) = 0) (type: boolean) + sampleDesc: BUCKET 1 OUT OF 4 Statistics: Num rows: 20 Data size: 120 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) diff --git a/ql/src/test/results/clientpositive/sample2.q.out b/ql/src/test/results/clientpositive/sample2.q.out index 5b6b0f5..b8e0686 100644 --- a/ql/src/test/results/clientpositive/sample2.q.out +++ b/ql/src/test/results/clientpositive/sample2.q.out @@ -62,6 +62,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + sampleDesc: BUCKET 1 OUT OF 2 Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) diff --git a/ql/src/test/results/clientpositive/sample4.q.out b/ql/src/test/results/clientpositive/sample4.q.out index b39cc2c..e1bc757 100644 --- a/ql/src/test/results/clientpositive/sample4.q.out +++ b/ql/src/test/results/clientpositive/sample4.q.out @@ -64,6 +64,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + sampleDesc: BUCKET 1 OUT OF 2 Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) diff --git a/ql/src/test/results/clientpositive/sample6.q.out b/ql/src/test/results/clientpositive/sample6.q.out index dd8d288..e1914fb 100644 --- a/ql/src/test/results/clientpositive/sample6.q.out +++ b/ql/src/test/results/clientpositive/sample6.q.out @@ -62,6 +62,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 4) = 0) (type: boolean) + sampleDesc: BUCKET 1 OUT OF 4 Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) @@ -645,6 +646,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 4) = 3) (type: boolean) + sampleDesc: BUCKET 4 OUT OF 4 Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) @@ -1042,6 +1044,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + sampleDesc: BUCKET 1 OUT OF 2 Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) @@ -2667,6 +2670,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + sampleDesc: BUCKET 1 OUT OF 2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) @@ -2994,6 +2998,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 4) = 1) (type: boolean) + sampleDesc: BUCKET 2 OUT OF 4 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) diff --git a/ql/src/test/results/clientpositive/sample9.q.out b/ql/src/test/results/clientpositive/sample9.q.out index a8c2e82..b5e625f 100644 --- a/ql/src/test/results/clientpositive/sample9.q.out +++ b/ql/src/test/results/clientpositive/sample9.q.out @@ -58,6 +58,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + sampleDesc: BUCKET 1 OUT OF 2 Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) diff --git a/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out b/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out index 056c9f1..691cb46 100644 --- a/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out +++ b/ql/src/test/results/clientpositive/smb_mapjoin_11.q.out @@ -1863,6 +1863,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 16) = 1) (type: boolean) + sampleDesc: BUCKET 2 OUT OF 16 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string), ds (type: string) @@ -1987,6 +1988,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 16) = 1) (type: boolean) + sampleDesc: BUCKET 2 OUT OF 16 Statistics: Num rows: 514 Data size: 5484 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string), ds (type: string) diff --git a/ql/src/test/results/clientpositive/spark/ptf.q.out b/ql/src/test/results/clientpositive/spark/ptf.q.out index 2e12c1d..95592d5 100644 --- a/ql/src/test/results/clientpositive/spark/ptf.q.out +++ b/ql/src/test/results/clientpositive/spark/ptf.q.out @@ -54,6 +54,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -68,6 +80,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -214,6 +258,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -228,6 +284,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -330,10 +405,22 @@ STAGE PLANS: Reducer 2 Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) - outputColumnNames: _col2, _col1, _col5 + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -445,6 +532,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -459,6 +558,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -578,6 +709,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -592,6 +735,39 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -714,6 +890,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -749,6 +937,39 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -876,6 +1097,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1034,6 +1267,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1129,7 +1374,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -1142,6 +1401,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1155,6 +1427,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -1254,7 +1545,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -1268,6 +1573,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1282,6 +1600,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1397,6 +1747,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1411,6 +1773,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1529,9 +1923,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: -+ @@ -1545,6 +1972,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1559,6 +2006,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1684,6 +2163,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1698,6 +2189,30 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -1841,6 +2356,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1874,6 +2401,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1991,6 +2563,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2144,6 +2728,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 + raw input shape: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) @@ -2158,6 +2754,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -2327,6 +2941,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -2347,6 +2993,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _wcol0 (type: bigint) @@ -2365,6 +3029,45 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + raw input shape: + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -2385,6 +3088,16 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2399,6 +3112,16 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col5 (type: int) @@ -2618,9 +3341,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -2634,6 +3397,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2648,6 +3431,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2797,6 +3612,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2811,6 +3645,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -2825,6 +3671,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2839,6 +3697,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2983,6 +3873,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -2997,6 +3906,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3011,6 +3939,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3156,6 +4116,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -3170,9 +4149,35 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -3186,6 +4191,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3200,6 +4218,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3346,9 +4396,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string) sort order: + @@ -3362,6 +4445,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3376,6 +4479,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -3516,9 +4651,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -3532,6 +4700,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3546,6 +4727,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out b/ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out index e0cea0d..22c8af3 100644 --- a/ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out +++ b/ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out @@ -37,6 +37,7 @@ POSTHOOK: Output: default@flights_tiny PREHOOK: query: -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -47,11 +48,10 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -62,6 +62,89 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: ++++ + Map-reduce partition columns: fl_num (type: string) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), arr_delay (type: float), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), VALUE._col2 (type: float), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: _col6 + raw input shape: + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### Baltimore 1142 2010 10 20 6 20 @@ -81,6 +164,7 @@ Chicago 897 2010 10 21 3 21 Chicago 897 2010 10 22 2 22 Washington 7291 2010 10 27 2 27 PREHOOK: query: -- 2. Matchpath on 1 partition +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -91,9 +175,8 @@ from matchpath(on ) where fl_num = 1142 PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- 2. Matchpath on 1 partition +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -104,6 +187,92 @@ from matchpath(on ) where fl_num = 1142 POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), arr_delay (type: float), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col2 (type: float), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + raw input shape: + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = 1142) (type: boolean) + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### Baltimore 1142 2010 10 20 6 20 @@ -112,6 +281,7 @@ Baltimore 1142 2010 10 22 4 22 Baltimore 1142 2010 10 25 3 25 Baltimore 1142 2010 10 26 2 26 PREHOOK: query: -- 3. empty partition. +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny @@ -121,9 +291,8 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- 3. empty partition. +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny @@ -133,5 +302,93 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 24 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = -1142) (type: boolean) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), '-1142' (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col5 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col2 (type: float), '-1142' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string + type: SUBQUERY + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + raw input shape: + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '-1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/ptf_streaming.q.out b/ql/src/test/results/clientpositive/spark/ptf_streaming.q.out index d759fac..a4bc083 100644 --- a/ql/src/test/results/clientpositive/spark/ptf_streaming.q.out +++ b/ql/src/test/results/clientpositive/spark/ptf_streaming.q.out @@ -54,6 +54,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -68,6 +80,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -214,6 +258,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -228,6 +284,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -350,6 +425,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -462,7 +549,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -475,6 +576,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -488,6 +602,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -587,7 +720,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -601,6 +748,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -615,6 +775,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -733,9 +925,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -749,6 +974,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -763,6 +1008,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -883,9 +1160,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -899,6 +1209,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -913,6 +1243,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1033,9 +1395,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -1049,6 +1444,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1063,6 +1478,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1200,6 +1647,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1233,6 +1692,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1373,9 +1877,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -1389,6 +1933,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1403,6 +1967,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1552,6 +2148,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1566,6 +2181,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -1580,6 +2207,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1594,6 +2233,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1736,9 +2407,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -1752,6 +2456,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1766,6 +2483,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/spark/sample10.q.out b/ql/src/test/results/clientpositive/spark/sample10.q.out index b63718b..2d6ca1b 100644 --- a/ql/src/test/results/clientpositive/spark/sample10.q.out +++ b/ql/src/test/results/clientpositive/spark/sample10.q.out @@ -102,6 +102,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 4) = 0) (type: boolean) + sampleDesc: BUCKET 1 OUT OF 4 Statistics: Num rows: 20 Data size: 120 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) diff --git a/ql/src/test/results/clientpositive/spark/sample2.q.out b/ql/src/test/results/clientpositive/spark/sample2.q.out index eac6fb4..d06291c 100644 --- a/ql/src/test/results/clientpositive/spark/sample2.q.out +++ b/ql/src/test/results/clientpositive/spark/sample2.q.out @@ -60,6 +60,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + sampleDesc: BUCKET 1 OUT OF 2 Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) diff --git a/ql/src/test/results/clientpositive/spark/sample4.q.out b/ql/src/test/results/clientpositive/spark/sample4.q.out index 8030f74..ecee494 100644 --- a/ql/src/test/results/clientpositive/spark/sample4.q.out +++ b/ql/src/test/results/clientpositive/spark/sample4.q.out @@ -62,6 +62,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + sampleDesc: BUCKET 1 OUT OF 2 Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) diff --git a/ql/src/test/results/clientpositive/spark/sample6.q.out b/ql/src/test/results/clientpositive/spark/sample6.q.out index 47da933..a44d21f 100644 --- a/ql/src/test/results/clientpositive/spark/sample6.q.out +++ b/ql/src/test/results/clientpositive/spark/sample6.q.out @@ -60,6 +60,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 4) = 0) (type: boolean) + sampleDesc: BUCKET 1 OUT OF 4 Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) @@ -495,6 +496,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 4) = 3) (type: boolean) + sampleDesc: BUCKET 4 OUT OF 4 Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) @@ -898,6 +900,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + sampleDesc: BUCKET 1 OUT OF 2 Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) @@ -2541,6 +2544,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + sampleDesc: BUCKET 1 OUT OF 2 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) @@ -2874,6 +2878,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 4) = 1) (type: boolean) + sampleDesc: BUCKET 2 OUT OF 4 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) diff --git a/ql/src/test/results/clientpositive/spark/sample9.q.out b/ql/src/test/results/clientpositive/spark/sample9.q.out index ff5d08e..c9823f7 100644 --- a/ql/src/test/results/clientpositive/spark/sample9.q.out +++ b/ql/src/test/results/clientpositive/spark/sample9.q.out @@ -58,6 +58,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 2) = 0) (type: boolean) + sampleDesc: BUCKET 1 OUT OF 2 Statistics: Num rows: 500 Data size: 5301 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string) diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out index 6881804..c3f996f 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_11.q.out @@ -1916,6 +1916,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 16) = 1) (type: boolean) + sampleDesc: BUCKET 2 OUT OF 16 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string), ds (type: string) @@ -2007,6 +2008,7 @@ STAGE PLANS: Filter Operator isSamplingPred: true predicate: (((hash(key) & 2147483647) % 16) = 1) (type: boolean) + sampleDesc: BUCKET 2 OUT OF 16 Statistics: Num rows: 514 Data size: 5484 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: key (type: int), value (type: string), ds (type: string) diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out index 53c28c2..c48482d 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out @@ -314,6 +314,25 @@ STAGE PLANS: outputColumnNames: _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -463,6 +482,25 @@ STAGE PLANS: outputColumnNames: _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col2 is not null) (type: boolean) diff --git a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out index 5b6242e..cbfec89 100644 --- a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out @@ -304,6 +304,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -321,6 +333,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -681,6 +725,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -698,6 +754,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -903,10 +978,22 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) - outputColumnNames: _col2, _col1, _col5 + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1167,6 +1254,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1184,6 +1283,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1456,6 +1587,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1473,6 +1616,39 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -1755,6 +1931,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1797,6 +1985,39 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -2091,6 +2312,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2435,6 +2668,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2588,7 +2833,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -2653,6 +2912,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2669,6 +2941,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -2863,7 +3154,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -2929,6 +3234,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2946,6 +3264,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3209,6 +3559,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3226,6 +3588,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3499,9 +3893,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -3518,6 +3945,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3535,6 +3982,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3828,6 +4307,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3845,6 +4336,30 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -4282,6 +4797,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -4322,6 +4849,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -4542,6 +5114,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -4824,6 +5408,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 + raw input shape: Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) @@ -4841,6 +5437,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -5271,6 +5885,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -5310,6 +5956,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _wcol0 (type: bigint) @@ -5331,6 +5995,45 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + raw input shape: + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -5370,6 +6073,16 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -5387,6 +6100,16 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col5 (type: int) @@ -5794,9 +6517,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -5813,6 +6576,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -5830,6 +6613,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6154,6 +6969,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6171,6 +7005,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -6188,6 +7034,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6205,6 +7063,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6512,6 +7402,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -6529,6 +7438,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6546,6 +7474,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6865,6 +7825,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -6882,9 +7861,35 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -6901,6 +7906,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6918,6 +7936,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -7268,9 +8318,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string) sort order: + @@ -7287,6 +8370,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -7304,6 +8407,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -7620,9 +8755,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -7639,6 +8807,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -7656,6 +8837,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/subquery_in.q.out b/ql/src/test/results/clientpositive/subquery_in.q.out index 3a5e77b..d4dec94 100644 --- a/ql/src/test/results/clientpositive/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/subquery_in.q.out @@ -269,6 +269,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -434,6 +453,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col0 is not null) (type: boolean) diff --git a/ql/src/test/results/clientpositive/subquery_in_having.q.out b/ql/src/test/results/clientpositive/subquery_in_having.q.out index 1046c9e..365c6ab 100644 --- a/ql/src/test/results/clientpositive/subquery_in_having.q.out +++ b/ql/src/test/results/clientpositive/subquery_in_having.q.out @@ -1348,6 +1348,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col0 + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(MAX)~ Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _wcol0 is not null (type: boolean) diff --git a/ql/src/test/results/clientpositive/subquery_notin.q.out b/ql/src/test/results/clientpositive/subquery_notin.q.out index 5811394..41e7239 100644 --- a/ql/src/test/results/clientpositive/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/subquery_notin.q.out @@ -337,6 +337,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and (_col0 is null or _col1 is null)) (type: boolean) @@ -477,6 +496,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -589,6 +627,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -721,6 +778,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -881,6 +957,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -1057,6 +1152,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) diff --git a/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out b/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out index 8c6b202..eae5c40 100644 --- a/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out +++ b/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out @@ -222,6 +222,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col0 is not null) (type: boolean) @@ -375,6 +394,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col0 is not null) (type: boolean) @@ -803,6 +841,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and (_col0 is null or _col1 is null)) (type: boolean) @@ -943,6 +1000,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) diff --git a/ql/src/test/results/clientpositive/tez/ptf.q.out b/ql/src/test/results/clientpositive/tez/ptf.q.out index 3f5f2c6..2a92d77 100644 --- a/ql/src/test/results/clientpositive/tez/ptf.q.out +++ b/ql/src/test/results/clientpositive/tez/ptf.q.out @@ -54,6 +54,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -68,6 +80,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -214,6 +258,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -228,6 +284,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -330,10 +405,22 @@ STAGE PLANS: Reducer 2 Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) - outputColumnNames: _col2, _col1, _col5 + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -445,6 +532,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -459,6 +558,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -578,6 +709,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -592,6 +735,39 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -714,6 +890,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -749,6 +937,39 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -876,6 +1097,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1034,6 +1267,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1129,7 +1374,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -1142,6 +1401,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1155,6 +1427,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -1254,7 +1545,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -1268,6 +1573,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1282,6 +1600,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1397,6 +1747,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1411,6 +1773,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1529,9 +1923,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: -+ @@ -1545,6 +1972,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2(DESC), _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1559,6 +2006,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1684,6 +2163,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1698,6 +2189,30 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -1841,6 +2356,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1874,6 +2401,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1991,6 +2563,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2144,6 +2728,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 + raw input shape: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) @@ -2158,6 +2754,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -2327,6 +2941,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2347,6 +2973,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -2367,6 +3025,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _wcol0 (type: bigint) @@ -2385,6 +3061,45 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + raw input shape: + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -2613,9 +3328,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -2629,6 +3384,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2643,6 +3418,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2792,6 +3599,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2806,6 +3632,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -2820,6 +3658,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2834,6 +3684,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2978,6 +3860,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -2992,6 +3893,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3006,6 +3926,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3151,6 +4103,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -3165,9 +4136,35 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -3181,6 +4178,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3195,6 +4205,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3341,9 +4383,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string) sort order: + @@ -3357,6 +4432,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3371,6 +4466,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -3511,9 +4638,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -3527,6 +4687,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3541,6 +4714,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/tez/ptf_matchpath.q.out b/ql/src/test/results/clientpositive/tez/ptf_matchpath.q.out new file mode 100644 index 0000000..fc2b034 --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/ptf_matchpath.q.out @@ -0,0 +1,394 @@ +PREHOOK: query: DROP TABLE flights_tiny +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE flights_tiny +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table flights_tiny ( +ORIGIN_CITY_NAME string, +DEST_CITY_NAME string, +YEAR int, +MONTH int, +DAY_OF_MONTH int, +ARR_DELAY float, +FL_NUM string +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: create table flights_tiny ( +ORIGIN_CITY_NAME string, +DEST_CITY_NAME string, +YEAR int, +MONTH int, +DAY_OF_MONTH int, +ARR_DELAY float, +FL_NUM string +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. basic Matchpath test +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. basic Matchpath test +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: ++++ + Map-reduce partition columns: fl_num (type: string) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), arr_delay (type: float), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), VALUE._col2 (type: float), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: _col6 + raw input shape: + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +Baltimore 1142 2010 10 20 6 20 +Baltimore 1142 2010 10 21 5 21 +Baltimore 1142 2010 10 22 4 22 +Baltimore 1142 2010 10 25 3 25 +Baltimore 1142 2010 10 26 2 26 +Baltimore 1599 2010 10 21 2 21 +Baltimore 1599 2010 10 25 3 25 +Baltimore 1599 2010 10 26 2 26 +Chicago 1531 2010 10 21 2 21 +Chicago 1531 2010 10 25 3 25 +Chicago 1531 2010 10 26 2 26 +Chicago 361 2010 10 20 2 20 +Chicago 897 2010 10 20 4 20 +Chicago 897 2010 10 21 3 21 +Chicago 897 2010 10 22 2 22 +Washington 7291 2010 10 27 2 27 +PREHOOK: query: -- 2. Matchpath on 1 partition +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +PREHOOK: type: QUERY +POSTHOOK: query: -- 2. Matchpath on 1 partition +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), arr_delay (type: float), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col2 (type: float), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + raw input shape: + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = 1142) (type: boolean) + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +Baltimore 1142 2010 10 20 6 20 +Baltimore 1142 2010 10 21 5 21 +Baltimore 1142 2010 10 22 4 22 +Baltimore 1142 2010 10 25 3 25 +Baltimore 1142 2010 10 26 2 26 +PREHOOK: query: -- 3. empty partition. +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +POSTHOOK: query: -- 3. empty partition. +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 24 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = -1142) (type: boolean) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), '-1142' (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col5 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col2 (type: float), '-1142' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string + type: SUBQUERY + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + raw input shape: + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '-1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out b/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out index 79cb92c..9be0819 100644 --- a/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out +++ b/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out @@ -54,6 +54,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -68,6 +80,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -214,6 +258,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -228,6 +284,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -350,6 +425,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -462,7 +549,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -475,6 +576,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -488,6 +602,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -587,7 +720,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -601,6 +748,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -615,6 +775,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -733,9 +925,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -749,6 +974,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -763,6 +1008,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -883,9 +1160,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -899,6 +1209,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -913,6 +1243,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1033,9 +1395,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -1049,6 +1444,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1063,6 +1478,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1200,6 +1647,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1233,6 +1692,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1373,9 +1877,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -1389,6 +1933,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1403,6 +1967,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1552,6 +2148,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1566,6 +2181,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -1580,6 +2207,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1594,6 +2233,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1736,9 +2407,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -1752,6 +2456,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1766,6 +2483,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/tez/subquery_in.q.out b/ql/src/test/results/clientpositive/tez/subquery_in.q.out index 69eb568..f84277c 100644 --- a/ql/src/test/results/clientpositive/tez/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/tez/subquery_in.q.out @@ -326,6 +326,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -483,6 +502,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col0 is not null) (type: boolean) diff --git a/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out index d5d6dc3..e9c2960 100644 --- a/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out @@ -304,6 +304,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -321,6 +333,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -682,6 +726,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -699,6 +755,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -904,10 +979,22 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) - outputColumnNames: _col2, _col1, _col5 + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1168,6 +1255,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1185,6 +1284,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1457,6 +1588,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1474,6 +1617,39 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -1756,6 +1932,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1798,6 +1986,39 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -2092,6 +2313,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2438,6 +2671,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2591,7 +2836,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -2656,6 +2915,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2672,6 +2944,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -2866,7 +3157,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -2932,6 +3237,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2949,6 +3267,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3212,6 +3562,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3229,6 +3591,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3502,9 +3896,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -3521,6 +3948,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3538,6 +3985,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3831,6 +4310,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3848,6 +4339,30 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -4285,6 +4800,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -4326,6 +4853,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -4546,6 +5118,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -4828,6 +5412,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 + raw input shape: Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) @@ -4845,6 +5441,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -5275,6 +5889,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -5300,6 +5926,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -5339,6 +5997,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _wcol0 (type: bigint) @@ -5360,6 +6036,45 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + raw input shape: + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -5792,9 +6507,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -5811,6 +6566,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -5828,6 +6603,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6152,6 +6959,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6169,6 +6995,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -6186,6 +7024,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6203,6 +7053,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6510,6 +7392,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -6527,6 +7428,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6544,6 +7464,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6863,6 +7815,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string) @@ -6880,9 +7851,35 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -6899,6 +7896,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6916,6 +7926,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -7266,9 +8308,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string) sort order: + @@ -7285,6 +8360,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -7302,6 +8397,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -7618,9 +8745,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) sort order: ++ @@ -7637,6 +8797,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -7654,6 +8827,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/union_remove_6_subq.q.out b/ql/src/test/results/clientpositive/union_remove_6_subq.q.out index be15c1f..98c0df1 100644 --- a/ql/src/test/results/clientpositive/union_remove_6_subq.q.out +++ b/ql/src/test/results/clientpositive/union_remove_6_subq.q.out @@ -546,6 +546,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: bigint + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: avg + window function: GenericUDAFAverageEvaluatorDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _wcol0 (type: double) diff --git a/ql/src/test/results/clientpositive/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/vectorized_ptf.q.out index 6bd98b7..ec10a92 100644 --- a/ql/src/test/results/clientpositive/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/vectorized_ptf.q.out @@ -298,6 +298,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -362,6 +374,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -711,6 +755,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -775,6 +831,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -974,10 +1049,22 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) - outputColumnNames: _col2, _col1, _col5 + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1232,6 +1319,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1296,6 +1395,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1562,6 +1693,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1626,6 +1769,39 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -1902,6 +2078,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2037,6 +2225,39 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -2258,6 +2479,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2600,6 +2833,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2891,7 +3136,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -2955,6 +3214,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3018,6 +3290,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -3207,7 +3498,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -3272,6 +3577,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3336,6 +3654,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3593,6 +3943,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3657,6 +4019,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3924,9 +4318,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -3990,6 +4417,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -4054,6 +4501,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -4341,6 +4820,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -4405,6 +4896,30 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -4769,6 +5284,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -4963,6 +5490,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -5177,6 +5749,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -5499,6 +6083,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 + raw input shape: Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -5563,6 +6159,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -5986,6 +6600,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -6067,6 +6693,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -6170,6 +6828,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _wcol0 (type: bigint) @@ -6238,6 +6914,45 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + raw input shape: + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -6635,9 +7350,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -6701,6 +7456,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -6765,6 +7540,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -7083,6 +7890,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7147,6 +7973,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7211,6 +8049,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7275,6 +8125,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -7576,6 +8458,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7640,6 +8541,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7704,6 +8624,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -8017,6 +8969,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -8081,9 +9052,35 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -8147,6 +9144,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -8211,6 +9221,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -8555,9 +9597,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -8621,6 +9696,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -8685,6 +9780,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -8995,9 +10122,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -9061,6 +10221,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -9125,6 +10298,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/windowing_streaming.q.out b/ql/src/test/results/clientpositive/windowing_streaming.q.out index d45646a..b17d96d 100644 --- a/ql/src/test/results/clientpositive/windowing_streaming.q.out +++ b/ql/src/test/results/clientpositive/windowing_streaming.q.out @@ -80,6 +80,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 + partition by: _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col0 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _wcol0 (type: int) @@ -136,6 +155,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 + partition by: _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col0 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 < 4) (type: boolean) @@ -290,6 +328,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: tinyint, _col1: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 < 5) (type: boolean)