diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 78ef0a0..94e665b 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -140,6 +140,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\ orc_vectorization_ppd.q,\ parallel.q,\ ptf.q,\ + ptf_matchpath.q,\ ptf_streaming.q,\ sample1.q,\ selectDistinctStar.q,\ diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index daf6cb8..149f911 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -33,7 +33,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -478,11 +477,11 @@ else if (ent.getValue() instanceof Map) { json.put(ent.getKey().toString(), ent.getValue().toString()); } } - else if (ent.getValue() instanceof Serializable) { + else if (ent.getValue() != null) { if (out != null) { out.println(); } - JSONObject jsonOut = outputPlan((Serializable) ent.getValue(), out, + JSONObject jsonOut = outputPlan(ent.getValue(), out, extended, jsonOutput, jsonOutput ? 0 : indent + 2); if (jsonOutput) { json.put(ent.getKey().toString(), jsonOut); @@ -518,11 +517,11 @@ private JSONArray outputList(List l, PrintStream out, boolean hasHeader, } nl = true; } - else if (o instanceof Serializable) { + else { if (first_el && (out != null) && hasHeader) { out.println(); } - JSONObject jsonOut = outputPlan((Serializable) o, out, extended, + JSONObject jsonOut = outputPlan(o, out, extended, jsonOutput, jsonOutput ? 0 : (hasHeader ? indent + 2 : indent)); if (jsonOutput) { outputArray.put(jsonOut); @@ -553,12 +552,12 @@ private boolean isPrintable(Object val) { return false; } - private JSONObject outputPlan(Serializable work, + private JSONObject outputPlan(Object work, PrintStream out, boolean extended, boolean jsonOutput, int indent) throws Exception { return outputPlan(work, out, extended, jsonOutput, indent, ""); } - private JSONObject outputPlan(Serializable work, PrintStream out, + private JSONObject outputPlan(Object work, PrintStream out, boolean extended, boolean jsonOutput, int indent, String appendToHeader) throws Exception { // Check if work has an explain annotation Annotation note = AnnotationUtils.getAnnotation(work.getClass(), Explain.class); @@ -678,7 +677,7 @@ private JSONObject outputPlan(Serializable work, PrintStream out, } // Try this as a map - try { + if (val instanceof Map) { // Go through the map and print out the stuff Map mp = (Map) val; @@ -692,22 +691,10 @@ private JSONObject outputPlan(Serializable work, PrintStream out, } continue; } - catch (ClassCastException ce) { - // Ignore - all this means is that this is not a map - } // Try this as a list - try { - List l; - - try { - l = (List) val; - } catch (ClassCastException e) { - Set s = (Set) val; - l = new LinkedList(); - l.addAll(s); - } - + if (val instanceof List || val instanceof Set) { + List l = val instanceof List ? (List)val : new ArrayList((Set)val); if (out != null && !skipHeader && l != null && !l.isEmpty()) { out.print(header); } @@ -720,18 +707,13 @@ private JSONObject outputPlan(Serializable work, PrintStream out, continue; } - catch (ClassCastException ce) { - // Ignore - } // Finally check if it is serializable try { - Serializable s = (Serializable) val; - if (!skipHeader && out != null) { out.println(header); } - JSONObject jsonOut = outputPlan(s, out, extended, jsonOutput, ind); + JSONObject jsonOut = outputPlan(val, out, extended, jsonOutput, ind); if (jsonOutput) { if (!skipHeader) { json.put(header, jsonOut); @@ -779,7 +761,7 @@ private boolean shouldPrint(Explain exp, Object val) { return true; } - private JSONObject outputPlan(Task task, + private JSONObject outputPlan(Task task, PrintStream out, JSONObject parentJSON, boolean extended, boolean jsonOutput, int indent) throws Exception { @@ -805,7 +787,7 @@ private JSONObject outputPlan(Task task, return null; } - private JSONObject outputDependencies(Task task, + private JSONObject outputDependencies(Task task, PrintStream out, JSONObject parentJson, boolean jsonOutput, boolean taskType, int indent) throws Exception { @@ -830,7 +812,7 @@ private JSONObject outputDependencies(Task task, else { StringBuffer s = new StringBuffer(); first = true; - for (Task parent : task.getParentTasks()) { + for (Task parent : task.getParentTasks()) { if (!first) { s.append(", "); } @@ -847,7 +829,7 @@ private JSONObject outputDependencies(Task task, } } - Task currBackupTask = task.getBackupTask(); + Task currBackupTask = task.getBackupTask(); if (currBackupTask != null) { if (out != null) { out.print(" has a backup stage: "); @@ -862,7 +844,7 @@ private JSONObject outputDependencies(Task task, && ((ConditionalTask) task).getListTasks() != null) { StringBuffer s = new StringBuffer(); first = true; - for (Task con : ((ConditionalTask) task).getListTasks()) { + for (Task con : ((ConditionalTask) task).getListTasks()) { if (!first) { s.append(", "); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java index f49b7cd..d3e9992 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; +import org.apache.hadoop.hive.ql.parse.RowResolver; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -68,6 +69,9 @@ import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFInputDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.ShapeDetails; import org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef; import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef; import org.apache.hadoop.hive.ql.udf.ptf.Noop; @@ -260,34 +264,34 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, PTFDesc conf = op.getConf(); //Since we cannot know what columns will be needed by a PTF chain, //we do not prune columns on PTFOperator for PTF chains. - if (!conf.forWindowing() && !Noop.class.isInstance(conf.getFuncDef().getTFunction())) { + PartitionedTableFunctionDef funcDef = conf.getFuncDef(); + if (!conf.forWindowing() && !Noop.class.isInstance(funcDef.getTFunction())) { return super.process(nd, stack, cppCtx, nodeOutputs); } - - List prunedCols = cppCtx.getPrunedColList(op.getChildOperators().get(0)); - - WindowTableFunctionDef def = null; - if (conf.forWindowing()) { - def = (WindowTableFunctionDef) conf.getFuncDef(); + + //we create a copy of prunedCols to create a list of pruned columns for PTFOperator + List prunedCols = + new ArrayList(cppCtx.getPrunedColList(op.getChildOperators().get(0))); + if (funcDef instanceof WindowTableFunctionDef) { + WindowTableFunctionDef def = (WindowTableFunctionDef) funcDef; prunedCols = Utilities.mergeUniqElems(getWindowFunctionColumns(def), prunedCols); - prunedCols = prunedColumnsList(prunedCols, def); } + + List newRS = prunedColumnsList(prunedCols, op.getSchema(), funcDef); + + op.getSchema().setSignature(new ArrayList(newRS)); - RowSchema oldRS = op.getSchema(); - ArrayList sig = buildPrunedRR(prunedCols, oldRS); - op.getSchema().setSignature(sig); - - prunedCols = def == null ? prunedCols : prunedInputList(prunedCols, def); - cppCtx.getPrunedColLists().put(op, prunedCols); + ShapeDetails outputShape = funcDef.getStartOfChain().getInput().getOutputShape(); + cppCtx.getPrunedColLists().put(op, outputShape.getColumnNames()); return null; } - private static ArrayList buildPrunedRR(List prunedCols, - RowSchema oldRS) throws SemanticException{ + private List buildPrunedRS(List prunedCols, RowSchema oldRS) + throws SemanticException { ArrayList sig = new ArrayList(); HashSet prunedColsSet = new HashSet(prunedCols); - for(ColumnInfo cInfo : oldRS.getSignature()) { - if ( prunedColsSet.contains(cInfo.getInternalName())) { + for (ColumnInfo cInfo : oldRS.getSignature()) { + if (prunedColsSet.contains(cInfo.getInternalName())) { sig.add(cInfo); } } @@ -305,48 +309,74 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, return columns; } + private RowResolver buildPrunedRR(List prunedCols, RowSchema oldRS) + throws SemanticException { + RowResolver resolver = new RowResolver(); + HashSet prunedColsSet = new HashSet(prunedCols); + for (ColumnInfo cInfo : oldRS.getSignature()) { + if (prunedColsSet.contains(cInfo.getInternalName())) { + resolver.put(cInfo.getTabAlias(), cInfo.getAlias(), cInfo); + } + } + return resolver; + } + /* * add any input columns referenced in WindowFn args or expressions. */ - private ArrayList prunedColumnsList(List prunedCols, - WindowTableFunctionDef tDef) { - //we create a copy of prunedCols to create a list of pruned columns for PTFOperator - ArrayList mergedColList = new ArrayList(prunedCols); - if ( tDef.getWindowFunctions() != null ) { - for(WindowFunctionDef wDef : tDef.getWindowFunctions() ) { - if ( wDef.getArgs() == null) { - continue; + private List prunedColumnsList(List prunedCols, RowSchema oldRS, + PartitionedTableFunctionDef pDef) throws SemanticException { + pDef.getOutputShape().setRr(null); + pDef.getOutputShape().setColumnNames(null); + if (pDef instanceof WindowTableFunctionDef) { + WindowTableFunctionDef tDef = (WindowTableFunctionDef) pDef; + if (tDef.getWindowFunctions() != null) { + for (WindowFunctionDef wDef : tDef.getWindowFunctions()) { + if (wDef.getArgs() == null) { + continue; + } + for (PTFExpressionDef arg : wDef.getArgs()) { + ExprNodeDesc exprNode = arg.getExprNode(); + Utilities.mergeUniqElems(prunedCols, exprNode.getCols()); + } + } + } + if (tDef.getPartition() != null) { + for (PTFExpressionDef col : tDef.getPartition().getExpressions()) { + ExprNodeDesc exprNode = col.getExprNode(); + Utilities.mergeUniqElems(prunedCols, exprNode.getCols()); } - for(PTFExpressionDef arg : wDef.getArgs()) { - ExprNodeDesc exprNode = arg.getExprNode(); - Utilities.mergeUniqElems(mergedColList, exprNode.getCols()); + } + if (tDef.getOrder() != null) { + for (PTFExpressionDef col : tDef.getOrder().getExpressions()) { + ExprNodeDesc exprNode = col.getExprNode(); + Utilities.mergeUniqElems(prunedCols, exprNode.getCols()); } } + } else { + pDef.getOutputShape().setRr(buildPrunedRR(prunedCols, oldRS)); } - if(tDef.getPartition() != null){ - for(PTFExpressionDef col : tDef.getPartition().getExpressions()){ - ExprNodeDesc exprNode = col.getExprNode(); - Utilities.mergeUniqElems(mergedColList, exprNode.getCols()); - } - } - if(tDef.getOrder() != null){ - for(PTFExpressionDef col : tDef.getOrder().getExpressions()){ - ExprNodeDesc exprNode = col.getExprNode(); - Utilities.mergeUniqElems(mergedColList, exprNode.getCols()); - } - } - return mergedColList; + + PTFInputDef input = pDef.getInput(); + if (input instanceof PartitionedTableFunctionDef) { + return prunedColumnsList(prunedCols, oldRS, (PartitionedTableFunctionDef)input); + } + + ArrayList inputColumns = prunedInputList(prunedCols, input); + input.getOutputShape().setRr(buildPrunedRR(inputColumns, oldRS)); + input.getOutputShape().setColumnNames(inputColumns); + + return buildPrunedRS(prunedCols, oldRS); } /* * from the prunedCols list filter out columns that refer to WindowFns or WindowExprs * the returned list is set as the prunedList needed by the PTFOp. */ - private ArrayList prunedInputList(List prunedCols, - WindowTableFunctionDef tDef) { + private ArrayList prunedInputList(List prunedCols, PTFInputDef tDef) { ArrayList prunedInputCols = new ArrayList(); - StructObjectInspector OI = tDef.getInput().getOutputShape().getOI(); + StructObjectInspector OI = tDef.getOutputShape().getOI(); for(StructField f : OI.getAllStructFieldRefs()) { String fName = f.getFieldName(); if ( prunedCols.contains(fName)) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java index 3ac3245..2f31eed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java @@ -22,14 +22,16 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.PTFUtils; import org.apache.hadoop.hive.ql.parse.LeadLagInfo; -import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order; -import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputType; +import org.apache.hadoop.hive.ql.plan.ptf.PTFInputDef; import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + @Explain(displayName = "PTF Operator") public class PTFDesc extends AbstractOperatorDesc { private static final long serialVersionUID = 1L; @@ -62,6 +64,19 @@ public PartitionedTableFunctionDef getStartOfChain() { return funcDef == null ? null : funcDef.getStartOfChain(); } + @Explain(displayName = "Function definitions") + public List getFuncDefExplain() { + if (funcDef == null) { + return null; + } + List inputs = new ArrayList(); + for (PTFInputDef current = funcDef; current != null; current = current.getInput()) { + inputs.add(current); + } + Collections.reverse(inputs); + return inputs; + } + public LeadLagInfo getLlInfo() { return llInfo; } @@ -70,10 +85,19 @@ public void setLlInfo(LeadLagInfo llInfo) { this.llInfo = llInfo; } + @Explain(displayName = "Lead/Lag information") + public String getLlInfoExplain() { + if (llInfo != null && llInfo.getLeadLagExprs() != null) { + return PlanUtils.getExprListString(llInfo.getLeadLagExprs()); + } + return null; + } + public boolean forWindowing() { - return funcDef != null && (funcDef instanceof WindowTableFunctionDef); + return funcDef instanceof WindowTableFunctionDef; } + @Explain(displayName = "Map-side function", displayOnlyOnTrue = true) public boolean isMapSide() { return isMapSide; } @@ -89,5 +113,4 @@ public Configuration getCfg() { public void setCfg(Configuration cfg) { this.cfg = cfg; } - } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index b62ffed..46fd6dc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -923,7 +923,7 @@ public static ReadEntity addInput(Set inputs, ReadEntity newInput) { return null; } - public static String getExprListString(Collection exprs) { + public static String getExprListString(Collection exprs) { StringBuffer sb = new StringBuffer(); boolean first = true; for (ExprNodeDesc expr: exprs) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java index 07590c0..f692fa2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java @@ -32,4 +32,10 @@ public void setDirection(Direction direction) { } public abstract int getAmt(); + + @Override + public String toString() { + return direction == null ? "" : + direction + "(" + (getAmt() == Integer.MAX_VALUE ? "MAX" : getAmt()) + ")"; + } } \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/OrderExpressionDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/OrderExpressionDef.java index e367d13..eb0f18e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/OrderExpressionDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/OrderExpressionDef.java @@ -19,7 +19,9 @@ package org.apache.hadoop.hive.ql.plan.ptf; import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order; +import org.apache.hadoop.hive.ql.plan.Explain; +@Explain(displayName = "order definition") public class OrderExpressionDef extends PTFExpressionDef { private Order order; @@ -29,6 +31,7 @@ public OrderExpressionDef(PTFExpressionDef e) { order = Order.ASC; } + @Explain(displayName = "order") public Order getOrder() { return order; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java index 5d200fb..a0370bf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.PTFUtils; +import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -58,6 +59,11 @@ public void setExprNode(ExprNodeDesc exprNode) { this.exprNode = exprNode; } + @Explain(displayName = "expr") + public String getExprNodeExplain() { + return exprNode == null ? null : exprNode.getExprString(); + } + public ExprNodeEvaluator getExprEvaluator() { return exprEvaluator; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java index 19ed2f2..95296c0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java @@ -19,6 +19,10 @@ package org.apache.hadoop.hive.ql.plan.ptf; +import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.util.StringUtils; + public abstract class PTFInputDef { private String expressionTreeString; private ShapeDetails outputShape; @@ -36,9 +40,17 @@ public ShapeDetails getOutputShape() { return outputShape; } + @Explain(displayName = "output shape") + public String getOutputShapeExplain() { + RowSchema schema = outputShape.getRr().getRowSchema(); + return StringUtils.join(", ", schema.getSignature()); + } + public void setOutputShape(ShapeDetails outputShape) { this.outputShape = outputShape; } + + @Explain(displayName = "input alias") public String getAlias() { return alias; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java index 11ef932..227b117 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java @@ -19,11 +19,14 @@ package org.apache.hadoop.hive.ql.plan.ptf; import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputType; +import org.apache.hadoop.hive.ql.plan.Explain; +@Explain(displayName = "Input definition") public class PTFQueryInputDef extends PTFInputDef { private String destination; private PTFQueryInputType type; + @Explain(displayName = "destination") public String getDestination() { return destination; } @@ -40,6 +43,11 @@ public void setType(PTFQueryInputType type) { this.type = type; } + @Explain(displayName = "type") + public String getTypeExplain() { + return type.name(); + } + @Override public PTFInputDef getInput() { return null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java index 327304c..967caaa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java @@ -21,8 +21,11 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec; +import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator; +@Explain(displayName = "Partition table definition") public class PartitionedTableFunctionDef extends PTFInputDef { private String name; private String resolverClassName; @@ -35,6 +38,7 @@ private TableFunctionEvaluator tFunction; boolean transformsRawInput; + @Explain(displayName = "name") public String getName() { return name; } @@ -47,6 +51,11 @@ public ShapeDetails getRawInputShape() { return rawInputShape; } + @Explain(displayName = "raw input shape") + public ShapeDetails getRawInputShapeExplain() { + return rawInputShape; + } + public void setRawInputShape(ShapeDetails rawInputShape) { this.rawInputShape = rawInputShape; } @@ -72,6 +81,21 @@ public PartitionDef getPartition() { return partition; } + @Explain(displayName = "partition by") + public String getPartitionExplain() { + if (partition == null || partition.getExpressions() == null) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (PTFExpressionDef expression : partition.getExpressions()) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(expression.getExprNode().getExprString()); + } + return builder.toString(); + } + public void setPartition(PartitionDef partition) { this.partition = partition; } @@ -84,9 +108,28 @@ public void setOrder(OrderDef order) { this.order = order; } + @Explain(displayName = "order by") + public String getOrderExplain() { + if (order == null || order.getExpressions() == null) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (OrderExpressionDef expression : order.getExpressions()) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(expression.getExprNode().getExprString()); + if (expression.getOrder() == PTFInvocationSpec.Order.DESC) { + builder.append("(DESC)"); + } + } + return builder.toString(); + } + public TableFunctionEvaluator getTFunction() { return tFunction; } + public void setTFunction(TableFunctionEvaluator tFunction) { this.tFunction = tFunction; } @@ -99,6 +142,21 @@ public void setArgs(List args) { this.args = args; } + @Explain(displayName = "arguments") + public String getArgsExplain() { + if (args == null) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (PTFExpressionDef expression : args) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(expression.getExprNode().getExprString()); + } + return builder.toString(); + } + public void addArg(PTFExpressionDef arg) { args = args == null ? new ArrayList() : args; args.add(arg); @@ -111,6 +169,7 @@ public PartitionedTableFunctionDef getStartOfChain() { return this; } + @Explain(displayName = "transforms raw input", displayOnlyOnTrue=true) public boolean isTransformsRawInput() { return transformsRawInput; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java index b96e9d6..de18575 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java @@ -19,15 +19,12 @@ package org.apache.hadoop.hive.ql.plan.ptf; -public class WindowExpressionDef extends PTFExpressionDef { - private String alias; - - public WindowExpressionDef() {} +import org.apache.hadoop.hive.ql.plan.Explain; - public WindowExpressionDef(PTFExpressionDef eDef) { - super(eDef); - } +public abstract class WindowExpressionDef extends PTFExpressionDef { + private String alias; + @Explain(displayName = "alias") public String getAlias() { return alias; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java index 949ed10..e08bdd5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java @@ -38,4 +38,9 @@ public BoundaryDef getEnd() { public void setEnd(BoundaryDef end) { this.end = end; } + + @Override + public String toString() { + return start + "~" + end; + } } \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java index e4ea358..ed6c671 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java @@ -21,8 +21,10 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +@Explain(displayName = "window function definition") public class WindowFunctionDef extends WindowExpressionDef { String name; boolean isStar; @@ -32,6 +34,7 @@ GenericUDAFEvaluator wFnEval; boolean pivotResult; + @Explain(displayName = "name") public String getName() { return name; } @@ -40,6 +43,7 @@ public void setName(String name) { this.name = name; } + @Explain(displayName = "isStar", displayOnlyOnTrue = true) public boolean isStar() { return isStar; } @@ -48,6 +52,7 @@ public void setStar(boolean isStar) { this.isStar = isStar; } + @Explain(displayName = "isDistinct", displayOnlyOnTrue = true) public boolean isDistinct() { return isDistinct; } @@ -69,6 +74,21 @@ public void addArg(PTFExpressionDef arg) { args.add(arg); } + @Explain(displayName = "arguments") + public String getArgsExplain() { + if (args == null) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (PTFExpressionDef expression : args) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(expression.getExprNode().getExprString()); + } + return builder.toString(); + } + public WindowFrameDef getWindowFrame() { return windowFrame; } @@ -77,6 +97,11 @@ public void setWindowFrame(WindowFrameDef windowFrame) { this.windowFrame = windowFrame; } + @Explain(displayName = "window frame") + public String getWindowFrameExplain() { + return windowFrame == null ? null : windowFrame.toString(); + } + public GenericUDAFEvaluator getWFnEval() { return wFnEval; } @@ -85,6 +110,12 @@ public void setWFnEval(GenericUDAFEvaluator wFnEval) { this.wFnEval = wFnEval; } + @Explain(displayName = "window function") + public String getWFnEvalExplain() { + return wFnEval == null ? null : wFnEval.getClass().getSimpleName(); + } + + @Explain(displayName = "isPivotResult", displayOnlyOnTrue = true) public boolean isPivotResult() { return pivotResult; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java index 083aaf2..97ba17e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java @@ -18,21 +18,26 @@ package org.apache.hadoop.hive.ql.plan.ptf; -import java.util.List; +import org.apache.hadoop.hive.ql.plan.Explain; +import java.util.List; +@Explain(displayName = "Windowing table definition") public class WindowTableFunctionDef extends PartitionedTableFunctionDef { List windowFunctions; int rankLimit = -1; int rankLimitFunction; + @Explain(displayName = "window functions") public List getWindowFunctions() { return windowFunctions; } + public void setWindowFunctions(List windowFunctions) { this.windowFunctions = windowFunctions; } + public int getRankLimit() { return rankLimit; } diff --git a/ql/src/test/queries/clientpositive/ptf_matchpath.q b/ql/src/test/queries/clientpositive/ptf_matchpath.q index 80dbe29..6487135 100644 --- a/ql/src/test/queries/clientpositive/ptf_matchpath.q +++ b/ql/src/test/queries/clientpositive/ptf_matchpath.q @@ -15,6 +15,17 @@ LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt' OVERWRITE INTO TABLE -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ); + select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -26,6 +37,17 @@ from matchpath(on ); -- 2. Matchpath on 1 partition +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142; + select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -37,6 +59,17 @@ from matchpath(on where fl_num = 1142; -- 3. empty partition. +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ); + + select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny diff --git a/ql/src/test/results/clientpositive/correlationoptimizer12.q.out b/ql/src/test/results/clientpositive/correlationoptimizer12.q.out index 3a7247f..159b0c0 100644 --- a/ql/src/test/results/clientpositive/correlationoptimizer12.q.out +++ b/ql/src/test/results/clientpositive/correlationoptimizer12.q.out @@ -43,6 +43,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -114,6 +132,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) diff --git a/ql/src/test/results/clientpositive/ctas_colname.q.out b/ql/src/test/results/clientpositive/ctas_colname.q.out index 4eb827c..a83af21 100644 --- a/ql/src/test/results/clientpositive/ctas_colname.q.out +++ b/ql/src/test/results/clientpositive/ctas_colname.q.out @@ -189,6 +189,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _wcol0 (type: int) @@ -336,6 +355,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col0, 1 + name: lead + window function: GenericUDAFLeadEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _wcol0 (type: string) diff --git a/ql/src/test/results/clientpositive/groupby_grouping_window.q.out b/ql/src/test/results/clientpositive/groupby_grouping_window.q.out index c0e6e5f..b2434c5 100644 --- a/ql/src/test/results/clientpositive/groupby_grouping_window.q.out +++ b/ql/src/test/results/clientpositive/groupby_grouping_window.q.out @@ -93,6 +93,25 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3 Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: int, _col2: int, _col3: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col3 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 1 Data size: 6 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: int), _col2 (type: int), _col3 (type: int), _wcol0 (type: int) diff --git a/ql/src/test/results/clientpositive/groupby_resolution.q.out b/ql/src/test/results/clientpositive/groupby_resolution.q.out index e5d22a3..c371f5c 100644 --- a/ql/src/test/results/clientpositive/groupby_resolution.q.out +++ b/ql/src/test/results/clientpositive/groupby_resolution.q.out @@ -677,6 +677,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: bigint + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: 0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _wcol0 (type: int) diff --git a/ql/src/test/results/clientpositive/ptf.q.out b/ql/src/test/results/clientpositive/ptf.q.out index 1c54855..747041c 100644 --- a/ql/src/test/results/clientpositive/ptf.q.out +++ b/ql/src/test/results/clientpositive/ptf.q.out @@ -48,6 +48,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -72,6 +84,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -220,6 +264,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -244,6 +300,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -340,10 +415,22 @@ STAGE PLANS: value expressions: p_size (type: int) Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) - outputColumnNames: _col2, _col1, _col5 + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -449,6 +536,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -473,6 +572,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -586,6 +717,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -610,6 +753,39 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -726,6 +902,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -781,6 +969,39 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -889,6 +1110,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1028,6 +1261,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1160,7 +1405,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -1172,6 +1431,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1195,6 +1467,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -1289,7 +1580,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -1302,6 +1607,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1326,6 +1644,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1435,6 +1785,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1459,6 +1821,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1571,9 +1965,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -1597,6 +2024,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1621,6 +2068,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1740,6 +2219,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1764,6 +2255,30 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -1888,6 +2403,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1952,6 +2479,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -2063,6 +2635,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2243,6 +2827,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 + raw input shape: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2267,6 +2863,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -2429,6 +3043,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2459,6 +3085,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -2502,6 +3160,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _wcol0 (type: bigint) @@ -2530,6 +3206,45 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + raw input shape: + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -2736,9 +3451,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -2762,6 +3517,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2786,6 +3561,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2929,6 +3736,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2953,6 +3779,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2977,6 +3815,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3001,6 +3851,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3139,6 +4021,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3163,6 +4064,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3187,6 +4107,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3326,6 +4278,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3350,9 +4321,35 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -3376,6 +4373,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3400,6 +4410,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3540,9 +4582,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -3566,6 +4641,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3590,6 +4685,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -3724,9 +4851,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -3750,6 +4910,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3774,6 +4947,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/ptf_matchpath.q.out b/ql/src/test/results/clientpositive/ptf_matchpath.q.out index e0cea0d..aaa66cf 100644 --- a/ql/src/test/results/clientpositive/ptf_matchpath.q.out +++ b/ql/src/test/results/clientpositive/ptf_matchpath.q.out @@ -37,6 +37,7 @@ POSTHOOK: Output: default@flights_tiny PREHOOK: query: -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -47,11 +48,10 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -62,6 +62,83 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: ++++ + Map-reduce partition columns: fl_num (type: string) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), arr_delay (type: float), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), VALUE._col2 (type: float), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: _col6 + raw input shape: + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### Baltimore 1142 2010 10 20 6 20 @@ -81,6 +158,7 @@ Chicago 897 2010 10 21 3 21 Chicago 897 2010 10 22 2 22 Washington 7291 2010 10 27 2 27 PREHOOK: query: -- 2. Matchpath on 1 partition +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -91,9 +169,8 @@ from matchpath(on ) where fl_num = 1142 PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- 2. Matchpath on 1 partition +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -104,6 +181,86 @@ from matchpath(on ) where fl_num = 1142 POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), arr_delay (type: float), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col2 (type: float), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + raw input shape: + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = 1142) (type: boolean) + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### Baltimore 1142 2010 10 20 6 20 @@ -112,6 +269,7 @@ Baltimore 1142 2010 10 22 4 22 Baltimore 1142 2010 10 25 3 25 Baltimore 1142 2010 10 26 2 26 PREHOOK: query: -- 3. empty partition. +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny @@ -121,9 +279,8 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- 3. empty partition. +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny @@ -133,5 +290,87 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 24 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = -1142) (type: boolean) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), '-1142' (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col5 (type: float) + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col2 (type: float), '-1142' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string + type: SUBQUERY + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + raw input shape: + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '-1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/ptf_streaming.q.out b/ql/src/test/results/clientpositive/ptf_streaming.q.out index 2284f0e..33ee089 100644 --- a/ql/src/test/results/clientpositive/ptf_streaming.q.out +++ b/ql/src/test/results/clientpositive/ptf_streaming.q.out @@ -48,6 +48,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -72,6 +84,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -220,6 +264,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -244,6 +300,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -347,6 +422,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -475,7 +562,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -487,6 +588,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -510,6 +624,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -604,7 +737,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -617,6 +764,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -641,6 +801,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -753,9 +945,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -779,6 +1004,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -803,6 +1048,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -917,9 +1194,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -943,6 +1253,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -967,6 +1297,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1081,9 +1443,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -1107,6 +1502,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1131,6 +1546,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1249,6 +1696,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1313,6 +1772,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1447,9 +1951,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -1473,6 +2017,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1497,6 +2061,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1640,6 +2236,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1664,6 +2279,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1688,6 +2315,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1712,6 +2351,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1848,9 +2519,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -1874,6 +2578,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1898,6 +2615,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/quotedid_basic.q.out b/ql/src/test/results/clientpositive/quotedid_basic.q.out index 6c7a017..2dbe996 100644 --- a/ql/src/test/results/clientpositive/quotedid_basic.q.out +++ b/ql/src/test/results/clientpositive/quotedid_basic.q.out @@ -198,6 +198,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _wcol0 (type: int) @@ -283,6 +302,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _wcol0 (type: int) diff --git a/ql/src/test/results/clientpositive/spark/ptf.q.out b/ql/src/test/results/clientpositive/spark/ptf.q.out index bce5153..82cd156 100644 --- a/ql/src/test/results/clientpositive/spark/ptf.q.out +++ b/ql/src/test/results/clientpositive/spark/ptf.q.out @@ -54,6 +54,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -68,6 +80,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -214,6 +258,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -228,6 +284,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -330,10 +405,22 @@ STAGE PLANS: Reducer 2 Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) - outputColumnNames: _col2, _col1, _col5 + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -445,6 +532,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -459,6 +558,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -578,6 +709,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -592,6 +735,39 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -714,6 +890,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -749,6 +937,39 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -876,6 +1097,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1034,6 +1267,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1129,7 +1374,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -1142,6 +1401,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1155,6 +1427,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -1254,7 +1545,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -1268,6 +1573,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1282,6 +1600,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1397,6 +1747,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1411,6 +1773,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1529,9 +1923,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -1545,6 +1972,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1559,6 +2006,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1684,6 +2163,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1698,6 +2189,30 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -1841,6 +2356,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1874,6 +2401,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1991,6 +2563,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2157,6 +2741,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 + raw input shape: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) @@ -2171,6 +2767,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -2340,6 +2954,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -2360,6 +3006,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _wcol0 (type: bigint) @@ -2378,6 +3042,45 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + raw input shape: + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -2398,6 +3101,16 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2412,6 +3125,16 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col5 (type: int) @@ -2631,9 +3354,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -2647,6 +3410,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -2661,6 +3444,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2810,6 +3625,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -2824,6 +3658,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -2838,6 +3684,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2852,6 +3710,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2996,6 +3886,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -3010,6 +3919,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3024,6 +3952,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3169,6 +4129,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -3183,9 +4162,35 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -3199,6 +4204,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -3213,6 +4231,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3359,9 +4409,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) sort order: ++ @@ -3375,6 +4458,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -3389,6 +4492,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -3529,9 +4664,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -3545,6 +4713,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3559,6 +4740,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out b/ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out index e0cea0d..22c8af3 100644 --- a/ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out +++ b/ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out @@ -37,6 +37,7 @@ POSTHOOK: Output: default@flights_tiny PREHOOK: query: -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -47,11 +48,10 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -62,6 +62,89 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: ++++ + Map-reduce partition columns: fl_num (type: string) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), arr_delay (type: float), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), VALUE._col2 (type: float), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: _col6 + raw input shape: + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### Baltimore 1142 2010 10 20 6 20 @@ -81,6 +164,7 @@ Chicago 897 2010 10 21 3 21 Chicago 897 2010 10 22 2 22 Washington 7291 2010 10 27 2 27 PREHOOK: query: -- 2. Matchpath on 1 partition +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -91,9 +175,8 @@ from matchpath(on ) where fl_num = 1142 PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- 2. Matchpath on 1 partition +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -104,6 +187,92 @@ from matchpath(on ) where fl_num = 1142 POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), arr_delay (type: float), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col2 (type: float), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + raw input shape: + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = 1142) (type: boolean) + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### Baltimore 1142 2010 10 20 6 20 @@ -112,6 +281,7 @@ Baltimore 1142 2010 10 22 4 22 Baltimore 1142 2010 10 25 3 25 Baltimore 1142 2010 10 26 2 26 PREHOOK: query: -- 3. empty partition. +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny @@ -121,9 +291,8 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- 3. empty partition. +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny @@ -133,5 +302,93 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 24 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = -1142) (type: boolean) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), '-1142' (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col5 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col2 (type: float), '-1142' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string + type: SUBQUERY + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + raw input shape: + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '-1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/spark/ptf_streaming.q.out b/ql/src/test/results/clientpositive/spark/ptf_streaming.q.out index 4e820c1..9cf4a3e 100644 --- a/ql/src/test/results/clientpositive/spark/ptf_streaming.q.out +++ b/ql/src/test/results/clientpositive/spark/ptf_streaming.q.out @@ -54,6 +54,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -68,6 +80,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -214,6 +258,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -228,6 +284,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -350,6 +425,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -462,7 +549,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -475,6 +576,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -488,6 +602,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -587,7 +720,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -601,6 +748,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -615,6 +775,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -733,9 +925,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -749,6 +974,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -763,6 +1008,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -883,9 +1160,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -899,6 +1209,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -913,6 +1243,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1033,9 +1395,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -1049,6 +1444,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1063,6 +1478,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1200,6 +1647,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1233,6 +1692,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1373,9 +1877,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -1389,6 +1933,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -1403,6 +1967,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1552,6 +2148,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -1566,6 +2181,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -1580,6 +2207,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1594,6 +2233,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1736,9 +2407,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -1752,6 +2456,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1766,6 +2483,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/spark/subquery_in.q.out b/ql/src/test/results/clientpositive/spark/subquery_in.q.out index 53c28c2..c48482d 100644 --- a/ql/src/test/results/clientpositive/spark/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/spark/subquery_in.q.out @@ -314,6 +314,25 @@ STAGE PLANS: outputColumnNames: _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -463,6 +482,25 @@ STAGE PLANS: outputColumnNames: _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col2 is not null) (type: boolean) diff --git a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out index f3f7970..c846710 100644 --- a/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out @@ -304,6 +304,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -321,6 +333,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -681,6 +725,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -698,6 +754,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -903,10 +978,22 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) - outputColumnNames: _col2, _col1, _col5 + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1167,6 +1254,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1184,6 +1283,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1456,6 +1587,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1473,6 +1616,39 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -1755,6 +1931,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1797,6 +1985,39 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -2091,6 +2312,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2435,6 +2668,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2588,7 +2833,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -2653,6 +2912,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2669,6 +2941,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -2863,7 +3154,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -2929,6 +3234,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2946,6 +3264,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3209,6 +3559,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3226,6 +3588,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3499,9 +3893,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -3518,6 +3945,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3535,6 +3982,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3828,6 +4307,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3845,6 +4336,30 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -4282,6 +4797,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -4322,6 +4849,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -4542,6 +5114,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -4841,6 +5425,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 + raw input shape: Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) @@ -4858,6 +5454,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -5288,6 +5902,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -5327,6 +5973,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _wcol0 (type: bigint) @@ -5348,6 +6012,45 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + raw input shape: + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -5387,6 +6090,16 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -5404,6 +6117,16 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col5 (type: int) @@ -5811,9 +6534,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -5830,6 +6593,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -5847,6 +6630,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6171,6 +6986,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -6188,6 +7022,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -6205,6 +7051,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6222,6 +7080,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6529,6 +7419,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -6546,6 +7455,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6563,6 +7491,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6882,6 +7842,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -6899,9 +7878,35 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -6918,6 +7923,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -6935,6 +7953,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -7285,9 +8335,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) sort order: ++ @@ -7304,6 +8387,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -7321,6 +8424,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -7637,9 +8772,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -7656,6 +8824,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -7673,6 +8854,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/subquery_in.q.out b/ql/src/test/results/clientpositive/subquery_in.q.out index 3a5e77b..d4dec94 100644 --- a/ql/src/test/results/clientpositive/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/subquery_in.q.out @@ -269,6 +269,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -434,6 +453,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col0 is not null) (type: boolean) diff --git a/ql/src/test/results/clientpositive/subquery_in_having.q.out b/ql/src/test/results/clientpositive/subquery_in_having.q.out index 1046c9e..365c6ab 100644 --- a/ql/src/test/results/clientpositive/subquery_in_having.q.out +++ b/ql/src/test/results/clientpositive/subquery_in_having.q.out @@ -1348,6 +1348,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col0 + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(MAX)~ Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _wcol0 is not null (type: boolean) diff --git a/ql/src/test/results/clientpositive/subquery_notin.q.out b/ql/src/test/results/clientpositive/subquery_notin.q.out index 5811394..ef14440 100644 --- a/ql/src/test/results/clientpositive/subquery_notin.q.out +++ b/ql/src/test/results/clientpositive/subquery_notin.q.out @@ -337,6 +337,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and (_col0 is null or _col1 is null)) (type: boolean) @@ -477,6 +495,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -589,6 +625,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -721,6 +775,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -881,6 +953,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -1057,6 +1147,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) diff --git a/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out b/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out index 8c6b202..de2b4af 100644 --- a/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out +++ b/ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out @@ -222,6 +222,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col0 is not null) (type: boolean) @@ -375,6 +393,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col0 is not null) (type: boolean) @@ -803,6 +839,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and (_col0 is null or _col1 is null)) (type: boolean) @@ -943,6 +997,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) diff --git a/ql/src/test/results/clientpositive/tez/ptf.q.out b/ql/src/test/results/clientpositive/tez/ptf.q.out index 3d4f110..3e428d9 100644 --- a/ql/src/test/results/clientpositive/tez/ptf.q.out +++ b/ql/src/test/results/clientpositive/tez/ptf.q.out @@ -54,6 +54,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -68,6 +80,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -214,6 +258,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -228,6 +284,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -330,10 +405,22 @@ STAGE PLANS: Reducer 2 Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) - outputColumnNames: _col2, _col1, _col5 + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -445,6 +532,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -459,6 +558,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -578,6 +709,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -592,6 +735,39 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -714,6 +890,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -749,6 +937,39 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -876,6 +1097,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1034,6 +1267,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1129,7 +1374,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -1142,6 +1401,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1155,6 +1427,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -1254,7 +1545,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -1268,6 +1573,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1282,6 +1600,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1397,6 +1747,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1411,6 +1773,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1529,9 +1923,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -1545,6 +1972,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1559,6 +2006,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1684,6 +2163,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1698,6 +2189,30 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -1841,6 +2356,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1874,6 +2401,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1991,6 +2563,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2157,6 +2741,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 + raw input shape: Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) @@ -2171,6 +2767,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -2340,6 +2954,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2360,6 +2986,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -2380,6 +3038,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _wcol0 (type: bigint) @@ -2398,6 +3074,45 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + raw input shape: + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -2626,9 +3341,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -2642,6 +3397,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -2656,6 +3431,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2805,6 +3612,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -2819,6 +3645,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -2833,6 +3671,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2847,6 +3697,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2991,6 +3873,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -3005,6 +3906,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3019,6 +3939,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3164,6 +4116,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -3178,9 +4149,35 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -3194,6 +4191,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -3208,6 +4218,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3354,9 +4396,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) sort order: ++ @@ -3370,6 +4445,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -3384,6 +4479,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -3524,9 +4651,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -3540,6 +4700,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3554,6 +4727,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/tez/ptf_matchpath.q.out b/ql/src/test/results/clientpositive/tez/ptf_matchpath.q.out new file mode 100644 index 0000000..fc2b034 --- /dev/null +++ b/ql/src/test/results/clientpositive/tez/ptf_matchpath.q.out @@ -0,0 +1,394 @@ +PREHOOK: query: DROP TABLE flights_tiny +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE flights_tiny +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table flights_tiny ( +ORIGIN_CITY_NAME string, +DEST_CITY_NAME string, +YEAR int, +MONTH int, +DAY_OF_MONTH int, +ARR_DELAY float, +FL_NUM string +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: create table flights_tiny ( +ORIGIN_CITY_NAME string, +DEST_CITY_NAME string, +YEAR int, +MONTH int, +DAY_OF_MONTH int, +ARR_DELAY float, +FL_NUM string +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. basic Matchpath test +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. basic Matchpath test +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: ++++ + Map-reduce partition columns: fl_num (type: string) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), arr_delay (type: float), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), VALUE._col2 (type: float), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: _col6 + raw input shape: + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +Baltimore 1142 2010 10 20 6 20 +Baltimore 1142 2010 10 21 5 21 +Baltimore 1142 2010 10 22 4 22 +Baltimore 1142 2010 10 25 3 25 +Baltimore 1142 2010 10 26 2 26 +Baltimore 1599 2010 10 21 2 21 +Baltimore 1599 2010 10 25 3 25 +Baltimore 1599 2010 10 26 2 26 +Chicago 1531 2010 10 21 2 21 +Chicago 1531 2010 10 25 3 25 +Chicago 1531 2010 10 26 2 26 +Chicago 361 2010 10 20 2 20 +Chicago 897 2010 10 20 4 20 +Chicago 897 2010 10 21 3 21 +Chicago 897 2010 10 22 2 22 +Washington 7291 2010 10 27 2 27 +PREHOOK: query: -- 2. Matchpath on 1 partition +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +PREHOOK: type: QUERY +POSTHOOK: query: -- 2. Matchpath on 1 partition +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), arr_delay (type: float), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col2 (type: float), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: bigint), VALUE._col4 (type: string), VALUE._col5 (type: struct) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + raw input shape: + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = 1142) (type: boolean) + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +Baltimore 1142 2010 10 20 6 20 +Baltimore 1142 2010 10 21 5 21 +Baltimore 1142 2010 10 22 4 22 +Baltimore 1142 2010 10 25 3 25 +Baltimore 1142 2010 10 26 2 26 +PREHOOK: query: -- 3. empty partition. +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +POSTHOOK: query: -- 3. empty partition. +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 24 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = -1142) (type: boolean) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), '-1142' (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col5 (type: float) + Reducer 2 + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), KEY.reducesinkkey2 (type: int), KEY.reducesinkkey3 (type: int), KEY.reducesinkkey4 (type: int), VALUE._col2 (type: float), '-1142' (type: string) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string + type: SUBQUERY + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + raw input shape: + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '-1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out b/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out index 1423f64..caaefe3 100644 --- a/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out +++ b/ql/src/test/results/clientpositive/tez/ptf_streaming.q.out @@ -54,6 +54,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -68,6 +80,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -214,6 +258,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -228,6 +284,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -350,6 +425,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -462,7 +549,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -475,6 +576,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -488,6 +602,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -587,7 +720,21 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -601,6 +748,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -615,6 +775,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -733,9 +925,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -749,6 +974,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -763,6 +1008,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -883,9 +1160,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -899,6 +1209,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -913,6 +1243,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1033,9 +1395,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -1049,6 +1444,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1063,6 +1478,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1200,6 +1647,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1233,6 +1692,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1373,9 +1877,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -1389,6 +1933,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -1403,6 +1967,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1552,6 +2148,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -1566,6 +2181,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -1580,6 +2207,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1594,6 +2233,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1736,9 +2407,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -1752,6 +2456,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1766,6 +2483,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/tez/subquery_in.q.out b/ql/src/test/results/clientpositive/tez/subquery_in.q.out index 69eb568..f84277c 100644 --- a/ql/src/test/results/clientpositive/tez/subquery_in.q.out +++ b/ql/src/test/results/clientpositive/tez/subquery_in.q.out @@ -326,6 +326,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -483,6 +502,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col0 is not null) (type: boolean) diff --git a/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out index c2147d8..da92594 100644 --- a/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out @@ -304,6 +304,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -321,6 +333,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -682,6 +726,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -699,6 +755,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -904,10 +979,22 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) - outputColumnNames: _col2, _col1, _col5 + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1168,6 +1255,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1185,6 +1284,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1457,6 +1588,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1474,6 +1617,39 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -1756,6 +1932,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1798,6 +1986,39 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -2092,6 +2313,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2438,6 +2671,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2591,7 +2836,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -2656,6 +2915,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2672,6 +2944,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -2866,7 +3157,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -2932,6 +3237,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2949,6 +3267,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3212,6 +3562,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3229,6 +3591,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3502,9 +3896,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -3521,6 +3948,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3538,6 +3985,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3831,6 +4310,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3848,6 +4339,30 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -4285,6 +4800,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -4326,6 +4853,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -4546,6 +5118,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -4845,6 +5429,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 + raw input shape: Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) @@ -4862,6 +5458,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -5292,6 +5906,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -5317,6 +5943,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -5356,6 +6014,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _wcol0 (type: bigint) @@ -5377,6 +6053,45 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + raw input shape: + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -5809,9 +6524,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -5828,6 +6583,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -5845,6 +6620,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6169,6 +6976,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -6186,6 +7012,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -6203,6 +7041,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6220,6 +7070,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6527,6 +7409,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -6544,6 +7445,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6561,6 +7481,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6880,6 +7832,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -6897,9 +7868,35 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -6916,6 +7913,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -6933,6 +7943,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -7283,9 +8325,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) sort order: ++ @@ -7302,6 +8377,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -7319,6 +8414,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -7635,9 +8762,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -7654,6 +8814,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -7671,6 +8844,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/union_remove_6_subq.q.out b/ql/src/test/results/clientpositive/union_remove_6_subq.q.out index be15c1f..98c0df1 100644 --- a/ql/src/test/results/clientpositive/union_remove_6_subq.q.out +++ b/ql/src/test/results/clientpositive/union_remove_6_subq.q.out @@ -546,6 +546,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: bigint + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: avg + window function: GenericUDAFAverageEvaluatorDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _wcol0 (type: double) diff --git a/ql/src/test/results/clientpositive/vectorized_ptf.q.out b/ql/src/test/results/clientpositive/vectorized_ptf.q.out index cb99b54..ffef745 100644 --- a/ql/src/test/results/clientpositive/vectorized_ptf.q.out +++ b/ql/src/test/results/clientpositive/vectorized_ptf.q.out @@ -298,6 +298,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -362,6 +374,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -711,6 +755,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -775,6 +831,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -974,10 +1049,22 @@ STAGE PLANS: Needs Tagging: false Reduce Operator Tree: Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string), VALUE._col3 (type: int) - outputColumnNames: _col2, _col1, _col5 + expressions: KEY.reducesinkkey1 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: int) + outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1232,6 +1319,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1296,6 +1395,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1562,6 +1693,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1626,6 +1769,39 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -1902,6 +2078,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2037,6 +2225,39 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -2258,6 +2479,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2600,6 +2833,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2891,7 +3136,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -2955,6 +3214,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3018,6 +3290,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -3207,7 +3498,21 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -3272,6 +3577,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3336,6 +3654,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3593,6 +3943,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3657,6 +4019,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3924,9 +4318,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -3990,6 +4417,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -4054,6 +4501,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -4341,6 +4820,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -4405,6 +4896,30 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -4769,6 +5284,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -4963,6 +5490,51 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -5177,6 +5749,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -5562,6 +6146,18 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 + raw input shape: Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -5626,6 +6222,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -6049,6 +6663,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -6130,6 +6756,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -6233,6 +6891,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _col2 (type: string), _col5 (type: int), _wcol0 (type: bigint) @@ -6301,6 +6977,45 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + raw input shape: + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -6698,9 +7413,49 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -6764,6 +7519,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -6828,6 +7603,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -7146,6 +7953,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7210,6 +8036,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7274,6 +8112,18 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7338,6 +8188,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -7639,6 +8521,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7703,6 +8604,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7767,6 +8687,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -8080,6 +9032,25 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -8144,9 +9115,35 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -8210,6 +9207,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -8274,6 +9284,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -8618,9 +9660,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -8684,6 +9759,26 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -8748,6 +9843,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -9058,9 +10185,42 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -9124,6 +10284,19 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + raw input shape: + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -9188,6 +10361,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git a/ql/src/test/results/clientpositive/windowing_streaming.q.out b/ql/src/test/results/clientpositive/windowing_streaming.q.out index d45646a..b17d96d 100644 --- a/ql/src/test/results/clientpositive/windowing_streaming.q.out +++ b/ql/src/test/results/clientpositive/windowing_streaming.q.out @@ -80,6 +80,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 + partition by: _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col0 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _wcol0 (type: int) @@ -136,6 +155,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 + partition by: _col1 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col0 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 < 4) (type: boolean) @@ -290,6 +328,25 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: tinyint, _col1: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + raw input shape: + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 < 5) (type: boolean)