diff --git itests/src/test/resources/testconfiguration.properties itests/src/test/resources/testconfiguration.properties index 12fcd6a..839778f 100644 --- itests/src/test/resources/testconfiguration.properties +++ itests/src/test/resources/testconfiguration.properties @@ -139,6 +139,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\ orc_vectorization_ppd.q,\ parallel.q,\ ptf.q,\ + ptf_matchpath.q,\ ptf_streaming.q,\ sample1.q,\ selectDistinctStar.q,\ diff --git itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 8e00ee3..2cfe586 100644 --- itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -1017,7 +1017,7 @@ private int executeClientInternal(String commands) { } command = ""; } - if (SessionState.get() != null) { + if (rc == 0 && SessionState.get() != null) { SessionState.get().setLastCommand(null); // reset } return rc; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java index daf6cb8..2e3ae8a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java @@ -553,12 +553,12 @@ private boolean isPrintable(Object val) { return false; } - private JSONObject outputPlan(Serializable work, + private JSONObject outputPlan(Object work, PrintStream out, boolean extended, boolean jsonOutput, int indent) throws Exception { return outputPlan(work, out, extended, jsonOutput, indent, ""); } - private JSONObject outputPlan(Serializable work, PrintStream out, + private JSONObject outputPlan(Object work, PrintStream out, boolean extended, boolean jsonOutput, int indent, String appendToHeader) throws Exception { // Check if work has an explain annotation Annotation note = AnnotationUtils.getAnnotation(work.getClass(), Explain.class); @@ -619,6 +619,9 @@ private JSONObject outputPlan(Serializable work, PrintStream out, Arrays.sort(methods, new MethodComparator()); for (Method m : methods) { + if (m.getDeclaringClass() == Object.class) { + continue; + } int prop_indents = jsonOutput ? 0 : indent + 2; note = AnnotationUtils.getAnnotation(m, Explain.class); @@ -667,6 +670,30 @@ private JSONObject outputPlan(Serializable work, PrintStream out, } continue; } + if (val instanceof Explain.Explainables) { + Explain.Explainables iterable = (Explain.Explainables) val; + out.println(header); + JSONArray outputArray = new JSONArray(); + Explain.Explainable explain; + for (explain = iterable.next(); explain != null; explain = iterable.next()) { + JSONObject jsonOut = outputPlan(explain, out, extended, jsonOutput, indent + 4); + if (jsonOutput) { + outputArray.put(jsonOut); + } + } + if (jsonOutput) { + json.put(header, outputArray); + } + continue; + } + if (val instanceof Explain.Explainable) { + out.println(header); + JSONObject jsonOut = outputPlan(val, out, extended, jsonOutput, indent + 4); + if (jsonOutput) { + json.put(header, jsonOut); + } + continue; + } int ind = 0; if (!jsonOutput) { @@ -972,4 +999,6 @@ public String getName() { colList.add(tmpFieldSchema); return colList; } + + } diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java index 57ce849..8605cc5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java @@ -60,6 +60,9 @@ import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFInputDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.ShapeDetails; import org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef; import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef; import org.apache.hadoop.hive.ql.udf.ptf.Noop; @@ -267,36 +270,36 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, PTFDesc conf = op.getConf(); //Since we cannot know what columns will be needed by a PTF chain, //we do not prune columns on PTFOperator for PTF chains. - if (!conf.forWindowing() && !Noop.class.isInstance(conf.getFuncDef().getTFunction())) { + PartitionedTableFunctionDef funcDef = conf.getFuncDef(); + if (!conf.forWindowing() && !Noop.class.isInstance(funcDef.getTFunction())) { return super.process(nd, stack, cppCtx, nodeOutputs); } - List prunedCols = cppCtx.getPrunedColList(op.getChildOperators().get(0)); - RowResolver oldRR = cppCtx.getOpToParseCtxMap().get(op).getRowResolver(); + + //we create a copy of prunedCols to create a list of pruned columns for PTFOperator + List prunedCols = + new ArrayList(cppCtx.getPrunedColList(op.getChildOperators().get(0))); - WindowTableFunctionDef def = null; - if (conf.forWindowing()) { - def = (WindowTableFunctionDef) conf.getFuncDef(); - prunedCols = prunedColumnsList(prunedCols, def); - } - ArrayList sig = new ArrayList(); - RowResolver newRR = buildPrunedRR(prunedCols, oldRR, sig); + RowResolver oldRR = cppCtx.getOpToParseCtxMap().get(op).getRowResolver(); + RowResolver newRR = prunedColumnsList(prunedCols, oldRR, funcDef); + + List signature = newRR.getRowSchema().getSignature(); + op.getSchema().setSignature(new ArrayList(signature)); cppCtx.getOpToParseCtxMap().get(op).setRowResolver(newRR); - op.getSchema().setSignature(sig); - prunedCols = def == null ? prunedCols : prunedInputList(prunedCols, def); - cppCtx.getPrunedColLists().put(op, prunedCols); + ShapeDetails outputShape = funcDef.getStartOfChain().getInput().getOutputShape(); + cppCtx.getPrunedColLists().put(op, outputShape.getColumnNames()); return null; } - private RowResolver buildPrunedRR(List prunedCols, - RowResolver oldRR, ArrayList sig) throws SemanticException{ + // should keep column order of oldRR + private RowResolver buildPrunedRR(List prunedCols, RowResolver oldRR) + throws SemanticException { RowResolver newRR = new RowResolver(); HashSet prunedColsSet = new HashSet(prunedCols); - for(ColumnInfo cInfo : oldRR.getRowSchema().getSignature()) { - if ( prunedColsSet.contains(cInfo.getInternalName())) { + for (ColumnInfo cInfo : oldRR.getRowSchema().getSignature()) { + if (prunedColsSet.contains(cInfo.getInternalName())) { String[] nm = oldRR.reverseLookup(cInfo.getInternalName()); newRR.put(nm[0], nm[1], cInfo); - sig.add(cInfo); } } return newRR; @@ -305,45 +308,59 @@ private RowResolver buildPrunedRR(List prunedCols, /* * add any input columns referenced in WindowFn args or expressions. */ - private ArrayList prunedColumnsList(List prunedCols, - WindowTableFunctionDef tDef) { - //we create a copy of prunedCols to create a list of pruned columns for PTFOperator - ArrayList mergedColList = new ArrayList(prunedCols); - if ( tDef.getWindowFunctions() != null ) { - for(WindowFunctionDef wDef : tDef.getWindowFunctions() ) { - if ( wDef.getArgs() == null) { - continue; + private RowResolver prunedColumnsList(List prunedCols, RowResolver oldRR, + PartitionedTableFunctionDef pDef) throws SemanticException { + if (pDef instanceof WindowTableFunctionDef) { + WindowTableFunctionDef tDef = (WindowTableFunctionDef) pDef; + if (tDef.getWindowFunctions() != null) { + for (WindowFunctionDef wDef : tDef.getWindowFunctions()) { + if (wDef.getArgs() == null) { + continue; + } + for (PTFExpressionDef arg : wDef.getArgs()) { + ExprNodeDesc exprNode = arg.getExprNode(); + Utilities.mergeUniqElems(prunedCols, exprNode.getCols()); + } + } + } + if (tDef.getPartition() != null) { + for (PTFExpressionDef col : tDef.getPartition().getExpressions()) { + ExprNodeDesc exprNode = col.getExprNode(); + Utilities.mergeUniqElems(prunedCols, exprNode.getCols()); } - for(PTFExpressionDef arg : wDef.getArgs()) { - ExprNodeDesc exprNode = arg.getExprNode(); - Utilities.mergeUniqElems(mergedColList, exprNode.getCols()); + } + if (tDef.getOrder() != null) { + for (PTFExpressionDef col : tDef.getOrder().getExpressions()) { + ExprNodeDesc exprNode = col.getExprNode(); + Utilities.mergeUniqElems(prunedCols, exprNode.getCols()); } } + pDef.getOutputShape().setRr(null); + pDef.getOutputShape().setColumnNames(null); + } else { + pDef.getOutputShape().setRr(buildPrunedRR(prunedCols, oldRR)); + pDef.getOutputShape().setColumnNames(prunedCols); + } + + PTFInputDef input = pDef.getInput(); + if (input instanceof PartitionedTableFunctionDef) { + return prunedColumnsList(prunedCols, oldRR, (PartitionedTableFunctionDef)input); } - if(tDef.getPartition() != null){ - for(PTFExpressionDef col : tDef.getPartition().getExpressions()){ - ExprNodeDesc exprNode = col.getExprNode(); - Utilities.mergeUniqElems(mergedColList, exprNode.getCols()); - } - } - if(tDef.getOrder() != null){ - for(PTFExpressionDef col : tDef.getOrder().getExpressions()){ - ExprNodeDesc exprNode = col.getExprNode(); - Utilities.mergeUniqElems(mergedColList, exprNode.getCols()); - } - } - return mergedColList; + ArrayList inputColumns = prunedInputList(prunedCols, input); + input.getOutputShape().setRr(buildPrunedRR(inputColumns, oldRR)); + input.getOutputShape().setColumnNames(inputColumns); + + return buildPrunedRR(prunedCols, oldRR); } /* * from the prunedCols list filter out columns that refer to WindowFns or WindowExprs * the returned list is set as the prunedList needed by the PTFOp. */ - private ArrayList prunedInputList(List prunedCols, - WindowTableFunctionDef tDef) { + private ArrayList prunedInputList(List prunedCols, PTFInputDef tDef) { ArrayList prunedInputCols = new ArrayList(); - StructObjectInspector OI = tDef.getInput().getOutputShape().getOI(); + StructObjectInspector OI = tDef.getOutputShape().getOI(); for(StructField f : OI.getAllStructFieldRefs()) { String fName = f.getFieldName(); if ( prunedCols.contains(fName)) { diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java index a3408a0..368b572 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java @@ -20,6 +20,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.util.Iterator; /** * Explain. @@ -34,4 +35,17 @@ boolean displayOnlyOnTrue() default false; boolean skipHeader() default false; + + public static interface Explainable { + } + + public static class Explainables { + private final Iterator iterator; + public Explainables(Iterable iterable) { + this.iterator = iterable.iterator(); + } + public Explain.Explainable next() { + return iterator.hasNext() ? iterator.next() : null; + } + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java index 3ac3245..0b9fd2e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java @@ -22,14 +22,16 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.PTFUtils; import org.apache.hadoop.hive.ql.parse.LeadLagInfo; -import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order; -import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputType; +import org.apache.hadoop.hive.ql.plan.ptf.PTFInputDef; import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + @Explain(displayName = "PTF Operator") public class PTFDesc extends AbstractOperatorDesc { private static final long serialVersionUID = 1L; @@ -62,6 +64,19 @@ public PartitionedTableFunctionDef getStartOfChain() { return funcDef == null ? null : funcDef.getStartOfChain(); } + @Explain(displayName = "Function definitions") + public Explain.Explainables getFuncDefExplain() { + if (funcDef == null) { + return null; + } + List inputs = new ArrayList(); + for (PTFInputDef current = funcDef; current != null; current = current.getInput()) { + inputs.add(current); + } + Collections.reverse(inputs); + return new Explain.Explainables(inputs); + } + public LeadLagInfo getLlInfo() { return llInfo; } @@ -70,10 +85,19 @@ public void setLlInfo(LeadLagInfo llInfo) { this.llInfo = llInfo; } + @Explain(displayName = "Lead/Lag information") + public String getLlInfoExplain() { + if (llInfo != null && llInfo.getLeadLagExprs() != null) { + return PlanUtils.getExprListString(llInfo.getLeadLagExprs()); + } + return null; + } + public boolean forWindowing() { - return funcDef != null && (funcDef instanceof WindowTableFunctionDef); + return funcDef instanceof WindowTableFunctionDef; } + @Explain(displayName = "Map-side function", displayOnlyOnTrue = true) public boolean isMapSide() { return isMapSide; } @@ -89,5 +113,4 @@ public Configuration getCfg() { public void setCfg(Configuration cfg) { this.cfg = cfg; } - } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index b62ffed..46fd6dc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@ -923,7 +923,7 @@ public static ReadEntity addInput(Set inputs, ReadEntity newInput) { return null; } - public static String getExprListString(Collection exprs) { + public static String getExprListString(Collection exprs) { StringBuffer sb = new StringBuffer(); boolean first = true; for (ExprNodeDesc expr: exprs) { diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java index 07590c0..f692fa2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java @@ -32,4 +32,10 @@ public void setDirection(Direction direction) { } public abstract int getAmt(); + + @Override + public String toString() { + return direction == null ? "" : + direction + "(" + (getAmt() == Integer.MAX_VALUE ? "MAX" : getAmt()) + ")"; + } } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/OrderExpressionDef.java ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/OrderExpressionDef.java index e367d13..eb0f18e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/OrderExpressionDef.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/OrderExpressionDef.java @@ -19,7 +19,9 @@ package org.apache.hadoop.hive.ql.plan.ptf; import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order; +import org.apache.hadoop.hive.ql.plan.Explain; +@Explain(displayName = "order definition") public class OrderExpressionDef extends PTFExpressionDef { private Order order; @@ -29,6 +31,7 @@ public OrderExpressionDef(PTFExpressionDef e) { order = Order.ASC; } + @Explain(displayName = "order") public Order getOrder() { return order; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java index 5d200fb..aeae8bb 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java @@ -20,10 +20,11 @@ import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.PTFUtils; +import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -public class PTFExpressionDef { +public class PTFExpressionDef implements Explain.Explainable { String expressionTreeString; ExprNodeDesc exprNode; transient ExprNodeEvaluator exprEvaluator; @@ -58,6 +59,11 @@ public void setExprNode(ExprNodeDesc exprNode) { this.exprNode = exprNode; } + @Explain(displayName = "expr") + public String getExprNodeExplain() { + return exprNode == null ? null : exprNode.getExprString(); + } + public ExprNodeEvaluator getExprEvaluator() { return exprEvaluator; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java index 19ed2f2..3eb7c44 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java @@ -19,7 +19,11 @@ package org.apache.hadoop.hive.ql.plan.ptf; -public abstract class PTFInputDef { +import org.apache.hadoop.hive.ql.exec.RowSchema; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.util.StringUtils; + +public abstract class PTFInputDef implements Explain.Explainable { private String expressionTreeString; private ShapeDetails outputShape; private String alias; @@ -36,9 +40,17 @@ public ShapeDetails getOutputShape() { return outputShape; } + @Explain(displayName = "output shape") + public String getOutputShapeExplain() { + RowSchema schema = outputShape.getRr().getRowSchema(); + return StringUtils.join(", ", schema.getSignature()); + } + public void setOutputShape(ShapeDetails outputShape) { this.outputShape = outputShape; } + + @Explain(displayName = "input alias") public String getAlias() { return alias; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java index 11ef932..227b117 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java @@ -19,11 +19,14 @@ package org.apache.hadoop.hive.ql.plan.ptf; import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputType; +import org.apache.hadoop.hive.ql.plan.Explain; +@Explain(displayName = "Input definition") public class PTFQueryInputDef extends PTFInputDef { private String destination; private PTFQueryInputType type; + @Explain(displayName = "destination") public String getDestination() { return destination; } @@ -40,6 +43,11 @@ public void setType(PTFQueryInputType type) { this.type = type; } + @Explain(displayName = "type") + public String getTypeExplain() { + return type.name(); + } + @Override public PTFInputDef getInput() { return null; diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java index 327304c..967caaa 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java @@ -21,8 +21,11 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec; +import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator; +@Explain(displayName = "Partition table definition") public class PartitionedTableFunctionDef extends PTFInputDef { private String name; private String resolverClassName; @@ -35,6 +38,7 @@ private TableFunctionEvaluator tFunction; boolean transformsRawInput; + @Explain(displayName = "name") public String getName() { return name; } @@ -47,6 +51,11 @@ public ShapeDetails getRawInputShape() { return rawInputShape; } + @Explain(displayName = "raw input shape") + public ShapeDetails getRawInputShapeExplain() { + return rawInputShape; + } + public void setRawInputShape(ShapeDetails rawInputShape) { this.rawInputShape = rawInputShape; } @@ -72,6 +81,21 @@ public PartitionDef getPartition() { return partition; } + @Explain(displayName = "partition by") + public String getPartitionExplain() { + if (partition == null || partition.getExpressions() == null) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (PTFExpressionDef expression : partition.getExpressions()) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(expression.getExprNode().getExprString()); + } + return builder.toString(); + } + public void setPartition(PartitionDef partition) { this.partition = partition; } @@ -84,9 +108,28 @@ public void setOrder(OrderDef order) { this.order = order; } + @Explain(displayName = "order by") + public String getOrderExplain() { + if (order == null || order.getExpressions() == null) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (OrderExpressionDef expression : order.getExpressions()) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(expression.getExprNode().getExprString()); + if (expression.getOrder() == PTFInvocationSpec.Order.DESC) { + builder.append("(DESC)"); + } + } + return builder.toString(); + } + public TableFunctionEvaluator getTFunction() { return tFunction; } + public void setTFunction(TableFunctionEvaluator tFunction) { this.tFunction = tFunction; } @@ -99,6 +142,21 @@ public void setArgs(List args) { this.args = args; } + @Explain(displayName = "arguments") + public String getArgsExplain() { + if (args == null) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (PTFExpressionDef expression : args) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(expression.getExprNode().getExprString()); + } + return builder.toString(); + } + public void addArg(PTFExpressionDef arg) { args = args == null ? new ArrayList() : args; args.add(arg); @@ -111,6 +169,7 @@ public PartitionedTableFunctionDef getStartOfChain() { return this; } + @Explain(displayName = "transforms raw input", displayOnlyOnTrue=true) public boolean isTransformsRawInput() { return transformsRawInput; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java index b96e9d6..de18575 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java @@ -19,15 +19,12 @@ package org.apache.hadoop.hive.ql.plan.ptf; -public class WindowExpressionDef extends PTFExpressionDef { - private String alias; - - public WindowExpressionDef() {} +import org.apache.hadoop.hive.ql.plan.Explain; - public WindowExpressionDef(PTFExpressionDef eDef) { - super(eDef); - } +public abstract class WindowExpressionDef extends PTFExpressionDef { + private String alias; + @Explain(displayName = "alias") public String getAlias() { return alias; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java index 949ed10..e08bdd5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java @@ -38,4 +38,9 @@ public BoundaryDef getEnd() { public void setEnd(BoundaryDef end) { this.end = end; } + + @Override + public String toString() { + return start + "~" + end; + } } \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java index e4ea358..ed6c671 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java @@ -21,8 +21,10 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +@Explain(displayName = "window function definition") public class WindowFunctionDef extends WindowExpressionDef { String name; boolean isStar; @@ -32,6 +34,7 @@ GenericUDAFEvaluator wFnEval; boolean pivotResult; + @Explain(displayName = "name") public String getName() { return name; } @@ -40,6 +43,7 @@ public void setName(String name) { this.name = name; } + @Explain(displayName = "isStar", displayOnlyOnTrue = true) public boolean isStar() { return isStar; } @@ -48,6 +52,7 @@ public void setStar(boolean isStar) { this.isStar = isStar; } + @Explain(displayName = "isDistinct", displayOnlyOnTrue = true) public boolean isDistinct() { return isDistinct; } @@ -69,6 +74,21 @@ public void addArg(PTFExpressionDef arg) { args.add(arg); } + @Explain(displayName = "arguments") + public String getArgsExplain() { + if (args == null) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (PTFExpressionDef expression : args) { + if (builder.length() > 0) { + builder.append(", "); + } + builder.append(expression.getExprNode().getExprString()); + } + return builder.toString(); + } + public WindowFrameDef getWindowFrame() { return windowFrame; } @@ -77,6 +97,11 @@ public void setWindowFrame(WindowFrameDef windowFrame) { this.windowFrame = windowFrame; } + @Explain(displayName = "window frame") + public String getWindowFrameExplain() { + return windowFrame == null ? null : windowFrame.toString(); + } + public GenericUDAFEvaluator getWFnEval() { return wFnEval; } @@ -85,6 +110,12 @@ public void setWFnEval(GenericUDAFEvaluator wFnEval) { this.wFnEval = wFnEval; } + @Explain(displayName = "window function") + public String getWFnEvalExplain() { + return wFnEval == null ? null : wFnEval.getClass().getSimpleName(); + } + + @Explain(displayName = "isPivotResult", displayOnlyOnTrue = true) public boolean isPivotResult() { return pivotResult; } diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java index 083aaf2..1d3b03c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java @@ -18,9 +18,12 @@ package org.apache.hadoop.hive.ql.plan.ptf; -import java.util.List; +import org.apache.hadoop.hive.ql.plan.Explain; +import java.util.ArrayList; +import java.util.List; +@Explain(displayName = "Windowing table definition") public class WindowTableFunctionDef extends PartitionedTableFunctionDef { List windowFunctions; @@ -33,6 +36,14 @@ public void setWindowFunctions(List windowFunctions) { this.windowFunctions = windowFunctions; } + @Explain(displayName = "window functions") + public Explain.Explainables getWindowFunctionsExplain() { + if (windowFunctions == null) { + return null; + } + return new Explain.Explainables(new ArrayList(windowFunctions)); + } + public int getRankLimit() { return rankLimit; } diff --git ql/src/test/queries/clientpositive/ptf_matchpath.q ql/src/test/queries/clientpositive/ptf_matchpath.q index 80dbe29..6487135 100644 --- ql/src/test/queries/clientpositive/ptf_matchpath.q +++ ql/src/test/queries/clientpositive/ptf_matchpath.q @@ -15,6 +15,17 @@ LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt' OVERWRITE INTO TABLE -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ); + select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -26,6 +37,17 @@ from matchpath(on ); -- 2. Matchpath on 1 partition +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142; + select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -37,6 +59,17 @@ from matchpath(on where fl_num = 1142; -- 3. empty partition. +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ); + + select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny diff --git ql/src/test/results/clientpositive/correlationoptimizer12.q.out ql/src/test/results/clientpositive/correlationoptimizer12.q.out index c32e41e..fe4b345 100644 --- ql/src/test/results/clientpositive/correlationoptimizer12.q.out +++ ql/src/test/results/clientpositive/correlationoptimizer12.q.out @@ -43,6 +43,23 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -118,6 +135,23 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) diff --git ql/src/test/results/clientpositive/ctas_colname.q.out ql/src/test/results/clientpositive/ctas_colname.q.out index 95c7acb..5ea7f86 100644 --- ql/src/test/results/clientpositive/ctas_colname.q.out +++ ql/src/test/results/clientpositive/ctas_colname.q.out @@ -189,6 +189,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _wcol0 (type: int) @@ -336,6 +354,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col0, 1 + name: lead + window function: GenericUDAFLeadEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _wcol0 (type: string) diff --git ql/src/test/results/clientpositive/groupby_resolution.q.out ql/src/test/results/clientpositive/groupby_resolution.q.out index c611f7d..9b866b6 100644 --- ql/src/test/results/clientpositive/groupby_resolution.q.out +++ ql/src/test/results/clientpositive/groupby_resolution.q.out @@ -709,6 +709,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: bigint + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: 0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 83 Data size: 881 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint), _wcol0 (type: int) diff --git ql/src/test/results/clientpositive/ptf.q.out ql/src/test/results/clientpositive/ptf.q.out index f678035..ad7f473 100644 --- ql/src/test/results/clientpositive/ptf.q.out +++ ql/src/test/results/clientpositive/ptf.q.out @@ -46,6 +46,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -70,6 +81,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -216,6 +258,17 @@ STAGE PLANS: Extract Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -240,6 +293,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -338,6 +409,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -441,6 +523,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -465,6 +558,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -576,6 +700,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -600,6 +735,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -714,6 +881,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -769,6 +947,38 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -875,6 +1085,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1016,6 +1237,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1148,7 +1380,20 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -1159,6 +1404,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1182,6 +1439,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -1276,7 +1551,20 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -1287,6 +1575,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1311,6 +1611,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1418,6 +1749,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1442,6 +1784,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1552,9 +1925,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -1576,6 +1979,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1600,6 +2021,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1717,6 +2169,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1741,6 +2204,29 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -1863,6 +2349,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1927,6 +2424,50 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -2036,6 +2577,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2218,6 +2770,17 @@ STAGE PLANS: Extract Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2242,6 +2805,23 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -2402,6 +2982,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2432,6 +3023,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -2475,6 +3097,23 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2499,6 +3138,44 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -2703,9 +3380,45 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -2727,6 +3440,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2751,6 +3482,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2892,6 +3654,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2914,6 +3693,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2936,6 +3726,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -2960,6 +3761,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3096,6 +3928,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3118,6 +3967,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3142,6 +4008,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3279,6 +4176,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3301,9 +4215,33 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -3325,6 +4263,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3349,6 +4299,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3487,9 +4468,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -3511,6 +4522,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3535,6 +4564,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -3667,9 +4727,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -3691,6 +4781,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3715,6 +4817,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git ql/src/test/results/clientpositive/ptf_matchpath.q.out ql/src/test/results/clientpositive/ptf_matchpath.q.out index e0cea0d..37677dc 100644 --- ql/src/test/results/clientpositive/ptf_matchpath.q.out +++ ql/src/test/results/clientpositive/ptf_matchpath.q.out @@ -37,6 +37,7 @@ POSTHOOK: Output: default@flights_tiny PREHOOK: query: -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -47,11 +48,10 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -62,6 +62,80 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: ++++ + Map-reduce partition columns: fl_num (type: string) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float), fl_num (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reduce Operator Tree: + Extract + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: _col6 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### Baltimore 1142 2010 10 20 6 20 @@ -81,6 +155,7 @@ Chicago 897 2010 10 21 3 21 Chicago 897 2010 10 22 2 22 Washington 7291 2010 10 27 2 27 PREHOOK: query: -- 2. Matchpath on 1 partition +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -91,9 +166,8 @@ from matchpath(on ) where fl_num = 1142 PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- 2. Matchpath on 1 partition +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -104,6 +178,83 @@ from matchpath(on ) where fl_num = 1142 POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float), fl_num (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reduce Operator Tree: + Extract + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = 1142) (type: boolean) + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### Baltimore 1142 2010 10 20 6 20 @@ -112,6 +263,7 @@ Baltimore 1142 2010 10 22 4 22 Baltimore 1142 2010 10 25 3 25 Baltimore 1142 2010 10 26 2 26 PREHOOK: query: -- 3. empty partition. +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny @@ -121,9 +273,8 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- 3. empty partition. +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny @@ -133,5 +284,84 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 24 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = -1142) (type: boolean) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), '-1142' (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: float), '-1142' (type: string) + Reduce Operator Tree: + Extract + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string + type: SUBQUERY + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '-1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/ptf_streaming.q.out ql/src/test/results/clientpositive/ptf_streaming.q.out index 9cf645d..dcfb5b8 100644 --- ql/src/test/results/clientpositive/ptf_streaming.q.out +++ ql/src/test/results/clientpositive/ptf_streaming.q.out @@ -46,6 +46,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -70,6 +81,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -216,6 +258,17 @@ STAGE PLANS: Extract Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -240,6 +293,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -341,6 +412,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -473,7 +555,20 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -484,6 +579,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -507,6 +614,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -601,7 +726,20 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -612,6 +750,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -636,6 +786,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -746,9 +927,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -770,6 +981,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -794,6 +1023,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -906,9 +1166,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -930,6 +1220,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -954,6 +1262,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1066,9 +1405,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -1090,6 +1459,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1114,6 +1501,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1230,6 +1648,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1294,6 +1723,50 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1426,9 +1899,45 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -1450,6 +1959,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1474,6 +2001,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1615,6 +2173,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1637,6 +2212,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1659,6 +2245,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1683,6 +2280,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1817,9 +2445,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false table: @@ -1841,6 +2499,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1865,6 +2535,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git ql/src/test/results/clientpositive/quotedid_basic.q.out ql/src/test/results/clientpositive/quotedid_basic.q.out index b8cd4e9..3fe8a6c 100644 --- ql/src/test/results/clientpositive/quotedid_basic.q.out +++ ql/src/test/results/clientpositive/quotedid_basic.q.out @@ -202,6 +202,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _wcol0 (type: int) @@ -287,6 +305,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _wcol0 (type: int) diff --git ql/src/test/results/clientpositive/spark/ptf.q.out ql/src/test/results/clientpositive/spark/ptf.q.out index 8ca5496..f766e67 100644 --- ql/src/test/results/clientpositive/spark/ptf.q.out +++ ql/src/test/results/clientpositive/spark/ptf.q.out @@ -52,6 +52,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -66,6 +77,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -210,6 +252,17 @@ STAGE PLANS: Extract Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -224,6 +277,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -328,6 +399,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -437,6 +519,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -451,6 +544,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -568,6 +692,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -582,6 +717,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -702,6 +869,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -737,6 +915,38 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -862,6 +1072,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1022,6 +1243,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1117,7 +1349,20 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -1129,6 +1374,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1142,6 +1399,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -1241,7 +1516,20 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -1253,6 +1541,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1267,6 +1567,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1380,6 +1711,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1394,6 +1736,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1510,9 +1883,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -1524,6 +1927,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1538,6 +1959,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1661,6 +2113,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1675,6 +2138,29 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -1816,6 +2302,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1849,6 +2346,50 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1964,6 +2505,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2132,6 +2684,17 @@ STAGE PLANS: Extract Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) @@ -2146,6 +2709,23 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -2315,6 +2895,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -2335,6 +2946,23 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) @@ -2349,6 +2977,44 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -2367,6 +3033,15 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2379,6 +3054,15 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col5 (type: int) @@ -2596,9 +3280,45 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -2610,6 +3330,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -2624,6 +3362,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2771,6 +3540,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -2783,6 +3569,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -2795,6 +3592,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2809,6 +3617,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2951,6 +3790,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -2963,6 +3819,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2977,6 +3850,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3120,6 +4024,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -3132,9 +4053,33 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -3146,6 +4091,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -3160,6 +4117,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3304,9 +4292,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) sort order: ++ @@ -3318,6 +4336,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -3332,6 +4368,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -3470,9 +4537,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -3484,6 +4581,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3498,6 +4607,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out index e0cea0d..7098396 100644 --- ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out +++ ql/src/test/results/clientpositive/spark/ptf_matchpath.q.out @@ -37,6 +37,7 @@ POSTHOOK: Output: default@flights_tiny PREHOOK: query: -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -47,11 +48,10 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- SORT_QUERY_RESULTS -- 1. basic Matchpath test +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -62,6 +62,86 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: ++++ + Map-reduce partition columns: fl_num (type: string) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float), fl_num (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reducer 2 + Reduce Operator Tree: + Extract + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: _col6 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### Baltimore 1142 2010 10 20 6 20 @@ -81,6 +161,7 @@ Chicago 897 2010 10 21 3 21 Chicago 897 2010 10 22 2 22 Washington 7291 2010 10 27 2 27 PREHOOK: query: -- 2. Matchpath on 1 partition +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -91,9 +172,8 @@ from matchpath(on ) where fl_num = 1142 PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- 2. Matchpath on 1 partition +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on flights_tiny @@ -104,6 +184,89 @@ from matchpath(on ) where fl_num = 1142 POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float), fl_num (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reducer 2 + Reduce Operator Tree: + Extract + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = 1142) (type: boolean) + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### Baltimore 1142 2010 10 20 6 20 @@ -112,6 +275,7 @@ Baltimore 1142 2010 10 22 4 22 Baltimore 1142 2010 10 25 3 25 Baltimore 1142 2010 10 26 2 26 PREHOOK: query: -- 3. empty partition. +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny @@ -121,9 +285,8 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) PREHOOK: type: QUERY -PREHOOK: Input: default@flights_tiny -#### A masked pattern was here #### POSTHOOK: query: -- 3. empty partition. +explain select origin_city_name, fl_num, year, month, day_of_month, sz, tpath from matchpath(on (select * from flights_tiny where fl_num = -1142) flights_tiny @@ -133,5 +296,90 @@ from matchpath(on arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') ) POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Spark + Edges: + Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 2) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 24 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = -1142) (type: boolean) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), '-1142' (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: float), '-1142' (type: string) + Reducer 2 + Reduce Operator Tree: + Extract + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string + type: SUBQUERY + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '-1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY POSTHOOK: Input: default@flights_tiny #### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/ptf_streaming.q.out ql/src/test/results/clientpositive/spark/ptf_streaming.q.out index f5ee72d..55d884d 100644 --- ql/src/test/results/clientpositive/spark/ptf_streaming.q.out +++ ql/src/test/results/clientpositive/spark/ptf_streaming.q.out @@ -52,6 +52,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -66,6 +77,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -210,6 +252,17 @@ STAGE PLANS: Extract Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -224,6 +277,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -344,6 +415,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -460,7 +542,20 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -472,6 +567,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -485,6 +592,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -584,7 +709,20 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -596,6 +734,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -610,6 +760,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -726,9 +907,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -740,6 +951,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -754,6 +983,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -872,9 +1132,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -886,6 +1176,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -900,6 +1208,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1018,9 +1357,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -1032,6 +1401,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1046,6 +1433,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1181,6 +1599,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1214,6 +1643,50 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1352,9 +1825,45 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -1366,6 +1875,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -1380,6 +1907,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1527,6 +2085,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -1539,6 +2114,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -1551,6 +2137,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1565,6 +2162,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1705,9 +2333,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -1719,6 +2377,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1733,6 +2403,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git ql/src/test/results/clientpositive/spark/subquery_in.q.out ql/src/test/results/clientpositive/spark/subquery_in.q.out index 51b92a3..af3c7a92 100644 --- ql/src/test/results/clientpositive/spark/subquery_in.q.out +++ ql/src/test/results/clientpositive/spark/subquery_in.q.out @@ -322,6 +322,24 @@ STAGE PLANS: outputColumnNames: _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -471,6 +489,24 @@ STAGE PLANS: outputColumnNames: _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col2 is not null) (type: boolean) diff --git ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out index 020fdff..ceb5269 100644 --- ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out +++ ql/src/test/results/clientpositive/spark/vectorized_ptf.q.out @@ -302,6 +302,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -319,6 +330,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -677,6 +719,17 @@ STAGE PLANS: Extract Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -694,6 +747,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -901,6 +972,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1159,6 +1241,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1176,6 +1269,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1446,6 +1570,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1463,6 +1598,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -1743,6 +1910,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1785,6 +1963,38 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -2077,6 +2287,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2423,6 +2644,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2576,7 +2808,20 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -2640,6 +2885,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2656,6 +2913,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -2850,7 +3125,20 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -2914,6 +3202,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2931,6 +3231,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3192,6 +3523,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3209,6 +3551,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3480,9 +3853,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -3497,6 +3900,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3514,6 +3935,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3805,6 +4257,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3822,6 +4285,29 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -4257,6 +4743,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -4297,6 +4794,50 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -4515,6 +5056,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -4816,6 +5368,17 @@ STAGE PLANS: Extract Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) @@ -4833,6 +5396,23 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -5263,6 +5843,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -5302,6 +5913,23 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) @@ -5319,6 +5947,44 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -5356,6 +6022,15 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -5371,6 +6046,15 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col5 (type: int) @@ -5776,9 +6460,45 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -5793,6 +6513,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -5810,6 +6548,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6132,6 +6901,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -6147,6 +6933,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -6162,6 +6959,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6179,6 +6987,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6484,6 +7323,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -6499,6 +7355,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6516,6 +7389,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6833,6 +7737,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -6848,9 +7769,33 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -6865,6 +7810,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -6882,6 +7839,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -7230,9 +8218,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) sort order: ++ @@ -7247,6 +8265,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -7264,6 +8300,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -7578,9 +8645,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -7595,6 +8692,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -7612,6 +8721,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git ql/src/test/results/clientpositive/subquery_in.q.out ql/src/test/results/clientpositive/subquery_in.q.out index a2235af..c0a71d3 100644 --- ql/src/test/results/clientpositive/subquery_in.q.out +++ ql/src/test/results/clientpositive/subquery_in.q.out @@ -277,6 +277,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -446,6 +464,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col0 is not null) (type: boolean) diff --git ql/src/test/results/clientpositive/subquery_in_having.q.out ql/src/test/results/clientpositive/subquery_in_having.q.out index 03cc2af..d56e515 100644 --- ql/src/test/results/clientpositive/subquery_in_having.q.out +++ ql/src/test/results/clientpositive/subquery_in_having.q.out @@ -1386,6 +1386,23 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col0 + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(MAX)~ Statistics: Num rows: 15 Data size: 3173 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _wcol0 is not null (type: boolean) diff --git ql/src/test/results/clientpositive/subquery_notin.q.out ql/src/test/results/clientpositive/subquery_notin.q.out index 599a61e..64cab61 100644 --- ql/src/test/results/clientpositive/subquery_notin.q.out +++ ql/src/test/results/clientpositive/subquery_notin.q.out @@ -337,6 +337,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and (_col0 is null or _col1 is null)) (type: boolean) @@ -477,6 +495,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -589,6 +625,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -725,6 +779,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -885,6 +957,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -1061,6 +1151,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) diff --git ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out index 06d5708..1d9b4a2 100644 --- ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out +++ ql/src/test/results/clientpositive/subquery_unqualcolumnrefs.q.out @@ -230,6 +230,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col0 is not null) (type: boolean) @@ -383,6 +401,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col0 is not null) (type: boolean) @@ -819,6 +855,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and (_col0 is null or _col1 is null)) (type: boolean) @@ -959,6 +1013,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 + partition by: _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) diff --git ql/src/test/results/clientpositive/tez/ptf.q.out ql/src/test/results/clientpositive/tez/ptf.q.out index 6f9dd91..7463d30 100644 --- ql/src/test/results/clientpositive/tez/ptf.q.out +++ ql/src/test/results/clientpositive/tez/ptf.q.out @@ -52,6 +52,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -66,6 +77,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -210,6 +252,17 @@ STAGE PLANS: Extract Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -224,6 +277,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -328,6 +399,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -437,6 +519,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -451,6 +544,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -568,6 +692,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -582,6 +717,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -702,6 +869,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -737,6 +915,38 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -862,6 +1072,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1022,6 +1243,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1117,7 +1349,20 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -1129,6 +1374,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1142,6 +1399,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -1241,7 +1516,20 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -1253,6 +1541,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1267,6 +1567,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1380,6 +1711,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1394,6 +1736,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1510,9 +1883,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -1524,6 +1927,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1538,6 +1959,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1661,6 +2113,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1675,6 +2138,29 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -1816,6 +2302,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1849,6 +2346,50 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1964,6 +2505,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2132,6 +2684,17 @@ STAGE PLANS: Extract Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) @@ -2146,6 +2709,23 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -2313,6 +2893,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2333,6 +2924,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -2353,6 +2975,23 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) @@ -2367,6 +3006,44 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -2593,9 +3270,45 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -2607,6 +3320,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -2621,6 +3352,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2768,6 +3530,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -2780,6 +3559,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -2792,6 +3582,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2806,6 +3607,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -2948,6 +3780,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -2960,6 +3809,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2974,6 +3840,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3117,6 +4014,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -3129,9 +4043,33 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -3143,6 +4081,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -3157,6 +4107,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -3301,9 +4282,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) sort order: ++ @@ -3315,6 +4326,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -3329,6 +4358,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -3467,9 +4527,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -3481,6 +4571,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3495,6 +4597,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git ql/src/test/results/clientpositive/tez/ptf_matchpath.q.out ql/src/test/results/clientpositive/tez/ptf_matchpath.q.out new file mode 100644 index 0000000..5a81515 --- /dev/null +++ ql/src/test/results/clientpositive/tez/ptf_matchpath.q.out @@ -0,0 +1,385 @@ +PREHOOK: query: DROP TABLE flights_tiny +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE flights_tiny +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table flights_tiny ( +ORIGIN_CITY_NAME string, +DEST_CITY_NAME string, +YEAR int, +MONTH int, +DAY_OF_MONTH int, +ARR_DELAY float, +FL_NUM string +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: create table flights_tiny ( +ORIGIN_CITY_NAME string, +DEST_CITY_NAME string, +YEAR int, +MONTH int, +DAY_OF_MONTH int, +ARR_DELAY float, +FL_NUM string +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@flights_tiny +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/flights_tiny.txt' OVERWRITE INTO TABLE flights_tiny +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@flights_tiny +PREHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. basic Matchpath test +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +POSTHOOK: query: -- SORT_QUERY_RESULTS + +-- 1. basic Matchpath test +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: ++++ + Map-reduce partition columns: fl_num (type: string) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float), fl_num (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reducer 2 + Reduce Operator Tree: + Extract + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: _col6 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + distribute by fl_num + sort by year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +Baltimore 1142 2010 10 20 6 20 +Baltimore 1142 2010 10 21 5 21 +Baltimore 1142 2010 10 22 4 22 +Baltimore 1142 2010 10 25 3 25 +Baltimore 1142 2010 10 26 2 26 +Baltimore 1599 2010 10 21 2 21 +Baltimore 1599 2010 10 25 3 25 +Baltimore 1599 2010 10 26 2 26 +Chicago 1531 2010 10 21 2 21 +Chicago 1531 2010 10 25 3 25 +Chicago 1531 2010 10 26 2 26 +Chicago 361 2010 10 20 2 20 +Chicago 897 2010 10 20 4 20 +Chicago 897 2010 10 21 3 21 +Chicago 897 2010 10 22 2 22 +Washington 7291 2010 10 27 2 27 +PREHOOK: query: -- 2. Matchpath on 1 partition +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +PREHOOK: type: QUERY +POSTHOOK: query: -- 2. Matchpath on 1 partition +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), fl_num (type: string), year (type: int), month (type: int), day_of_month (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + value expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float), fl_num (type: string), BLOCK__OFFSET__INSIDE__FILE (type: bigint), INPUT__FILE__NAME (type: string), ROW__ID (type: struct) + Reducer 2 + Reduce Operator Tree: + Extract + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string, _col7: bigint, _col8: string, _col9: struct + type: TABLE + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + Statistics: Num rows: 17 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = 1142) (type: boolean) + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 8 Data size: 2531 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +where fl_num = 1142 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +Baltimore 1142 2010 10 20 6 20 +Baltimore 1142 2010 10 21 5 21 +Baltimore 1142 2010 10 22 4 22 +Baltimore 1142 2010 10 25 3 25 +Baltimore 1142 2010 10 26 2 26 +PREHOOK: query: -- 3. empty partition. +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +POSTHOOK: query: -- 3. empty partition. +explain +select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: flights_tiny + Statistics: Num rows: 24 Data size: 5379 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (fl_num = -1142) (type: boolean) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), dest_city_name (type: string), year (type: int), month (type: int), day_of_month (type: int), arr_delay (type: float) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: 0 (type: int), '-1142' (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int) + sort order: +++++ + Map-reduce partition columns: 0 (type: int) + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: float), '-1142' (type: string) + Reducer 2 + Reduce Operator Tree: + Extract + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: flights_tiny + output shape: _col0: string, _col1: string, _col2: int, _col3: int, _col4: int, _col5: float, _col6: string + type: SUBQUERY + Partition table definition + input alias: ptf_1 + arguments: 'LATE.LATE+', 'LATE', (_col5 > 15.0), 'origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath' + name: matchpath + order by: _col6, _col2, _col3, _col4 + output shape: origin_city_name: string, fl_num: string, year: int, month: int, day_of_month: int, sz: int, tpath: int + partition by: 0 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: origin_city_name (type: string), '-1142' (type: string), year (type: int), month (type: int), day_of_month (type: int), sz (type: int), tpath (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6 + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 12 Data size: 2689 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@flights_tiny +#### A masked pattern was here #### +POSTHOOK: query: select origin_city_name, fl_num, year, month, day_of_month, sz, tpath +from matchpath(on + (select * from flights_tiny where fl_num = -1142) flights_tiny + sort by fl_num, year, month, day_of_month + arg1('LATE.LATE+'), + arg2('LATE'), arg3(arr_delay > 15), + arg4('origin_city_name, fl_num, year, month, day_of_month, size(tpath) as sz, tpath[0].day_of_month as tpath') + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@flights_tiny +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/tez/ptf_streaming.q.out ql/src/test/results/clientpositive/tez/ptf_streaming.q.out index a935ef6..4d1da60 100644 --- ql/src/test/results/clientpositive/tez/ptf_streaming.q.out +++ ql/src/test/results/clientpositive/tez/ptf_streaming.q.out @@ -52,6 +52,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -66,6 +77,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -210,6 +252,17 @@ STAGE PLANS: Extract Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -224,6 +277,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -344,6 +415,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -460,7 +542,20 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -472,6 +567,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -485,6 +592,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -584,7 +709,20 @@ STAGE PLANS: alias: part Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -596,6 +734,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -610,6 +760,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -726,9 +907,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -740,6 +951,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -754,6 +983,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -872,9 +1132,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -886,6 +1176,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -900,6 +1208,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1018,9 +1357,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -1032,6 +1401,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1046,6 +1433,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1181,6 +1599,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noopstreaming + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: _col0 is not null (type: boolean) @@ -1214,6 +1643,50 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 1730 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -1352,9 +1825,45 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -1366,6 +1875,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -1380,6 +1907,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1527,6 +2085,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -1539,6 +2114,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -1551,6 +2137,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1565,6 +2162,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -1705,9 +2333,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -1719,6 +2377,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmapstreaming + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1733,6 +2403,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git ql/src/test/results/clientpositive/tez/subquery_in.q.out ql/src/test/results/clientpositive/tez/subquery_in.q.out index 8bc7892..db10ab5 100644 --- ql/src/test/results/clientpositive/tez/subquery_in.q.out +++ ql/src/test/results/clientpositive/tez/subquery_in.q.out @@ -338,6 +338,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 <= 2) (type: boolean) @@ -495,6 +513,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: ((_wcol0 <= 2) and _col0 is not null) (type: boolean) diff --git ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out index a814849..76e2e04 100644 --- ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out +++ ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out @@ -302,6 +302,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -319,6 +330,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -678,6 +720,17 @@ STAGE PLANS: Extract Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -695,6 +748,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -902,6 +973,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1160,6 +1242,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1177,6 +1270,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1447,6 +1571,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -1464,6 +1599,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -1744,6 +1911,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1786,6 +1964,38 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -2078,6 +2288,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2426,6 +2647,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2579,7 +2811,20 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -2643,6 +2888,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2659,6 +2916,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -2853,7 +3128,20 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -2917,6 +3205,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -2934,6 +3234,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3195,6 +3526,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3212,6 +3554,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3483,9 +3856,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) sort order: +++ @@ -3500,6 +3903,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3517,6 +3938,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3808,6 +4260,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -3825,6 +4288,29 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -4260,6 +4746,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -4301,6 +4798,50 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -4519,6 +5060,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -4820,6 +5372,17 @@ STAGE PLANS: Extract Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string), _col1 (type: string) @@ -4837,6 +5400,23 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -5265,6 +5845,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -5290,6 +5881,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -5329,6 +5951,23 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string), _col1 (type: string) @@ -5346,6 +5985,44 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -5776,9 +6453,45 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -5793,6 +6506,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -5810,6 +6541,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6132,6 +6894,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -6147,6 +6926,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -6162,6 +6952,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6179,6 +6980,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6484,6 +7316,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -6499,6 +7348,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -6516,6 +7382,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -6833,6 +7730,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) @@ -6848,9 +7762,33 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -6865,6 +7803,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -6882,6 +7832,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -7230,9 +8211,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col2 (type: string) sort order: ++ @@ -7247,6 +8258,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) @@ -7264,6 +8293,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -7578,9 +8638,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string), _col2 (type: string), _col1 (type: string) sort order: ++++ @@ -7595,6 +8685,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col2 (type: string), _col1 (type: string) @@ -7612,6 +8714,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git ql/src/test/results/clientpositive/vectorized_ptf.q.out ql/src/test/results/clientpositive/vectorized_ptf.q.out index 1e3c43c..b1d18d3 100644 --- ql/src/test/results/clientpositive/vectorized_ptf.q.out +++ ql/src/test/results/clientpositive/vectorized_ptf.q.out @@ -296,6 +296,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -360,6 +371,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -707,6 +749,17 @@ STAGE PLANS: Extract Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: j + output shape: _col1: string, _col2: string, _col5: int + type: SUBQUERY + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -771,6 +824,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), (_col5 - _wcol0) (type: int) @@ -972,6 +1043,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -1224,6 +1306,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1288,6 +1381,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -1552,6 +1676,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -1616,6 +1751,38 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), (_col5 - _wcol2) (type: int) @@ -1890,6 +2057,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -2025,6 +2203,38 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col2, 1, _col2 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: int), _wcol0 (type: int), _wcol1 (type: int), _col2 (type: int), (_col2 - _wcol2) (type: int) @@ -2244,6 +2454,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2588,6 +2809,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col3: string, _col4: string, _col5: int, _col6: string, _col7: double, _col8: string + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -2879,7 +3111,20 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name, p_size(DESC) + output shape: p_name: string, p_mfgr: string, p_size: int + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string), p_size (type: int) sort order: ++- @@ -2942,6 +3187,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1, _col5(DESC) + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3005,6 +3262,24 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1, _col5(DESC) + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1, _col5 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int) @@ -3194,7 +3469,20 @@ STAGE PLANS: Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE GatherStats: false PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: p_name + output shape: p_name: string, p_mfgr: string, p_size: int, p_retailprice: double + partition by: p_mfgr + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true Reduce Output Operator key expressions: p_mfgr (type: string), p_name (type: string) sort order: ++ @@ -3257,6 +3545,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3321,6 +3621,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3576,6 +3907,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -3640,6 +3982,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -3905,9 +4278,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -3969,6 +4372,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -4033,6 +4454,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -4318,6 +4770,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -4382,6 +4845,29 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol1 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: bigint), _wcol1 (type: double) @@ -4744,6 +5230,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: abc + name: noop + order by: _col1 + output shape: _col0: int, _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Filter Operator isSamplingPred: false @@ -4938,6 +5435,50 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol3 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ + window function definition + alias: _wcol4 + arguments: _col5, 1, _col5 + name: lag + window function: GenericUDAFLagEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 14 Data size: 8823 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: bigint), _col7 (type: double), _wcol3 (type: double), _col5 (type: int), (_col5 - _wcol4) (type: int) @@ -5150,6 +5691,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int) @@ -5537,6 +6089,17 @@ STAGE PLANS: Extract Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: mfgr_price_view + output shape: _col0: string, _col1: string, _col2: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col0 + output shape: _col0: string, _col1: string, _col2: double + partition by: _col0 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -5601,6 +6164,23 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col2 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(2)~ Statistics: Num rows: 13 Data size: 8021 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: string), _col2 (type: double), _wcol0 (type: double) @@ -6022,6 +6602,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col1 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -6103,6 +6694,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5, _col7 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int, _col7: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col7 + name: sum + window function: GenericUDAFSumDouble + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _col5 (type: int), _wcol0 (type: int), _wcol1 (type: int), _wcol2 (type: double) @@ -6206,6 +6828,23 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col5 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(5)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -6270,6 +6909,44 @@ STAGE PLANS: outputColumnNames: _col0, _col2, _col3, _col6 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: bigint, _col2: string, _col3: string, _col6: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col3, _col2 + partition by: _col3 + window functions: + window function definition + alias: _wcol1 + arguments: _col3, _col2 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col3, _col2 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol3 + arguments: _col3, _col2 + name: cume_dist + window function: GenericUDAFCumeDistEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol4 + arguments: _col6, true + name: first_value + window function: GenericUDAFFirstValueEvaluator + window frame: PRECEDING(2)~FOLLOWING(2) Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col3 (type: string), _col2 (type: string), _col6 (type: int), UDFToInteger(round(_col0, 1)) (type: int), _wcol1 (type: int), _wcol2 (type: int), _wcol3 (type: double), _wcol4 (type: int) @@ -6665,9 +7342,45 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -6729,6 +7442,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -6793,6 +7524,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -7109,6 +7871,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7171,6 +7950,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7233,6 +8023,17 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7297,6 +8098,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -7596,6 +8428,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7658,6 +8507,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -7722,6 +8588,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -8033,6 +8930,23 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -8095,9 +9009,33 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -8159,6 +9097,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -8223,6 +9173,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint) @@ -8565,9 +9546,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -8629,6 +9640,24 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 + transforms raw input: true + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -8693,6 +9722,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2, _col1 + partition by: _col2, _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col2, _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col2, _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) @@ -9001,9 +10061,39 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: part_orc + output shape: _col1: string, _col2: string, _col5: int + type: TABLE + Partition table definition + input alias: ptf_1 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + Partition table definition + input alias: ptf_2 + name: noop + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE + Map-side function: true File Output Operator compressed: false GlobalTableId: 0 @@ -9065,6 +10155,18 @@ STAGE PLANS: Extract Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: PTFCOMPONENT + Partition table definition + input alias: ptf_1 + name: noopwithmap + order by: _col2, _col1 + output shape: _col1: string, _col2: string, _col5: int + partition by: _col2, _col1 + transforms raw input: true Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false @@ -9129,6 +10231,37 @@ STAGE PLANS: outputColumnNames: _col1, _col2, _col5 Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col1: string, _col2: string, _col5: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col2 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol1 + arguments: _col1 + name: dense_rank + window function: GenericUDAFDenseRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + window function definition + alias: _wcol2 + arguments: _col5 + name: sum + window function: GenericUDAFSumLong + window frame: PRECEDING(MAX)~ Statistics: Num rows: 26 Data size: 16042 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col2 (type: string), _col1 (type: string), _wcol0 (type: int), _wcol1 (type: int), _col5 (type: int), _wcol2 (type: bigint), _wcol2 (type: bigint) diff --git ql/src/test/results/clientpositive/windowing_streaming.q.out ql/src/test/results/clientpositive/windowing_streaming.q.out index ac9e180..fb9798d 100644 --- ql/src/test/results/clientpositive/windowing_streaming.q.out +++ ql/src/test/results/clientpositive/windowing_streaming.q.out @@ -80,6 +80,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 + partition by: _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col0 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col1 (type: string), _wcol0 (type: int) @@ -136,6 +154,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col0 + partition by: _col1 + window functions: + window function definition + alias: _wcol0 + arguments: _col0 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 26 Data size: 3147 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 < 4) (type: boolean) @@ -290,6 +326,24 @@ STAGE PLANS: outputColumnNames: _col0, _col1 Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: tinyint, _col1: double + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col1 + partition by: _col0 + window functions: + window function definition + alias: _wcol0 + arguments: _col1 + name: rank + window function: GenericUDAFRankEvaluator + window frame: PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (_wcol0 < 5) (type: boolean)