Index: ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java (revision 1527202) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java (working copy) @@ -19,9 +19,9 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; -import java.util.ArrayList; +import java.util.ArrayDeque; +import java.util.Deque; import java.util.List; -import java.util.Stack; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; @@ -30,12 +30,12 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFInputDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.plan.PTFDeserializer; import org.apache.hadoop.hive.ql.plan.api.OperatorType; +import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFInputDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLeadLag; import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator; import org.apache.hadoop.hive.serde2.SerDe; @@ -147,7 +147,7 @@ protected void setupKeysWrapper(ObjectInspector inputOI) throws HiveException { PartitionDef pDef = conf.getStartOfChain().getPartition(); - ArrayList exprs = pDef.getExpressions(); + List exprs = pDef.getExpressions(); int numExprs = exprs.size(); ExprNodeEvaluator[] keyFields = new ExprNodeEvaluator[numExprs]; ObjectInspector[] keyOIs = new ObjectInspector[numExprs]; @@ -220,7 +220,7 @@ */ private PTFPartition executeChain(PTFPartition part) throws HiveException { - Stack fnDefs = new Stack(); + Deque fnDefs = new ArrayDeque(); PTFInputDef iDef = conf.getFuncDef(); while (iDef instanceof PartitionedTableFunctionDef) { @@ -289,6 +289,4 @@ } } - - } Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (revision 1527202) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (working copy) @@ -67,14 +67,14 @@ import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFunctionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowTableFunctionDef; import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.SelectDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.TableScanDesc; +import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java (revision 1527202) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/PTFInvocationSpec.java (working copy) @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.parse; import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.hive.ql.exec.PTFUtils; @@ -124,7 +125,7 @@ public static class PartitionedTableFunctionSpec extends PTFInputSpec { String name; String alias; - ArrayList args; + List args; PartitioningSpec partitioning; PTFInputSpec input; public String getName() { @@ -139,10 +140,10 @@ public void setAlias(String alias) { this.alias = alias; } - public ArrayList getArgs() { + public List getArgs() { return args; } - public void setArgs(ArrayList args) { + public void setArgs(List args) { this.args = args; } public PartitioningSpec getPartitioning() { Index: ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java (revision 1527202) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java (working copy) @@ -18,7 +18,9 @@ package org.apache.hadoop.hive.ql.parse; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -60,22 +62,22 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.BoundaryDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.CurrentRowDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.OrderDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.OrderExpressionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFInputDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFQueryInputDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.RangeBoundaryDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.ShapeDetails; -import org.apache.hadoop.hive.ql.plan.PTFDesc.ValueBoundaryDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFrameDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFunctionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowTableFunctionDef; import org.apache.hadoop.hive.ql.plan.PTFDeserializer; +import org.apache.hadoop.hive.ql.plan.ptf.BoundaryDef; +import org.apache.hadoop.hive.ql.plan.ptf.CurrentRowDef; +import org.apache.hadoop.hive.ql.plan.ptf.OrderDef; +import org.apache.hadoop.hive.ql.plan.ptf.OrderExpressionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFInputDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFQueryInputDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.RangeBoundaryDef; +import org.apache.hadoop.hive.ql.plan.ptf.ShapeDetails; +import org.apache.hadoop.hive.ql.plan.ptf.ValueBoundaryDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLeadLag; import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator; @@ -224,7 +226,7 @@ private void translatePTFChain() throws SemanticException { - Stack ptfChain = new Stack(); + Deque ptfChain = new ArrayDeque(); PTFInputSpec currentSpec = ptfInvocation.getFunction(); while (currentSpec != null) { ptfChain.push(currentSpec); @@ -280,7 +282,7 @@ /* * translate args */ - ArrayList args = spec.getArgs(); + List args = spec.getArgs(); if (args != null) { for (ASTNode expr : args) @@ -303,7 +305,7 @@ if (tFn.transformsRawInput()) { StructObjectInspector rawInOutOI = tEval.getRawInputOI(); - ArrayList rawInOutColNames = tFn.getRawInputColumnNames(); + List rawInOutColNames = tFn.getRawInputColumnNames(); RowResolver rawInRR = buildRowResolverForPTF(def.getName(), spec.getAlias(), rawInOutOI, @@ -324,7 +326,7 @@ tFn.setupOutputOI(); StructObjectInspector outputOI = tEval.getOutputOI(); - ArrayList outColNames = tFn.getOutputColumnNames(); + List outColNames = tFn.getOutputColumnNames(); RowResolver outRR = buildRowResolverForPTF(def.getName(), spec.getAlias(), outputOI, @@ -566,8 +568,8 @@ } static void setupWdwFnEvaluator(WindowFunctionDef def) throws HiveException { - ArrayList args = def.getArgs(); - ArrayList argOIs = new ArrayList(); + List args = def.getArgs(); + List argOIs = new ArrayList(); ObjectInspector[] funcArgOIs = null; if (args != null) { @@ -619,7 +621,7 @@ } private ShapeDetails setupTableFnShape(String fnName, ShapeDetails inpShape, - StructObjectInspector OI, ArrayList columnNames, RowResolver rr) + StructObjectInspector OI, List columnNames, RowResolver rr) throws SemanticException { if (fnName.equals(FunctionRegistry.NOOP_TABLE_FUNCTION) || fnName.equals( @@ -630,7 +632,7 @@ } private ShapeDetails setupShape(StructObjectInspector OI, - ArrayList columnNames, + List columnNames, RowResolver rr) throws SemanticException { Map serdePropsMap = new LinkedHashMap(); SerDe serde = null; @@ -672,7 +674,7 @@ private ShapeDetails setupShapeForNoop(ShapeDetails inpShape, StructObjectInspector OI, - ArrayList columnNames, + List columnNames, RowResolver rr) throws SemanticException { ShapeDetails shp = new ShapeDetails(); @@ -738,7 +740,7 @@ throw new SemanticException("Ranking Functions can take no arguments"); } OrderDef oDef = wdwTFnDef.getOrder(); - ArrayList oExprs = oDef.getExpressions(); + List oExprs = oDef.getExpressions(); for (OrderExpressionDef oExpr : oExprs) { wFnDef.addArg(oExpr); } @@ -871,7 +873,7 @@ protected static RowResolver buildRowResolverForPTF(String tbFnName, String tabAlias, StructObjectInspector rowObjectInspector, - ArrayList outputColNames, RowResolver inputRR) throws SemanticException { + List outputColNames, RowResolver inputRR) throws SemanticException { if (tbFnName.equals(FunctionRegistry.NOOP_TABLE_FUNCTION) || tbFnName.equals(FunctionRegistry.NOOP_MAP_TABLE_FUNCTION)) { Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1527202) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -139,9 +139,6 @@ import org.apache.hadoop.hive.ql.plan.MapJoinDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.OrderExpressionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.plan.PlanUtils; import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.ScriptDesc; @@ -150,6 +147,9 @@ import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.UDTFDesc; import org.apache.hadoop.hive.ql.plan.UnionDesc; +import org.apache.hadoop.hive.ql.plan.ptf.OrderExpressionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.ResourceType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; @@ -9935,7 +9935,7 @@ RowResolver rsOpRR, RowResolver extractRR) throws SemanticException { - ArrayList partColList = tabDef.getPartition().getExpressions(); + List partColList = tabDef.getPartition().getExpressions(); for (PTFExpressionDef colDef : partColList) { partCols.add(colDef.getExprNode()); @@ -9950,7 +9950,7 @@ * we need to set includeKeyCols = false while creating the * ReduceSinkDesc */ - ArrayList orderColList = tabDef.getOrder().getExpressions(); + List orderColList = tabDef.getOrder().getExpressions(); for (int i = 0; i < orderColList.size(); i++) { OrderExpressionDef colDef = orderColList.get(i); org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order order = colDef.getOrder(); Index: ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java (revision 1527202) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDesc.java (working copy) @@ -18,29 +18,19 @@ package org.apache.hadoop.hive.ql.plan; -import java.util.ArrayList; -import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.PTFUtils; import org.apache.hadoop.hive.ql.parse.LeadLagInfo; import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order; import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputType; -import org.apache.hadoop.hive.ql.parse.RowResolver; -import org.apache.hadoop.hive.ql.parse.TypeCheckCtx; -import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; -import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator; -import org.apache.hadoop.hive.serde2.SerDe; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef; @Explain(displayName = "PTF Operator") -public class PTFDesc extends AbstractOperatorDesc -{ +public class PTFDesc extends AbstractOperatorDesc { private static final long serialVersionUID = 1L; @SuppressWarnings("unused") private static final Log LOG = LogFactory.getLog(PTFDesc.class.getName()); @@ -99,538 +89,4 @@ this.cfg = cfg; } - public abstract static class PTFInputDef { - String expressionTreeString; - ShapeDetails outputShape; - String alias; - - public String getExpressionTreeString() { - return expressionTreeString; - } - - public void setExpressionTreeString(String expressionTreeString) { - this.expressionTreeString = expressionTreeString; - } - - public ShapeDetails getOutputShape() { - return outputShape; - } - - public void setOutputShape(ShapeDetails outputShape) { - this.outputShape = outputShape; - } - public String getAlias() { - return alias; - } - public void setAlias(String alias) { - this.alias = alias; - } - - public abstract PTFInputDef getInput(); - } - - public static class PTFQueryInputDef extends PTFInputDef { - String destination; - PTFQueryInputType type; - public String getDestination() { - return destination; - } - public void setDestination(String destination) { - this.destination = destination; - } - public PTFQueryInputType getType() { - return type; - } - public void setType(PTFQueryInputType type) { - this.type = type; - } - - @Override - public PTFInputDef getInput() { - return null; - } - } - - public static class PartitionedTableFunctionDef extends PTFInputDef { - String name; - String resolverClassName; - ShapeDetails rawInputShape; - boolean carryForwardNames; - PTFInputDef input; - ArrayList args; - PartitionDef partition; - OrderDef order; - TableFunctionEvaluator tFunction; - boolean transformsRawInput; - public String getName() { - return name; - } - public void setName(String name) { - this.name = name; - } - public ShapeDetails getRawInputShape() { - return rawInputShape; - } - public void setRawInputShape(ShapeDetails rawInputShape) { - this.rawInputShape = rawInputShape; - } - public boolean isCarryForwardNames() { - return carryForwardNames; - } - public void setCarryForwardNames(boolean carryForwardNames) { - this.carryForwardNames = carryForwardNames; - } - @Override - public PTFInputDef getInput() { - return input; - } - public void setInput(PTFInputDef input) { - this.input = input; - } - public PartitionDef getPartition() { - return partition; - } - public void setPartition(PartitionDef partition) { - this.partition = partition; - } - public OrderDef getOrder() { - return order; - } - public void setOrder(OrderDef order) { - this.order = order; - } - public TableFunctionEvaluator getTFunction() { - return tFunction; - } - public void setTFunction(TableFunctionEvaluator tFunction) { - this.tFunction = tFunction; - } - public ArrayList getArgs() { - return args; - } - - public void setArgs(ArrayList args) { - this.args = args; - } - - public void addArg(PTFExpressionDef arg) { - args = args == null ? new ArrayList() : args; - args.add(arg); - } - - public PartitionedTableFunctionDef getStartOfChain() { - if (input instanceof PartitionedTableFunctionDef ) { - return ((PartitionedTableFunctionDef)input).getStartOfChain(); - } - return this; - } - public boolean isTransformsRawInput() { - return transformsRawInput; - } - public void setTransformsRawInput(boolean transformsRawInput) { - this.transformsRawInput = transformsRawInput; - } - public String getResolverClassName() { - return resolverClassName; - } - public void setResolverClassName(String resolverClassName) { - this.resolverClassName = resolverClassName; - } - } - - public static class WindowTableFunctionDef extends PartitionedTableFunctionDef { - ArrayList windowFunctions; - - public ArrayList getWindowFunctions() { - return windowFunctions; - } - public void setWindowFunctions(ArrayList windowFunctions) { - this.windowFunctions = windowFunctions; - } - } - - public static class ShapeDetails { - String serdeClassName; - Map serdeProps; - ArrayList columnNames; - transient StructObjectInspector OI; - transient SerDe serde; - transient RowResolver rr; - transient TypeCheckCtx typeCheckCtx; - - static{ - PTFUtils.makeTransient(ShapeDetails.class, "OI", "serde", "rr", "typeCheckCtx"); - } - - public String getSerdeClassName() { - return serdeClassName; - } - - public void setSerdeClassName(String serdeClassName) { - this.serdeClassName = serdeClassName; - } - - public Map getSerdeProps() { - return serdeProps; - } - - public void setSerdeProps(Map serdeProps) { - this.serdeProps = serdeProps; - } - - public ArrayList getColumnNames() { - return columnNames; - } - - public void setColumnNames(ArrayList columnNames) { - this.columnNames = columnNames; - } - - public StructObjectInspector getOI() { - return OI; - } - - public void setOI(StructObjectInspector oI) { - OI = oI; - } - - public SerDe getSerde() { - return serde; - } - - public void setSerde(SerDe serde) { - this.serde = serde; - } - - public RowResolver getRr() { - return rr; - } - - public void setRr(RowResolver rr) { - this.rr = rr; - } - - public TypeCheckCtx getTypeCheckCtx() { - return typeCheckCtx; - } - - public void setTypeCheckCtx(TypeCheckCtx typeCheckCtx) { - this.typeCheckCtx = typeCheckCtx; - } - } - - public static class PartitionDef { - ArrayList expressions; - - public ArrayList getExpressions() { - return expressions; - } - - public void setExpressions(ArrayList expressions) { - this.expressions = expressions; - } - public void addExpression(PTFExpressionDef e) { - expressions = expressions == null ? new ArrayList() : expressions; - expressions.add(e); - } - } - - public static class OrderDef { - ArrayList expressions; - - public OrderDef() {} - - public OrderDef(PartitionDef pDef) { - for(PTFExpressionDef eDef : pDef.getExpressions()) - { - addExpression(new OrderExpressionDef(eDef)); - } - } - - public ArrayList getExpressions() { - return expressions; - } - - public void setExpressions(ArrayList expressions) { - this.expressions = expressions; - } - public void addExpression(OrderExpressionDef e) { - expressions = expressions == null ? new ArrayList() : expressions; - expressions.add(e); - } - } - - public static class OrderExpressionDef extends PTFExpressionDef { - Order order; - - public OrderExpressionDef() {} - public OrderExpressionDef(PTFExpressionDef e) { - super(e); - order = Order.ASC; - } - - public Order getOrder() { - return order; - } - - public void setOrder(Order order) { - this.order = order; - } - } - - public static class WindowExpressionDef extends PTFExpressionDef { - String alias; - - public WindowExpressionDef() {} - public WindowExpressionDef(PTFExpressionDef eDef) { - super(eDef); - } - public String getAlias() { - return alias; - } - - public void setAlias(String alias) { - this.alias = alias; - } - } - - public static class WindowFunctionDef extends WindowExpressionDef - { - String name; - boolean isStar; - boolean isDistinct; - ArrayList args; - WindowFrameDef windowFrame; - GenericUDAFEvaluator wFnEval; - boolean pivotResult; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public boolean isStar() { - return isStar; - } - - public void setStar(boolean isStar) { - this.isStar = isStar; - } - - public boolean isDistinct() { - return isDistinct; - } - - public void setDistinct(boolean isDistinct) { - this.isDistinct = isDistinct; - } - - public ArrayList getArgs() { - return args; - } - - public void setArgs(ArrayList args) { - this.args = args; - } - - public void addArg(PTFExpressionDef arg) { - args = args == null ? new ArrayList() : args; - args.add(arg); - } - - public WindowFrameDef getWindowFrame() { - return windowFrame; - } - - public void setWindowFrame(WindowFrameDef windowFrame) { - this.windowFrame = windowFrame; - } - - public GenericUDAFEvaluator getWFnEval() { - return wFnEval; - } - - public void setWFnEval(GenericUDAFEvaluator wFnEval) { - this.wFnEval = wFnEval; - } - - public boolean isPivotResult() { - return pivotResult; - } - - public void setPivotResult(boolean pivotResult) { - this.pivotResult = pivotResult; - } - - } - - public static class WindowFrameDef - { - BoundaryDef start; - BoundaryDef end; - public BoundaryDef getStart() { - return start; - } - public void setStart(BoundaryDef start) { - this.start = start; - } - public BoundaryDef getEnd() { - return end; - } - public void setEnd(BoundaryDef end) { - this.end = end; - } - } - - public static abstract class BoundaryDef { - Direction direction; - - public Direction getDirection() { - return direction; - } - - public void setDirection(Direction direction) { - this.direction = direction; - } - - public abstract int getAmt(); - } - - public static class RangeBoundaryDef extends BoundaryDef { - int amt; - - public int compareTo(BoundaryDef other) - { - int c = getDirection().compareTo(other.getDirection()); - if ( c != 0) { - return c; - } - RangeBoundaryDef rb = (RangeBoundaryDef) other; - return getAmt() - rb.getAmt(); - } - - @Override - public int getAmt() { - return amt; - } - - public void setAmt(int amt) { - this.amt = amt; - } - } - - public static class CurrentRowDef extends BoundaryDef - { - public int compareTo(BoundaryDef other) - { - return getDirection().compareTo(other.getDirection()); - } - @Override - public Direction getDirection() { - return Direction.CURRENT; - } - - @Override - public int getAmt() { return 0; } - } - - public static class ValueBoundaryDef extends BoundaryDef - { - PTFExpressionDef expressionDef; - int amt; - - public int compareTo(BoundaryDef other) { - int c = getDirection().compareTo(other.getDirection()); - if ( c != 0) { - return c; - } - ValueBoundaryDef vb = (ValueBoundaryDef) other; - return getAmt() - vb.getAmt(); - } - - public PTFExpressionDef getExpressionDef() { - return expressionDef; - } - - public void setExpressionDef(PTFExpressionDef expressionDef) { - this.expressionDef = expressionDef; - } - - public ExprNodeDesc getExprNode() { - return expressionDef == null ? null : expressionDef.getExprNode(); - } - - public ExprNodeEvaluator getExprEvaluator() { - return expressionDef == null ? null : expressionDef.getExprEvaluator(); - } - - public ObjectInspector getOI() { - return expressionDef == null ? null : expressionDef.getOI(); - } - - @Override - public int getAmt() { - return amt; - } - - public void setAmt(int amt) { - this.amt = amt; - } - } - - public static class PTFExpressionDef - { - String expressionTreeString; - ExprNodeDesc exprNode; - transient ExprNodeEvaluator exprEvaluator; - transient ObjectInspector OI; - - static{ - PTFUtils.makeTransient(PTFExpressionDef.class, "exprEvaluator", "OI"); - } - - public PTFExpressionDef() {} - public PTFExpressionDef(PTFExpressionDef e) { - expressionTreeString = e.getExpressionTreeString(); - exprNode = e.getExprNode(); - exprEvaluator = e.getExprEvaluator(); - OI = e.getOI(); - } - - public String getExpressionTreeString() { - return expressionTreeString; - } - - public void setExpressionTreeString(String expressionTreeString) { - this.expressionTreeString = expressionTreeString; - } - - public ExprNodeDesc getExprNode() { - return exprNode; - } - - public void setExprNode(ExprNodeDesc exprNode) { - this.exprNode = exprNode; - } - - public ExprNodeEvaluator getExprEvaluator() { - return exprEvaluator; - } - - public void setExprEvaluator(ExprNodeEvaluator exprEvaluator) { - this.exprEvaluator = exprEvaluator; - } - - public ObjectInspector getOI() { - return OI; - } - - public void setOI(ObjectInspector oI) { - OI = oI; - } - } - } Index: ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java (revision 1527202) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java (working copy) @@ -18,12 +18,13 @@ package org.apache.hadoop.hive.ql.plan; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.Stack; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; @@ -31,16 +32,16 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.LeadLagInfo; import org.apache.hadoop.hive.ql.parse.WindowingExprNodeEvaluatorFactory; -import org.apache.hadoop.hive.ql.plan.PTFDesc.BoundaryDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFInputDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFQueryInputDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.ShapeDetails; -import org.apache.hadoop.hive.ql.plan.PTFDesc.ValueBoundaryDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFrameDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFunctionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.BoundaryDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFInputDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFQueryInputDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.ShapeDetails; +import org.apache.hadoop.hive.ql.plan.ptf.ValueBoundaryDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFLeadLag; import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator; @@ -76,7 +77,7 @@ } public void initializePTFChain(PartitionedTableFunctionDef tblFnDef) throws HiveException { - Stack ptfChain = new Stack(); + Deque ptfChain = new ArrayDeque(); PTFInputDef currentDef = tblFnDef; while (currentDef != null) { ptfChain.push(currentDef); @@ -188,8 +189,8 @@ } static void setupWdwFnEvaluator(WindowFunctionDef def) throws HiveException { - ArrayList args = def.getArgs(); - ArrayList argOIs = new ArrayList(); + List args = def.getArgs(); + List argOIs = new ArrayList(); ObjectInspector[] funcArgOIs = null; if (args != null) { Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java (working copy) @@ -0,0 +1,17 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + +import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction; + +public abstract class BoundaryDef { + Direction direction; + + public Direction getDirection() { + return direction; + } + + public void setDirection(Direction direction) { + this.direction = direction; + } + + public abstract int getAmt(); +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/CurrentRowDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/CurrentRowDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/CurrentRowDef.java (working copy) @@ -0,0 +1,20 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + +import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction; + +public class CurrentRowDef extends BoundaryDef { + + public int compareTo(BoundaryDef other) { + return getDirection().compareTo(other.getDirection()); + } + + @Override + public Direction getDirection() { + return Direction.CURRENT; + } + + @Override + public int getAmt() { + return 0; + } +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/OrderDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/OrderDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/OrderDef.java (working copy) @@ -0,0 +1,29 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + +import java.util.ArrayList; +import java.util.List; + +public class OrderDef { + List expressions; + + public OrderDef() {} + + public OrderDef(PartitionDef pDef) { + for(PTFExpressionDef eDef : pDef.getExpressions()) { + addExpression(new OrderExpressionDef(eDef)); + } + } + + public List getExpressions() { + return expressions; + } + + public void setExpressions(ArrayList expressions) { + this.expressions = expressions; + } + public void addExpression(OrderExpressionDef e) { + expressions = expressions == null ? new ArrayList() : expressions; + expressions.add(e); + } +} + Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/OrderExpressionDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/OrderExpressionDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/OrderExpressionDef.java (working copy) @@ -0,0 +1,22 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + +import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.Order; + +public class OrderExpressionDef extends PTFExpressionDef { + private Order order; + + public OrderExpressionDef() {} + public OrderExpressionDef(PTFExpressionDef e) { + super(e); + order = Order.ASC; + } + + public Order getOrder() { + return order; + } + + public void setOrder(Order order) { + this.order = order; + } +} + Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFExpressionDef.java (working copy) @@ -0,0 +1,58 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; +import org.apache.hadoop.hive.ql.exec.PTFUtils; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; + +public class PTFExpressionDef { + String expressionTreeString; + ExprNodeDesc exprNode; + transient ExprNodeEvaluator exprEvaluator; + transient ObjectInspector OI; + + static{ + PTFUtils.makeTransient(PTFExpressionDef.class, "exprEvaluator", "OI"); + } + + public PTFExpressionDef() {} + + public PTFExpressionDef(PTFExpressionDef e) { + expressionTreeString = e.getExpressionTreeString(); + exprNode = e.getExprNode(); + exprEvaluator = e.getExprEvaluator(); + OI = e.getOI(); + } + + public String getExpressionTreeString() { + return expressionTreeString; + } + + public void setExpressionTreeString(String expressionTreeString) { + this.expressionTreeString = expressionTreeString; + } + + public ExprNodeDesc getExprNode() { + return exprNode; + } + + public void setExprNode(ExprNodeDesc exprNode) { + this.exprNode = exprNode; + } + + public ExprNodeEvaluator getExprEvaluator() { + return exprEvaluator; + } + + public void setExprEvaluator(ExprNodeEvaluator exprEvaluator) { + this.exprEvaluator = exprEvaluator; + } + + public ObjectInspector getOI() { + return OI; + } + + public void setOI(ObjectInspector oI) { + OI = oI; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java (working copy) @@ -0,0 +1,32 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + + +public abstract class PTFInputDef { + private String expressionTreeString; + private ShapeDetails outputShape; + private String alias; + + public String getExpressionTreeString() { + return expressionTreeString; + } + + public void setExpressionTreeString(String expressionTreeString) { + this.expressionTreeString = expressionTreeString; + } + + public ShapeDetails getOutputShape() { + return outputShape; + } + + public void setOutputShape(ShapeDetails outputShape) { + this.outputShape = outputShape; + } + public String getAlias() { + return alias; + } + public void setAlias(String alias) { + this.alias = alias; + } + + public abstract PTFInputDef getInput(); +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFQueryInputDef.java (working copy) @@ -0,0 +1,29 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + +import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.PTFQueryInputType; + +public class PTFQueryInputDef extends PTFInputDef { + private String destination; + private PTFQueryInputType type; + + public String getDestination() { + return destination; + } + + public void setDestination(String destination) { + this.destination = destination; + } + + public PTFQueryInputType getType() { + return type; + } + + public void setType(PTFQueryInputType type) { + this.type = type; + } + + @Override + public PTFInputDef getInput() { + return null; + } +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionDef.java (working copy) @@ -0,0 +1,21 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + +import java.util.ArrayList; +import java.util.List; + + +public class PartitionDef { + private List expressions; + + public List getExpressions() { + return expressions; + } + + public void setExpressions(List expressions) { + this.expressions = expressions; + } + public void addExpression(PTFExpressionDef e) { + expressions = expressions == null ? new ArrayList() : expressions; + expressions.add(e); + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PartitionedTableFunctionDef.java (working copy) @@ -0,0 +1,111 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionEvaluator; + +public class PartitionedTableFunctionDef extends PTFInputDef { + private String name; + private String resolverClassName; + private ShapeDetails rawInputShape; + private boolean carryForwardNames; + private PTFInputDef input; + private List args; + private PartitionDef partition; + private OrderDef order; + private TableFunctionEvaluator tFunction; + boolean transformsRawInput; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public ShapeDetails getRawInputShape() { + return rawInputShape; + } + + public void setRawInputShape(ShapeDetails rawInputShape) { + this.rawInputShape = rawInputShape; + } + + public boolean isCarryForwardNames() { + return carryForwardNames; + } + + public void setCarryForwardNames(boolean carryForwardNames) { + this.carryForwardNames = carryForwardNames; + } + + @Override + public PTFInputDef getInput() { + return input; + } + + public void setInput(PTFInputDef input) { + this.input = input; + } + + public PartitionDef getPartition() { + return partition; + } + + public void setPartition(PartitionDef partition) { + this.partition = partition; + } + + public OrderDef getOrder() { + return order; + } + + public void setOrder(OrderDef order) { + this.order = order; + } + + public TableFunctionEvaluator getTFunction() { + return tFunction; + } + public void setTFunction(TableFunctionEvaluator tFunction) { + this.tFunction = tFunction; + } + + public List getArgs() { + return args; + } + + public void setArgs(List args) { + this.args = args; + } + + public void addArg(PTFExpressionDef arg) { + args = args == null ? new ArrayList() : args; + args.add(arg); + } + + public PartitionedTableFunctionDef getStartOfChain() { + if (input instanceof PartitionedTableFunctionDef ) { + return ((PartitionedTableFunctionDef)input).getStartOfChain(); + } + return this; + } + + public boolean isTransformsRawInput() { + return transformsRawInput; + } + + public void setTransformsRawInput(boolean transformsRawInput) { + this.transformsRawInput = transformsRawInput; + } + + public String getResolverClassName() { + return resolverClassName; + } + + public void setResolverClassName(String resolverClassName) { + this.resolverClassName = resolverClassName; + } +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/RangeBoundaryDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/RangeBoundaryDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/RangeBoundaryDef.java (working copy) @@ -0,0 +1,24 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + + +public class RangeBoundaryDef extends BoundaryDef { + private int amt; + + public int compareTo(BoundaryDef other) { + int c = getDirection().compareTo(other.getDirection()); + if (c != 0) { + return c; + } + RangeBoundaryDef rb = (RangeBoundaryDef) other; + return getAmt() - rb.getAmt(); + } + + @Override + public int getAmt() { + return amt; + } + + public void setAmt(int amt) { + this.amt = amt; + } +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ShapeDetails.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ShapeDetails.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ShapeDetails.java (working copy) @@ -0,0 +1,80 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hive.ql.exec.PTFUtils; +import org.apache.hadoop.hive.ql.parse.RowResolver; +import org.apache.hadoop.hive.ql.parse.TypeCheckCtx; +import org.apache.hadoop.hive.serde2.SerDe; +import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; + +public class ShapeDetails { + String serdeClassName; + Map serdeProps; + List columnNames; + transient StructObjectInspector OI; + transient SerDe serde; + transient RowResolver rr; + transient TypeCheckCtx typeCheckCtx; + + static { + PTFUtils.makeTransient(ShapeDetails.class, "OI", "serde", "rr", "typeCheckCtx"); + } + + public String getSerdeClassName() { + return serdeClassName; + } + + public void setSerdeClassName(String serdeClassName) { + this.serdeClassName = serdeClassName; + } + + public Map getSerdeProps() { + return serdeProps; + } + + public void setSerdeProps(Map serdeProps) { + this.serdeProps = serdeProps; + } + + public List getColumnNames() { + return columnNames; + } + + public void setColumnNames(List columnNames) { + this.columnNames = columnNames; + } + + public StructObjectInspector getOI() { + return OI; + } + + public void setOI(StructObjectInspector oI) { + OI = oI; + } + + public SerDe getSerde() { + return serde; + } + + public void setSerde(SerDe serde) { + this.serde = serde; + } + + public RowResolver getRr() { + return rr; + } + + public void setRr(RowResolver rr) { + this.rr = rr; + } + + public TypeCheckCtx getTypeCheckCtx() { + return typeCheckCtx; + } + + public void setTypeCheckCtx(TypeCheckCtx typeCheckCtx) { + this.typeCheckCtx = typeCheckCtx; + } +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ValueBoundaryDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ValueBoundaryDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ValueBoundaryDef.java (working copy) @@ -0,0 +1,48 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + +import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; +import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; + +public class ValueBoundaryDef extends BoundaryDef { + private PTFExpressionDef expressionDef; + private int amt; + + public int compareTo(BoundaryDef other) { + int c = getDirection().compareTo(other.getDirection()); + if (c != 0) { + return c; + } + ValueBoundaryDef vb = (ValueBoundaryDef) other; + return getAmt() - vb.getAmt(); + } + + public PTFExpressionDef getExpressionDef() { + return expressionDef; + } + + public void setExpressionDef(PTFExpressionDef expressionDef) { + this.expressionDef = expressionDef; + } + + public ExprNodeDesc getExprNode() { + return expressionDef == null ? null : expressionDef.getExprNode(); + } + + public ExprNodeEvaluator getExprEvaluator() { + return expressionDef == null ? null : expressionDef.getExprEvaluator(); + } + + public ObjectInspector getOI() { + return expressionDef == null ? null : expressionDef.getOI(); + } + + @Override + public int getAmt() { + return amt; + } + + public void setAmt(int amt) { + this.amt = amt; + } +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowExpressionDef.java (working copy) @@ -0,0 +1,20 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + + +public class WindowExpressionDef extends PTFExpressionDef { + private String alias; + + public WindowExpressionDef() {} + + public WindowExpressionDef(PTFExpressionDef eDef) { + super(eDef); + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java (working copy) @@ -0,0 +1,23 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + + +public class WindowFrameDef { + private BoundaryDef start; + private BoundaryDef end; + + public BoundaryDef getStart() { + return start; + } + + public void setStart(BoundaryDef start) { + this.start = start; + } + + public BoundaryDef getEnd() { + return end; + } + + public void setEnd(BoundaryDef end) { + this.end = end; + } +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFunctionDef.java (working copy) @@ -0,0 +1,78 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; + +public class WindowFunctionDef extends WindowExpressionDef { + String name; + boolean isStar; + boolean isDistinct; + List args; + WindowFrameDef windowFrame; + GenericUDAFEvaluator wFnEval; + boolean pivotResult; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public boolean isStar() { + return isStar; + } + + public void setStar(boolean isStar) { + this.isStar = isStar; + } + + public boolean isDistinct() { + return isDistinct; + } + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public List getArgs() { + return args; + } + + public void setArgs(List args) { + this.args = args; + } + + public void addArg(PTFExpressionDef arg) { + args = args == null ? new ArrayList() : args; + args.add(arg); + } + + public WindowFrameDef getWindowFrame() { + return windowFrame; + } + + public void setWindowFrame(WindowFrameDef windowFrame) { + this.windowFrame = windowFrame; + } + + public GenericUDAFEvaluator getWFnEval() { + return wFnEval; + } + + public void setWFnEval(GenericUDAFEvaluator wFnEval) { + this.wFnEval = wFnEval; + } + + public boolean isPivotResult() { + return pivotResult; + } + + public void setPivotResult(boolean pivotResult) { + this.pivotResult = pivotResult; + } + +} \ No newline at end of file Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowTableFunctionDef.java (working copy) @@ -0,0 +1,15 @@ +package org.apache.hadoop.hive.ql.plan.ptf; + +import java.util.List; + + +public class WindowTableFunctionDef extends PartitionedTableFunctionDef { + List windowFunctions; + + public List getWindowFunctions() { + return windowFunctions; + } + public void setWindowFunctions(List windowFunctions) { + this.windowFunctions = windowFunctions; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java (revision 1527202) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NPath.java (working copy) @@ -40,9 +40,9 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFInputDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFInputDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; @@ -101,7 +101,7 @@ { int sz = syFnRes.nextRow - (pItr.getIndex() - 1); Object selectListInput = NPath.getSelectListInput(iRow, - tDef.getInput().getOutputShape().getOI(), pItr, sz); + tableDef.getInput().getOutputShape().getOI(), pItr, sz); ArrayList oRow = new ArrayList(); for(ExprNodeEvaluator resExprEval : resultExprInfo.resultExprEvals) { @@ -161,7 +161,7 @@ NPath evaluator = (NPath) getEvaluator(); PartitionedTableFunctionDef tDef = evaluator.getTableDef(); - ArrayList args = tDef.getArgs(); + List args = tDef.getArgs(); int argsNum = args == null ? 0 : args.size(); if ( argsNum < 4 ) @@ -199,7 +199,7 @@ * validate and setup patternStr */ private void validateAndSetupPatternStr(NPath evaluator, - ArrayList args) throws SemanticException { + List args) throws SemanticException { PTFExpressionDef symboPatternArg = args.get(0); ObjectInspector symbolPatternArgOI = symboPatternArg.getOI(); @@ -219,7 +219,7 @@ * validate and setup SymbolInfo */ private void validateAndSetupSymbolInfo(NPath evaluator, - ArrayList args, + List args, int argsNum) throws SemanticException { int symbolArgsSz = argsNum - 2; if ( symbolArgsSz % 2 != 0) @@ -263,7 +263,7 @@ * validate and setup resultExprStr */ private void validateAndSetupResultExprStr(NPath evaluator, - ArrayList args, + List args, int argsNum) throws SemanticException { PTFExpressionDef resultExprArg = args.get(argsNum - 1); ObjectInspector resultExprArgOI = resultExprArg.getOI(); @@ -303,7 +303,7 @@ NPath evaluator = (NPath) getEvaluator(); PartitionedTableFunctionDef tDef = evaluator.getTableDef(); - ArrayList args = tDef.getArgs(); + List args = tDef.getArgs(); int argsNum = args.size(); validateAndSetupPatternStr(evaluator, args); Index: ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java (revision 1527202) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/Noop.java (working copy) @@ -18,43 +18,37 @@ package org.apache.hadoop.hive.ql.udf.ptf; -import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.hive.ql.exec.PTFPartition; import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -public class Noop extends TableFunctionEvaluator -{ +public class Noop extends TableFunctionEvaluator { @Override - public PTFPartition execute(PTFPartition iPart) throws HiveException - { + public PTFPartition execute(PTFPartition iPart) throws HiveException { return iPart; } @Override - protected void execute(PTFPartitionIterator pItr, PTFPartition oPart) - { + protected void execute(PTFPartitionIterator pItr, PTFPartition oPart) { throw new UnsupportedOperationException(); } - public static class NoopResolver extends TableFunctionResolver - { + public static class NoopResolver extends TableFunctionResolver { @Override - protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef) - { + protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef) { return new Noop(); } @Override - public void setupOutputOI() throws SemanticException - { + public void setupOutputOI() throws SemanticException { StructObjectInspector OI = getEvaluator().getTableDef().getInput().getOutputShape().getOI(); setOutputOI(OI); } @@ -75,7 +69,7 @@ * Set to null only because carryForwardNames is true. */ @Override - public ArrayList getOutputColumnNames() { + public List getOutputColumnNames() { return null; } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java (revision 1527202) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMap.java (working copy) @@ -24,7 +24,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; public class NoopWithMap extends Noop Index: ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java (revision 1527202) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java (working copy) @@ -24,7 +24,7 @@ import org.apache.hadoop.hive.ql.exec.PTFUtils; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -46,8 +46,7 @@ * * */ -public abstract class TableFunctionEvaluator -{ +public abstract class TableFunctionEvaluator { /* * how is this different from the OutpuShape set on the TableDef. * This is the OI of the object coming out of the PTF. @@ -59,52 +58,45 @@ * same comment as OI applies here. */ transient protected StructObjectInspector rawInputOI; - protected PartitionedTableFunctionDef tDef; + protected PartitionedTableFunctionDef tableDef; protected PTFDesc ptfDesc; boolean transformsRawInput; transient protected PTFPartition outputPartition; - static{ + static { + //TODO is this a bug? The field is not named outputOI it is named OI PTFUtils.makeTransient(TableFunctionEvaluator.class, "outputOI", "rawInputOI"); } - public StructObjectInspector getOutputOI() - { + public StructObjectInspector getOutputOI() { return OI; } - protected void setOutputOI(StructObjectInspector outputOI) - { + protected void setOutputOI(StructObjectInspector outputOI) { OI = outputOI; } - public PartitionedTableFunctionDef getTableDef() - { - return tDef; + public PartitionedTableFunctionDef getTableDef() { + return tableDef; } - public void setTableDef(PartitionedTableFunctionDef tDef) - { - this.tDef = tDef; + public void setTableDef(PartitionedTableFunctionDef tDef) { + this.tableDef = tDef; } - protected PTFDesc getQueryDef() - { + protected PTFDesc getQueryDef() { return ptfDesc; } - protected void setQueryDef(PTFDesc ptfDesc) - { + protected void setQueryDef(PTFDesc ptfDesc) { this.ptfDesc = ptfDesc; } - public StructObjectInspector getRawInputOI() - { + public StructObjectInspector getRawInputOI() { return rawInputOI; } - protected void setRawInputOI(StructObjectInspector rawInputOI) - { + protected void setRawInputOI(StructObjectInspector rawInputOI) { this.rawInputOI = rawInputOI; } @@ -117,17 +109,15 @@ } public PTFPartition execute(PTFPartition iPart) - throws HiveException - { + throws HiveException { PTFPartitionIterator pItr = iPart.iterator(); PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, pItr); if ( outputPartition == null ) { outputPartition = PTFPartition.create(ptfDesc.getCfg(), - tDef.getOutputShape().getSerde(), - OI, tDef.getOutputShape().getOI()); - } - else { + tableDef.getOutputShape().getSerde(), + OI, tableDef.getOutputShape().getOI()); + } else { outputPartition.reset(); } @@ -137,17 +127,14 @@ protected abstract void execute(PTFPartitionIterator pItr, PTFPartition oPart) throws HiveException; - public PTFPartition transformRawInput(PTFPartition iPart) throws HiveException - { - if ( !isTransformsRawInput()) - { - throw new HiveException(String.format("Internal Error: mapExecute called on function (%s)that has no Map Phase", tDef.getName())); + public PTFPartition transformRawInput(PTFPartition iPart) throws HiveException { + if (!isTransformsRawInput()) { + throw new HiveException(String.format("Internal Error: mapExecute called on function (%s)that has no Map Phase", tableDef.getName())); } return _transformRawInput(iPart); } - protected PTFPartition _transformRawInput(PTFPartition iPart) throws HiveException - { + protected PTFPartition _transformRawInput(PTFPartition iPart) throws HiveException { return null; } } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java (revision 1527202) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionResolver.java (working copy) @@ -18,7 +18,7 @@ package org.apache.hadoop.hive.ql.udf.ptf; -import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; @@ -27,7 +27,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFResolver; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; @@ -53,8 +53,7 @@ * */ @SuppressWarnings("deprecation") -public abstract class TableFunctionResolver -{ +public abstract class TableFunctionResolver { TableFunctionEvaluator evaluator; PTFDesc ptfDesc; @@ -65,8 +64,7 @@ * the transformsRawInput boolean. */ public void initialize(HiveConf cfg, PTFDesc ptfDesc, PartitionedTableFunctionDef tDef) - throws SemanticException - { + throws SemanticException { this.ptfDesc = ptfDesc; evaluator = createEvaluator(ptfDesc, tDef); @@ -79,16 +77,14 @@ * called during deserialization of a QueryDef during runtime. */ public void initialize(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef, TableFunctionEvaluator evaluator) - throws HiveException - { + throws HiveException { this.evaluator = evaluator; this.ptfDesc = ptfDesc; evaluator.setTableDef(tDef); evaluator.setQueryDef(ptfDesc); } - public TableFunctionEvaluator getEvaluator() - { + public TableFunctionEvaluator getEvaluator() { return evaluator; } @@ -105,7 +101,7 @@ * A PTF Function must provide the 'external' names of the columns in its Output. * */ - public abstract ArrayList getOutputColumnNames() throws SemanticException; + public abstract List getOutputColumnNames() throws SemanticException; /** @@ -127,10 +123,8 @@ * - subsequent to this call, a call to getRawInputOI call on the {@link TableFunctionEvaluator} must return the OI * of the output of this function. */ - public void setupRawInputOI() throws SemanticException - { - if (!transformsRawInput()) - { + public void setupRawInputOI() throws SemanticException { + if (!transformsRawInput()) { return; } throw new SemanticException( @@ -141,9 +135,8 @@ * A PTF Function must provide the 'external' names of the columns in the transformed Raw Input. * */ - public ArrayList getRawInputColumnNames() throws SemanticException { - if (!transformsRawInput()) - { + public List getRawInputColumnNames() throws SemanticException { + if (!transformsRawInput()) { return null; } throw new SemanticException( @@ -153,10 +146,8 @@ /* * Same responsibility as initializeOI, but for the RawInput. */ - public void initializeRawInputOI() throws HiveException - { - if (!transformsRawInput()) - { + public void initializeRawInputOI() throws HiveException { + if (!transformsRawInput()) { return; } throw new HiveException( @@ -166,21 +157,18 @@ /* * callback method used by subclasses to set the RawInputOI on the Evaluator. */ - protected void setRawInputOI(StructObjectInspector rawInputOI) - { + protected void setRawInputOI(StructObjectInspector rawInputOI) { evaluator.setRawInputOI(rawInputOI); } /* * callback method used by subclasses to set the OutputOI on the Evaluator. */ - protected void setOutputOI(StructObjectInspector outputOI) - { + protected void setOutputOI(StructObjectInspector outputOI) { evaluator.setOutputOI(outputOI); } - public PTFDesc getPtfDesc() - { + public PTFDesc getPtfDesc() { return ptfDesc; } Index: ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java (revision 1527202) +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/WindowingTableFunction.java (working copy) @@ -31,13 +31,13 @@ import org.apache.hadoop.hive.ql.parse.WindowingSpec.BoundarySpec; import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction; import org.apache.hadoop.hive.ql.plan.PTFDesc; -import org.apache.hadoop.hive.ql.plan.PTFDesc.BoundaryDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PTFExpressionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.PartitionedTableFunctionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.ValueBoundaryDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFrameDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFunctionDef; -import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.BoundaryDef; +import org.apache.hadoop.hive.ql.plan.ptf.PTFExpressionDef; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.ValueBoundaryDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowFrameDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowFunctionDef; +import org.apache.hadoop.hive.ql.plan.ptf.WindowTableFunctionDef; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.AggregationBuffer; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;