diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index 3bb8fa9..a80feb9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -116,7 +116,9 @@ import org.apache.hadoop.hive.ql.udf.generic.*; import org.apache.hadoop.hive.ql.udf.ptf.MatchPath.MatchPathResolver; import org.apache.hadoop.hive.ql.udf.ptf.Noop.NoopResolver; +import org.apache.hadoop.hive.ql.udf.ptf.NoopStreaming.NoopStreamingResolver; import org.apache.hadoop.hive.ql.udf.ptf.NoopWithMap.NoopWithMapResolver; +import org.apache.hadoop.hive.ql.udf.ptf.NoopWithMapStreaming.NoopWithMapStreamingResolver; import org.apache.hadoop.hive.ql.udf.ptf.TableFunctionResolver; import org.apache.hadoop.hive.ql.udf.ptf.WindowingTableFunction.WindowingTableFunctionResolver; import org.apache.hadoop.hive.ql.udf.xml.GenericUDFXPath; @@ -170,8 +172,10 @@ public static final String WINDOWING_TABLE_FUNCTION = "windowingtablefunction"; - public static final String NOOP_TABLE_FUNCTION = "noop"; - public static final String NOOP_MAP_TABLE_FUNCTION = "noopwithmap"; + private static final String NOOP_TABLE_FUNCTION = "noop"; + private static final String NOOP_MAP_TABLE_FUNCTION = "noopwithmap"; + private static final String NOOP_STREAMING_TABLE_FUNCTION = "noopstreaming"; + private static final String NOOP_STREAMING_MAP_TABLE_FUNCTION = "noopwithmapstreaming"; static Map windowFunctions = Collections.synchronizedMap(new LinkedHashMap()); @@ -442,6 +446,8 @@ registerTableFunction(NOOP_TABLE_FUNCTION, NoopResolver.class); registerTableFunction(NOOP_MAP_TABLE_FUNCTION, NoopWithMapResolver.class); + registerTableFunction(NOOP_STREAMING_TABLE_FUNCTION, NoopStreamingResolver.class); + registerTableFunction(NOOP_STREAMING_MAP_TABLE_FUNCTION, NoopWithMapStreamingResolver.class); registerTableFunction(WINDOWING_TABLE_FUNCTION, WindowingTableFunctionResolver.class); registerTableFunction("matchpath", MatchPathResolver.class); } @@ -1927,10 +1933,13 @@ public static TableFunctionResolver getWindowingTableFunction() { return getTableFunctionResolver(WINDOWING_TABLE_FUNCTION); } - - public static TableFunctionResolver getNoopTableFunction() - { - return getTableFunctionResolver(NOOP_TABLE_FUNCTION); + + public static boolean isNoopFunction(String fnName) { + fnName = fnName.toLowerCase(); + return fnName.equals(NOOP_MAP_TABLE_FUNCTION) || + fnName.equals(NOOP_STREAMING_MAP_TABLE_FUNCTION) || + fnName.equals(NOOP_TABLE_FUNCTION) || + fnName.equals(NOOP_STREAMING_TABLE_FUNCTION); } public static void registerTableFunction(String name, Class tFnCls) diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java index 4d314b7..7c5659b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java @@ -19,8 +19,6 @@ package org.apache.hadoop.hive.ql.exec; import java.io.Serializable; -import java.util.ArrayDeque; -import java.util.Deque; import java.util.Iterator; import java.util.List; import java.util.Stack; @@ -30,7 +28,6 @@ import org.apache.hadoop.hive.ql.exec.PTFPartition.PTFPartitionIterator; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.PTFDesc; import org.apache.hadoop.hive.ql.plan.PTFDeserializer; import org.apache.hadoop.hive.ql.plan.api.OperatorType; @@ -49,14 +46,18 @@ public class PTFOperator extends Operator implements Serializable { private static final long serialVersionUID = 1L; - PTFPartition inputPart; boolean isMapOperator; transient KeyWrapperFactory keyWrapperFactory; protected transient KeyWrapper currentKeys; protected transient KeyWrapper newKeys; + /* + * for map-side invocation of PTFs, we cannot utilize the currentkeys null check + * to decide on invoking startPartition in streaming mode. Hence this extra flag. + */ + transient boolean firstMapRow; transient HiveConf hiveConf; - + transient PTFInvocation ptfInvocation; /* * 1. Find out if the operator is invoked at Map-Side or Reduce-side @@ -68,12 +69,9 @@ protected void initializeOp(Configuration jobConf) throws HiveException { hiveConf = new HiveConf(jobConf, PTFOperator.class); // if the parent is ExtractOperator, this invocation is from reduce-side - Operator parentOp = getParentOperators().get(0); isMapOperator = conf.isMapSide(); reconstructQueryDef(hiveConf); - inputPart = createFirstPartitionForChain( - inputObjInspectors[0], hiveConf, isMapOperator); if (isMapOperator) { PartitionedTableFunctionDef tDef = conf.getStartOfChain(); @@ -83,6 +81,9 @@ protected void initializeOp(Configuration jobConf) throws HiveException { } setupKeysWrapper(inputObjInspectors[0]); + + ptfInvocation = setupChain(); + firstMapRow = true; super.initializeOp(jobConf); } @@ -90,21 +91,8 @@ protected void initializeOp(Configuration jobConf) throws HiveException { @Override protected void closeOp(boolean abort) throws HiveException { super.closeOp(abort); - if(inputPart.size() != 0){ - if (isMapOperator) { - processMapFunction(); - } else { - processInputPartition(); - } - } - inputPart.close(); - inputPart = null; - - for (PTFInputDef iDef = conf.getFuncDef(); iDef != null; iDef = iDef.getInput()) { - if (iDef instanceof PartitionedTableFunctionDef) { - ((PartitionedTableFunctionDef)iDef).getTFunction().close(); - } - } + ptfInvocation.finishPartition(); + ptfInvocation.close(); } @Override @@ -118,26 +106,28 @@ public void processOp(Object row, int tag) throws HiveException * - reset input Partition * - set currentKey to the newKey if it is null or has changed. */ - newKeys.getNewKey(row, inputPart.getInputOI()); + newKeys.getNewKey(row, inputObjInspectors[0]); boolean keysAreEqual = (currentKeys != null && newKeys != null)? newKeys.equals(currentKeys) : false; if (currentKeys != null && !keysAreEqual) { - processInputPartition(); - inputPart.reset(); + ptfInvocation.finishPartition(); } if (currentKeys == null || !keysAreEqual) { + ptfInvocation.startPartition(); if (currentKeys == null) { currentKeys = newKeys.copyKey(); } else { currentKeys.copyKey(newKeys); } } + } else if ( firstMapRow ) { + ptfInvocation.startPartition(); + firstMapRow = false; } - // add row to current Partition. - inputPart.append(row); + ptfInvocation.processRow(row); } /** @@ -180,28 +170,6 @@ protected void setupKeysWrapper(ObjectInspector inputOI) throws HiveException { newKeys = keyWrapperFactory.getKeyWrapper(); } - protected void processInputPartition() throws HiveException { - Iterator pItr = executeChain(inputPart); - - while (pItr.hasNext()) { - Object oRow = pItr.next(); - forward(oRow, outputObjInspector); - } - } - - protected void processMapFunction() throws HiveException { - PartitionedTableFunctionDef tDef = conf.getStartOfChain(); - - Iterator pItr = tDef.getTFunction().canIterateOutput() ? - tDef.getTFunction().transformRawInputIterator(inputPart.iterator()) : - tDef.getTFunction().transformRawInput(inputPart).iterator(); - - while (pItr.hasNext()) { - Object oRow = pItr.next(); - forward(oRow, outputObjInspector); - } - } - /** * @return the name of the operator */ @@ -219,19 +187,8 @@ static public String getOperatorName() { public OperatorType getType() { return OperatorType.PTF; } - - /** - * For all the table functions to be applied to the input - * hive table or query, push them on a stack. - * For each table function popped out of the stack, - * execute the function on the input partition - * and return an output partition. - * @param part - * @return - * @throws HiveException - */ - private Iterator executeChain(PTFPartition part) - throws HiveException { + + private PTFInvocation setupChain() { Stack fnDefs = new Stack(); PTFInputDef iDef = conf.getFuncDef(); @@ -239,63 +196,17 @@ public OperatorType getType() { fnDefs.push((PartitionedTableFunctionDef) iDef); iDef = ((PartitionedTableFunctionDef) iDef).getInput(); } - - PartitionedTableFunctionDef currFnDef; - int i = fnDefs.size(); - while (i > 1) { - currFnDef = fnDefs.pop(); - part = currFnDef.getTFunction().execute(part); - i--; - } - - currFnDef = fnDefs.pop(); - if (!currFnDef.getTFunction().canIterateOutput()) { - part = currFnDef.getTFunction().execute(part); - return part.iterator(); - } else { - return currFnDef.getTFunction().iterator(part.iterator()); + + PTFInvocation curr = null, first = null; + + while(!fnDefs.isEmpty()) { + PartitionedTableFunctionDef currFn = fnDefs.pop(); + curr = new PTFInvocation(curr, currFn.getTFunction()); + if ( first == null ) { + first = curr; + } } - - } - - - /** - * Create a new Partition. - * A partition has 2 OIs: the OI for the rows being put in and the OI for the rows - * coming out. You specify the output OI by giving the Serde to use to Serialize. - * Typically these 2 OIs are the same; but not always. For the - * first PTF in a chain the OI of the incoming rows is dictated by the Parent Op - * to this PTFOp. The output OI from the Partition is typically LazyBinaryStruct, but - * not always. In the case of Noop/NoopMap we keep the Strcuture the same as - * what is given to us. - *

- * The Partition we want to create here is for feeding the First table function in the chain. - * So for map-side processing use the Serde from the output Shape its InputDef. - * For reduce-side processing use the Serde from its RawInputShape(the shape - * after map-side processing). - * @param oi - * @param hiveConf - * @param isMapSide - * @return - * @throws HiveException - */ - public PTFPartition createFirstPartitionForChain(ObjectInspector oi, - HiveConf hiveConf, boolean isMapSide) throws HiveException { - PartitionedTableFunctionDef tabDef = conf.getStartOfChain(); - TableFunctionEvaluator tEval = tabDef.getTFunction(); - - PTFPartition part = null; - SerDe serde = isMapSide ? tabDef.getInput().getOutputShape().getSerde() : - tabDef.getRawInputShape().getSerde(); - StructObjectInspector outputOI = isMapSide ? tabDef.getInput().getOutputShape().getOI() : - tabDef.getRawInputShape().getOI(); - part = PTFPartition.create(conf.getCfg(), - serde, - (StructObjectInspector) oi, - outputOI); - - return part; - + return first; } public static void connectLeadLagFunctionsToPartition(PTFDesc ptfDesc, @@ -310,5 +221,190 @@ public static void connectLeadLagFunctionsToPartition(PTFDesc ptfDesc, llFn.setpItr(pItr); } } - + + /* + * Responsible for the flow of rows through the PTF Chain. + * An Invocation wraps a TableFunction. + * The PTFOp hands the chain each row through the processRow call. + * It also notifies the chain of when a Partition starts/finishes. + * + * There are several combinations depending + * whether the TableFunction and its successor support Streaming or Batch mode. + * + * Combination 1: Streaming + Streaming + * - Start Partition: invoke startPartition on tabFn. + * - Process Row: invoke process Row on tabFn. + * Any output rows hand to next tabFn in chain or forward to next Operator. + * - Finish Partition: invoke finishPartition on tabFn. + * Any output rows hand to next tabFn in chain or forward to next Operator. + * + * Combination 2: Streaming + Batch + * same as Combination 1 + * + * Combination 3: Batch + Batch + * - Start Partition: create or reset the Input Partition for the tabFn + * caveat is: if prev is also batch and it is not providing an Output Iterator + * then we can just use its Output Partition. + * - Process Row: collect row in Input Partition + * - Finish Partition : invoke evaluate on tabFn on Input Partition + * If function gives an Output Partition: set it on next Invocation's Input Partition + * If function gives an Output Iterator: iterate and call processRow on next Invocation. + * For last Invocation in chain: forward rows to next Operator. + * + * Combination 3: Batch + Stream + * Similar to Combination 3, except Finish Partition behavior slightly different + * - Finish Partition : invoke evaluate on tabFn on Input Partition + * iterate output rows: hand to next tabFn in chain or forward to next Operator. + * + */ + class PTFInvocation { + + PTFInvocation prev; + PTFInvocation next; + TableFunctionEvaluator tabFn; + PTFPartition inputPart; + PTFPartition outputPart; + Iterator outputPartRowsItr; + + public PTFInvocation(PTFInvocation prev, TableFunctionEvaluator tabFn) { + this.prev = prev; + this.tabFn = tabFn; + if ( prev != null ) { + prev.next = this; + } + } + + boolean isOutputIterator() { + return tabFn.canAcceptInputAsStream() || tabFn.canIterateOutput(); + } + + boolean isStreaming() { + return tabFn.canAcceptInputAsStream(); + } + + void startPartition() throws HiveException { + if ( isStreaming() ) { + tabFn.startPartition(); + } else { + if ( prev == null || prev.isOutputIterator() ) { + if ( inputPart == null ) { + createInputPartition(); + } else { + inputPart.reset(); + } + } + } + if ( next != null ) { + next.startPartition(); + } + } + + void processRow(Object row) throws HiveException { + if ( isStreaming() ) { + if ( prev == null ) { + /* + * this is needed because during Translation we are still assuming that rows + * are collected into a PTFPartition. + * @Todo make translation handle the case when the first PTF is Streaming. + */ + row = ObjectInspectorUtils.copyToStandardObject(row, inputObjInspectors[0], + ObjectInspectorCopyOption.WRITABLE); + } + handleOutputRows(tabFn.processRow(row)); + } else { + inputPart.append(row); + } + } + + void handleOutputRows(List outRows) throws HiveException { + if ( outRows != null ) { + for (Object orow : outRows ) { + if ( next != null ) { + next.processRow(orow); + } else { + forward(orow, outputObjInspector); + } + } + } + } + + void finishPartition() throws HiveException { + if ( isStreaming() ) { + handleOutputRows(tabFn.finishPartition()); + } else { + if ( tabFn.canIterateOutput() ) { + outputPartRowsItr = tabFn.iterator(inputPart.iterator()); + } else { + outputPart = tabFn.execute(inputPart); + outputPartRowsItr = outputPart.iterator(); + } + if ( next != null ) { + if (!next.isStreaming() && !isOutputIterator() ) { + next.inputPart = outputPart; + } else { + while(outputPartRowsItr.hasNext() ) { + next.processRow(outputPartRowsItr.next()); + } + } + } + } + + if ( next != null ) { + next.finishPartition(); + } else { + if (!isStreaming() ) { + while(outputPartRowsItr.hasNext() ) { + forward(outputPartRowsItr.next(), outputObjInspector); + } + } + } + } + + /** + * Create a new Partition. + * A partition has 2 OIs: the OI for the rows being put in and the OI for the rows + * coming out. You specify the output OI by giving the Serde to use to Serialize. + * Typically these 2 OIs are the same; but not always. For the + * first PTF in a chain the OI of the incoming rows is dictated by the Parent Op + * to this PTFOp. The output OI from the Partition is typically LazyBinaryStruct, but + * not always. In the case of Noop/NoopMap we keep the Strcuture the same as + * what is given to us. + *

+ * The Partition we want to create here is for feeding the First table function in the chain. + * So for map-side processing use the Serde from the output Shape its InputDef. + * For reduce-side processing use the Serde from its RawInputShape(the shape + * after map-side processing). + * @param oi + * @param hiveConf + * @param isMapSide + * @return + * @throws HiveException + */ + private void createInputPartition() throws HiveException { + PartitionedTableFunctionDef tabDef = tabFn.getTableDef(); + PTFInputDef inputDef = tabDef.getInput(); + ObjectInspector inputOI = conf.getStartOfChain() == tabDef ? + inputObjInspectors[0] : inputDef.getOutputShape().getOI(); + + SerDe serde = conf.isMapSide() ? tabDef.getInput().getOutputShape().getSerde() : + tabDef.getRawInputShape().getSerde(); + StructObjectInspector outputOI = conf.isMapSide() ? tabDef.getInput().getOutputShape().getOI() : + tabDef.getRawInputShape().getOI(); + inputPart = PTFPartition.create(conf.getCfg(), + serde, + (StructObjectInspector) inputOI, + outputOI); + } + + void close() { + if ( inputPart != null ) { + inputPart.close(); + } + tabFn.close(); + if ( next != null ) { + next.close(); + } + } + } + } diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java index 34aebf0..d1d866f 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java @@ -630,9 +630,7 @@ private static void validateValueBoundaryExprType(ObjectInspector OI) private ShapeDetails setupTableFnShape(String fnName, ShapeDetails inpShape, StructObjectInspector OI, List columnNames, RowResolver rr) throws SemanticException { - if (fnName.equals(FunctionRegistry.NOOP_TABLE_FUNCTION) - || fnName.equals( - FunctionRegistry.NOOP_MAP_TABLE_FUNCTION)) { + if (FunctionRegistry.isNoopFunction(fnName)) { return setupShapeForNoop(inpShape, OI, columnNames, rr); } return setupShape(OI, columnNames, rr); @@ -882,8 +880,7 @@ protected static RowResolver buildRowResolverForPTF(String tbFnName, String tabA StructObjectInspector rowObjectInspector, List outputColNames, RowResolver inputRR) throws SemanticException { - if (tbFnName.equals(FunctionRegistry.NOOP_TABLE_FUNCTION) || - tbFnName.equals(FunctionRegistry.NOOP_MAP_TABLE_FUNCTION)) { + if (FunctionRegistry.isNoopFunction(tbFnName)) { return buildRowResolverForNoop(tabAlias, rowObjectInspector, inputRR); } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopStreaming.java ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopStreaming.java new file mode 100644 index 0000000..d50a542 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopStreaming.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.ptf; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.PTFDesc; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; + +public class NoopStreaming extends Noop { + + List rows; + + NoopStreaming() { + rows = new ArrayList(); + } + + public boolean canAcceptInputAsStream() { + return true; + } + + public List processRow(Object row) throws HiveException { + if (!canAcceptInputAsStream() ) { + throw new HiveException(String.format( + "Internal error: PTF %s, doesn't support Streaming", + getClass().getName())); + } + rows.clear(); + rows.add(row); + return rows; + } + + public static class NoopStreamingResolver extends NoopResolver { + + @Override + protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef) { + return new NoopStreaming(); + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMapStreaming.java ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMapStreaming.java new file mode 100644 index 0000000..be1f9ab --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/NoopWithMapStreaming.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.udf.ptf; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.PTFDesc; +import org.apache.hadoop.hive.ql.plan.ptf.PartitionedTableFunctionDef; + +public class NoopWithMapStreaming extends NoopWithMap { + List rows; + + NoopWithMapStreaming() { + rows = new ArrayList(); + } + + public boolean canAcceptInputAsStream() { + return true; + } + + public List processRow(Object row) throws HiveException { + if (!canAcceptInputAsStream() ) { + throw new HiveException(String.format( + "Internal error: PTF %s, doesn't support Streaming", + getClass().getName())); + } + rows.clear(); + rows.add(row); + return rows; + } + + public static class NoopWithMapStreamingResolver extends NoopWithMapResolver { + + @Override + protected TableFunctionEvaluator createEvaluator(PTFDesc ptfDesc, PartitionedTableFunctionDef tDef) { + return new NoopStreaming(); + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java index 1087bbf..8a1e085 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.udf.ptf; import java.util.Iterator; +import java.util.List; import org.apache.hadoop.hive.ql.exec.PTFOperator; import org.apache.hadoop.hive.ql.exec.PTFPartition; @@ -30,6 +31,32 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +/* + * Interface Design: + * A TableFunction provides 2 interfaces of execution 'Batch' and 'Streaming'. + * - In Batch mode the contract is Partition in - Partition Out + * - In Streaming mode the contract is a stream of processRow calls - each of which may return 0 or more rows. + * + * A Partition is not just a batch of rows, it enables more than a single iteration of + * the i/p data: multiple passes, arbitrary access of input rows, relative navigation between + * rows(for e.g. lead/lag fns). Most PTFs will work in batch mode. + * + * The Streaming mode gives up on the capabilities of Partitions for the benefit of smaller footprint, + * and faster processing. Window Function processing is an e.g. of this: when there are only Ranking + * functions each row needs to be accessed once in the order it is provided; hence there is no need + * to hold all input rows in a Partition. The 'pattern' is: any time you want to only enhance/enrich + * an Input Row Streaming mode is the right choice. This is the fundamental difference between Ranking + * fns and UDAFs: Ranking functions keep the original data intact whereas UDAF only return aggregate + * information. + * + * Finally we have provided a 'mixed' mode where a non Streaming TableFunction can provide its output + * as an Iterator. As far as we can tell, this is a special case for Windowing handling. If Windowing + * is the only or last TableFunction in a chain, it makes no sense to collect the output rows into a + * output Partition. We justify the pollution of the api by the observation that Windowing is a very + * common use case. + * + */ + /** * Based on Hive {@link GenericUDAFEvaluator}. Break up the responsibility of the old AsbtractTableFunction * class into a Resolver and Evaluator. @@ -66,7 +93,6 @@ transient protected PTFPartition outputPartition; static { - //TODO is this a bug? The field is not named outputOI it is named OI PTFUtils.makeTransient(TableFunctionEvaluator.class, "outputOI", "rawInputOI"); } @@ -112,6 +138,9 @@ public void setTransformsRawInput(boolean transformsRawInput) { public PTFPartition execute(PTFPartition iPart) throws HiveException { + if ( ptfDesc.isMapSide() ) { + return transformRawInput(iPart); + } PTFPartitionIterator pItr = iPart.iterator(); PTFOperator.connectLeadLagFunctionsToPartition(ptfDesc, pItr); @@ -129,7 +158,7 @@ public PTFPartition execute(PTFPartition iPart) protected abstract void execute(PTFPartitionIterator pItr, PTFPartition oPart) throws HiveException; - public PTFPartition transformRawInput(PTFPartition iPart) throws HiveException { + protected PTFPartition transformRawInput(PTFPartition iPart) throws HiveException { if (!isTransformsRawInput()) { throw new HiveException(String.format("Internal Error: mapExecute called on function (%s)that has no Map Phase", tableDef.getName())); } @@ -152,6 +181,11 @@ public boolean canIterateOutput() { } public Iterator iterator(PTFPartitionIterator pItr) throws HiveException { + + if ( ptfDesc.isMapSide() ) { + return transformRawInputIterator(pItr); + } + if (!canIterateOutput()) { throw new HiveException( "Internal error: iterator called on a PTF that cannot provide its output as an Iterator"); @@ -161,7 +195,7 @@ public boolean canIterateOutput() { getClass().getName())); } - public Iterator transformRawInputIterator(PTFPartitionIterator pItr) throws HiveException { + protected Iterator transformRawInputIterator(PTFPartitionIterator pItr) throws HiveException { if (!canIterateOutput()) { throw new HiveException( "Internal error: iterator called on a PTF that cannot provide its output as an Iterator"); @@ -170,6 +204,46 @@ public boolean canIterateOutput() { "Internal error: PTF %s, provides no iterator method", getClass().getName())); } + + /* + * A TableFunction may be able to accept its input as a stream. + * In this case the contract is: + * - startPartition must be invoked to give the PTF a chance to initialize stream processing. + * - each input row is passed in via a processRow(or processRows) invocation. processRow + * can return 0 or more o/p rows. + * - finishPartition is invoked to give the PTF a chance to finish processing and return any + * remaining o/p rows. + */ + public boolean canAcceptInputAsStream() { + return false; + } + + public void startPartition() throws HiveException { + if (!canAcceptInputAsStream() ) { + throw new HiveException(String.format( + "Internal error: PTF %s, doesn't support Streaming", + getClass().getName())); + } + } + + public List processRow(Object row) throws HiveException { + if (!canAcceptInputAsStream() ) { + throw new HiveException(String.format( + "Internal error: PTF %s, doesn't support Streaming", + getClass().getName())); + } + + return null; + } + + public List finishPartition() throws HiveException { + if (!canAcceptInputAsStream() ) { + throw new HiveException(String.format( + "Internal error: PTF %s, doesn't support Streaming", + getClass().getName())); + } + return null; + } public void close() { if (outputPartition != null) { diff --git ql/src/test/queries/clientpositive/ptf_streaming.q ql/src/test/queries/clientpositive/ptf_streaming.q new file mode 100644 index 0000000..d36e540 --- /dev/null +++ ql/src/test/queries/clientpositive/ptf_streaming.q @@ -0,0 +1,150 @@ +DROP TABLE part; + +-- data setup +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +); + +LOAD DATA LOCAL INPATH '../../data/files/part_tiny.txt' overwrite into table part; + +create temporary function noopstreaming as 'org.apache.hadoop.hive.ql.udf.ptf.NoopStreaming$NoopStreamingResolver'; + +--1. test1 +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noopstreaming(on part + partition by p_mfgr + order by p_name + ); + + -- 2. testJoinWithNoop +select p_mfgr, p_name, +p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz +from noopstreaming (on (select p1.* from part p1 join part p2 on p1.p_partkey = p2.p_partkey) j +distribute by j.p_mfgr +sort by j.p_name) +; + +-- 7. testJoin +select abc.* +from noopstreaming(on part +partition by p_mfgr +order by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey; + +-- 9. testNoopWithMap +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name, p_size desc) as r +from noopwithmapstreaming(on part +partition by p_mfgr +order by p_name, p_size desc); + +-- 10. testNoopWithMapWithWindowing +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noopwithmapstreaming(on part + partition by p_mfgr + order by p_name); + +-- 12. testFunctionChain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noopstreaming(on noopwithmapstreaming(on noopstreaming(on part +partition by p_mfgr +order by p_mfgr, p_name +))); + +-- 12.1 testFunctionChain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noopstreaming(on noopwithmap(on noopstreaming(on part +partition by p_mfgr +order by p_mfgr, p_name +))); + +-- 12.2 testFunctionChain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on noopwithmapstreaming(on noopstreaming(on part +partition by p_mfgr +order by p_mfgr, p_name +))); + +-- 14. testPTFJoinWithWindowingWithCount +select abc.p_mfgr, abc.p_name, +rank() over (distribute by abc.p_mfgr sort by abc.p_name) as r, +dense_rank() over (distribute by abc.p_mfgr sort by abc.p_name) as dr, +count(abc.p_name) over (distribute by abc.p_mfgr sort by abc.p_name) as cd, +abc.p_retailprice, sum(abc.p_retailprice) over (distribute by abc.p_mfgr sort by abc.p_name rows between unbounded preceding and current row) as s1, +abc.p_size, abc.p_size - lag(abc.p_size,1,abc.p_size) over (distribute by abc.p_mfgr sort by abc.p_name) as deltaSz +from noopstreaming(on part +partition by p_mfgr +order by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +; + +-- 18. testMulti2OperatorsFunctionChainWithMap +select p_mfgr, p_name, +rank() over (partition by p_mfgr,p_name) as r, +dense_rank() over (partition by p_mfgr,p_name) as dr, +p_size, sum(p_size) over (partition by p_mfgr,p_name rows between unbounded preceding and current row) as s1 +from noopstreaming(on + noopwithmap(on + noop(on + noopstreaming(on part + partition by p_mfgr + order by p_mfgr) + ) + partition by p_mfgr,p_name + order by p_mfgr,p_name) + partition by p_mfgr,p_name + order by p_mfgr,p_name) ; + +-- 19. testMulti3OperatorsFunctionChain +select p_mfgr, p_name, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +p_size, sum(p_size) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on + noopstreaming(on + noop(on + noopstreaming(on part + partition by p_mfgr + order by p_mfgr) + ) + partition by p_mfgr,p_name + order by p_mfgr,p_name) + partition by p_mfgr + order by p_mfgr ) ; + +-- 23. testMultiOperatorChainWithDiffPartitionForWindow2 +select p_mfgr, p_name, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +p_size, +sum(p_size) over (partition by p_mfgr order by p_name range between unbounded preceding and current row) as s1, +sum(p_size) over (partition by p_mfgr order by p_name range between unbounded preceding and current row) as s2 +from noopwithmapstreaming(on + noop(on + noopstreaming(on part + partition by p_mfgr, p_name + order by p_mfgr, p_name) + )); diff --git ql/src/test/results/clientpositive/ptf_streaming.q.out ql/src/test/results/clientpositive/ptf_streaming.q.out new file mode 100644 index 0000000..04ceedf --- /dev/null +++ ql/src/test/results/clientpositive/ptf_streaming.q.out @@ -0,0 +1,673 @@ +PREHOOK: query: DROP TABLE part +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE part +POSTHOOK: type: DROPTABLE +PREHOOK: query: -- data setup +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: -- data setup +CREATE TABLE part( + p_partkey INT, + p_name STRING, + p_mfgr STRING, + p_brand STRING, + p_type STRING, + p_size INT, + p_container STRING, + p_retailprice DOUBLE, + p_comment STRING +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@part +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part_tiny.txt' overwrite into table part +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@part +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/part_tiny.txt' overwrite into table part +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@part +PREHOOK: query: create temporary function noopstreaming as 'org.apache.hadoop.hive.ql.udf.ptf.NoopStreaming$NoopStreamingResolver' +PREHOOK: type: CREATEFUNCTION +PREHOOK: Output: database:default +POSTHOOK: query: create temporary function noopstreaming as 'org.apache.hadoop.hive.ql.udf.ptf.NoopStreaming$NoopStreamingResolver' +POSTHOOK: type: CREATEFUNCTION +POSTHOOK: Output: database:default +PREHOOK: query: --1. test1 +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noopstreaming(on part + partition by p_mfgr + order by p_name + ) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: --1. test1 +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noopstreaming(on part + partition by p_mfgr + order by p_name + ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: -- 2. testJoinWithNoop +select p_mfgr, p_name, +p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz +from noopstreaming (on (select p1.* from part p1 join part p2 on p1.p_partkey = p2.p_partkey) j +distribute by j.p_mfgr +sort by j.p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 2. testJoinWithNoop +select p_mfgr, p_name, +p_size, p_size - lag(p_size,1,p_size) over (partition by p_mfgr order by p_name) as deltaSz +from noopstreaming (on (select p1.* from part p1 join part p2 on p1.p_partkey = p2.p_partkey) j +distribute by j.p_mfgr +sort by j.p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 0 +Manufacturer#1 almond antique burnished rose metallic 2 0 +Manufacturer#1 almond antique burnished rose metallic 2 0 +Manufacturer#1 almond antique burnished rose metallic 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 34 32 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 -28 +Manufacturer#1 almond aquamarine burnished black steel 28 22 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 14 +Manufacturer#2 almond antique violet chocolate turquoise 14 0 +Manufacturer#2 almond antique violet turquoise frosted 40 26 +Manufacturer#2 almond aquamarine midnight light salmon 2 -38 +Manufacturer#2 almond aquamarine rose maroon antique 25 23 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 -7 +Manufacturer#3 almond antique chartreuse khaki white 17 0 +Manufacturer#3 almond antique forest lavender goldenrod 14 -3 +Manufacturer#3 almond antique metallic orange dim 19 5 +Manufacturer#3 almond antique misty red olive 1 -18 +Manufacturer#3 almond antique olive coral navajo 45 44 +Manufacturer#4 almond antique gainsboro frosted violet 10 0 +Manufacturer#4 almond antique violet mint lemon 39 29 +Manufacturer#4 almond aquamarine floral ivory bisque 27 -12 +Manufacturer#4 almond aquamarine yellow dodger mint 7 -20 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 +Manufacturer#5 almond antique blue firebrick mint 31 0 +Manufacturer#5 almond antique medium spring khaki 6 -25 +Manufacturer#5 almond antique sky peru orange 2 -4 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 44 +Manufacturer#5 almond azure blanched chiffon midnight 23 -23 +PREHOOK: query: -- 7. testJoin +select abc.* +from noopstreaming(on part +partition by p_mfgr +order by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 7. testJoin +select abc.* +from noopstreaming(on part +partition by p_mfgr +order by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +15103 almond aquamarine dodger light gainsboro Manufacturer#5 Brand#53 ECONOMY BURNISHED STEEL 46 LG PACK 1018.1 packages hinder carefu +17273 almond antique forest lavender goldenrod Manufacturer#3 Brand#35 PROMO ANODIZED TIN 14 JUMBO CASE 1190.27 along the +17927 almond aquamarine yellow dodger mint Manufacturer#4 Brand#41 ECONOMY BRUSHED COPPER 7 SM PKG 1844.92 ites. eve +33357 almond azure aquamarine papaya violet Manufacturer#4 Brand#41 STANDARD ANODIZED TIN 12 WRAP CASE 1290.35 reful +40982 almond antique misty red olive Manufacturer#3 Brand#32 ECONOMY PLATED COPPER 1 LG PKG 1922.98 c foxes can s +42669 almond antique medium spring khaki Manufacturer#5 Brand#51 STANDARD BURNISHED TIN 6 MED CAN 1611.66 sits haggl +45261 almond aquamarine floral ivory bisque Manufacturer#4 Brand#42 SMALL PLATED STEEL 27 WRAP CASE 1206.26 careful +48427 almond antique violet mint lemon Manufacturer#4 Brand#42 PROMO POLISHED STEEL 39 SM CASE 1375.42 hely ironic i +49671 almond antique gainsboro frosted violet Manufacturer#4 Brand#41 SMALL BRUSHED BRASS 10 SM BOX 1620.67 ccounts run quick +65667 almond aquamarine pink moccasin thistle Manufacturer#1 Brand#12 LARGE BURNISHED STEEL 42 JUMBO CASE 1632.66 e across the expr +78486 almond azure blanched chiffon midnight Manufacturer#5 Brand#52 LARGE BRUSHED BRASS 23 MED BAG 1464.48 hely blith +85768 almond antique chartreuse lavender yellow Manufacturer#1 Brand#12 LARGE BRUSHED STEEL 34 SM BAG 1753.76 refull +86428 almond aquamarine burnished black steel Manufacturer#1 Brand#12 STANDARD ANODIZED STEEL 28 WRAP BAG 1414.42 arefully +90681 almond antique chartreuse khaki white Manufacturer#3 Brand#31 MEDIUM BURNISHED TIN 17 SM CASE 1671.68 are slyly after the sl +105685 almond antique violet chocolate turquoise Manufacturer#2 Brand#22 MEDIUM ANODIZED COPPER 14 MED CAN 1690.68 ly pending requ +110592 almond antique salmon chartreuse burlywood Manufacturer#1 Brand#15 PROMO BURNISHED NICKEL 6 JUMBO PKG 1602.59 to the furiously +112398 almond antique metallic orange dim Manufacturer#3 Brand#32 MEDIUM BURNISHED BRASS 19 JUMBO JAR 1410.39 ole car +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +121152 almond antique burnished rose metallic Manufacturer#1 Brand#14 PROMO PLATED TIN 2 JUMBO BOX 1173.15 e pinto beans h +132666 almond aquamarine rose maroon antique Manufacturer#2 Brand#24 SMALL POLISHED NICKEL 25 MED BOX 1698.66 even +144293 almond antique olive coral navajo Manufacturer#3 Brand#34 STANDARD POLISHED STEEL 45 JUMBO CAN 1337.29 ag furiously about +146985 almond aquamarine midnight light salmon Manufacturer#2 Brand#23 MEDIUM BURNISHED COPPER 2 SM CASE 2031.98 s cajole caref +155733 almond antique sky peru orange Manufacturer#5 Brand#53 SMALL PLATED BRASS 2 WRAP DRUM 1788.73 furiously. bra +191709 almond antique violet turquoise frosted Manufacturer#2 Brand#22 ECONOMY POLISHED STEEL 40 MED BOX 1800.7 haggle +192697 almond antique blue firebrick mint Manufacturer#5 Brand#52 MEDIUM BURNISHED TIN 31 LG DRUM 1789.69 ickly ir +195606 almond aquamarine sandy cyan gainsboro Manufacturer#2 Brand#25 STANDARD PLATED TIN 18 SM PKG 1701.6 ic de +PREHOOK: query: -- 9. testNoopWithMap +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name, p_size desc) as r +from noopwithmapstreaming(on part +partition by p_mfgr +order by p_name, p_size desc) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 9. testNoopWithMap +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name, p_size desc) as r +from noopwithmapstreaming(on part +partition by p_mfgr +order by p_name, p_size desc) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 +Manufacturer#1 almond antique burnished rose metallic 2 1 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 +Manufacturer#1 almond aquamarine burnished black steel 28 5 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 +Manufacturer#2 almond antique violet turquoise frosted 40 2 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 +Manufacturer#3 almond antique chartreuse khaki white 17 1 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 +Manufacturer#3 almond antique metallic orange dim 19 3 +Manufacturer#3 almond antique misty red olive 1 4 +Manufacturer#3 almond antique olive coral navajo 45 5 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 +Manufacturer#4 almond antique violet mint lemon 39 2 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 +Manufacturer#5 almond antique blue firebrick mint 31 1 +Manufacturer#5 almond antique medium spring khaki 6 2 +Manufacturer#5 almond antique sky peru orange 2 3 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 +PREHOOK: query: -- 10. testNoopWithMapWithWindowing +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noopwithmapstreaming(on part + partition by p_mfgr + order by p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 10. testNoopWithMapWithWindowing +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noopwithmapstreaming(on part + partition by p_mfgr + order by p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: -- 12. testFunctionChain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noopstreaming(on noopwithmapstreaming(on noopstreaming(on part +partition by p_mfgr +order by p_mfgr, p_name +))) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 12. testFunctionChain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noopstreaming(on noopwithmapstreaming(on noopstreaming(on part +partition by p_mfgr +order by p_mfgr, p_name +))) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: -- 12.1 testFunctionChain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noopstreaming(on noopwithmap(on noopstreaming(on part +partition by p_mfgr +order by p_mfgr, p_name +))) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 12.1 testFunctionChain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noopstreaming(on noopwithmap(on noopstreaming(on part +partition by p_mfgr +order by p_mfgr, p_name +))) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: -- 12.2 testFunctionChain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on noopwithmapstreaming(on noopstreaming(on part +partition by p_mfgr +order by p_mfgr, p_name +))) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 12.2 testFunctionChain +select p_mfgr, p_name, p_size, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +sum(p_retailprice) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on noopwithmapstreaming(on noopstreaming(on part +partition by p_mfgr +order by p_mfgr, p_name +))) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 2 1 1 1173.15 +Manufacturer#1 almond antique burnished rose metallic 2 1 1 2346.3 +Manufacturer#1 almond antique chartreuse lavender yellow 34 3 2 4100.06 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 4 3 5702.650000000001 +Manufacturer#1 almond aquamarine burnished black steel 28 5 4 7117.070000000001 +Manufacturer#1 almond aquamarine pink moccasin thistle 42 6 5 8749.730000000001 +Manufacturer#2 almond antique violet chocolate turquoise 14 1 1 1690.68 +Manufacturer#2 almond antique violet turquoise frosted 40 2 2 3491.38 +Manufacturer#2 almond aquamarine midnight light salmon 2 3 3 5523.360000000001 +Manufacturer#2 almond aquamarine rose maroon antique 25 4 4 7222.02 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 18 5 5 8923.62 +Manufacturer#3 almond antique chartreuse khaki white 17 1 1 1671.68 +Manufacturer#3 almond antique forest lavender goldenrod 14 2 2 2861.95 +Manufacturer#3 almond antique metallic orange dim 19 3 3 4272.34 +Manufacturer#3 almond antique misty red olive 1 4 4 6195.32 +Manufacturer#3 almond antique olive coral navajo 45 5 5 7532.61 +Manufacturer#4 almond antique gainsboro frosted violet 10 1 1 1620.67 +Manufacturer#4 almond antique violet mint lemon 39 2 2 2996.09 +Manufacturer#4 almond aquamarine floral ivory bisque 27 3 3 4202.35 +Manufacturer#4 almond aquamarine yellow dodger mint 7 4 4 6047.27 +Manufacturer#4 almond azure aquamarine papaya violet 12 5 5 7337.620000000001 +Manufacturer#5 almond antique blue firebrick mint 31 1 1 1789.69 +Manufacturer#5 almond antique medium spring khaki 6 2 2 3401.3500000000004 +Manufacturer#5 almond antique sky peru orange 2 3 3 5190.08 +Manufacturer#5 almond aquamarine dodger light gainsboro 46 4 4 6208.18 +Manufacturer#5 almond azure blanched chiffon midnight 23 5 5 7672.66 +PREHOOK: query: -- 14. testPTFJoinWithWindowingWithCount +select abc.p_mfgr, abc.p_name, +rank() over (distribute by abc.p_mfgr sort by abc.p_name) as r, +dense_rank() over (distribute by abc.p_mfgr sort by abc.p_name) as dr, +count(abc.p_name) over (distribute by abc.p_mfgr sort by abc.p_name) as cd, +abc.p_retailprice, sum(abc.p_retailprice) over (distribute by abc.p_mfgr sort by abc.p_name rows between unbounded preceding and current row) as s1, +abc.p_size, abc.p_size - lag(abc.p_size,1,abc.p_size) over (distribute by abc.p_mfgr sort by abc.p_name) as deltaSz +from noopstreaming(on part +partition by p_mfgr +order by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 14. testPTFJoinWithWindowingWithCount +select abc.p_mfgr, abc.p_name, +rank() over (distribute by abc.p_mfgr sort by abc.p_name) as r, +dense_rank() over (distribute by abc.p_mfgr sort by abc.p_name) as dr, +count(abc.p_name) over (distribute by abc.p_mfgr sort by abc.p_name) as cd, +abc.p_retailprice, sum(abc.p_retailprice) over (distribute by abc.p_mfgr sort by abc.p_name rows between unbounded preceding and current row) as s1, +abc.p_size, abc.p_size - lag(abc.p_size,1,abc.p_size) over (distribute by abc.p_mfgr sort by abc.p_name) as deltaSz +from noopstreaming(on part +partition by p_mfgr +order by p_name +) abc join part p1 on abc.p_partkey = p1.p_partkey +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 1 1 4 1173.15 1173.15 2 0 +Manufacturer#1 almond antique burnished rose metallic 1 1 4 1173.15 2346.3 2 0 +Manufacturer#1 almond antique burnished rose metallic 1 1 4 1173.15 3519.4500000000003 2 0 +Manufacturer#1 almond antique burnished rose metallic 1 1 4 1173.15 4692.6 2 0 +Manufacturer#1 almond antique chartreuse lavender yellow 5 2 5 1753.76 6446.360000000001 34 32 +Manufacturer#1 almond antique salmon chartreuse burlywood 6 3 6 1602.59 8048.950000000001 6 -28 +Manufacturer#1 almond aquamarine burnished black steel 7 4 7 1414.42 9463.37 28 22 +Manufacturer#1 almond aquamarine pink moccasin thistle 8 5 8 1632.66 11096.03 42 14 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 1 1690.68 1690.68 14 0 +Manufacturer#2 almond antique violet turquoise frosted 2 2 2 1800.7 3491.38 40 26 +Manufacturer#2 almond aquamarine midnight light salmon 3 3 3 2031.98 5523.360000000001 2 -38 +Manufacturer#2 almond aquamarine rose maroon antique 4 4 4 1698.66 7222.02 25 23 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 5 5 5 1701.6 8923.62 18 -7 +Manufacturer#3 almond antique chartreuse khaki white 1 1 1 1671.68 1671.68 17 0 +Manufacturer#3 almond antique forest lavender goldenrod 2 2 2 1190.27 2861.95 14 -3 +Manufacturer#3 almond antique metallic orange dim 3 3 3 1410.39 4272.34 19 5 +Manufacturer#3 almond antique misty red olive 4 4 4 1922.98 6195.32 1 -18 +Manufacturer#3 almond antique olive coral navajo 5 5 5 1337.29 7532.61 45 44 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 1 1620.67 1620.67 10 0 +Manufacturer#4 almond antique violet mint lemon 2 2 2 1375.42 2996.09 39 29 +Manufacturer#4 almond aquamarine floral ivory bisque 3 3 3 1206.26 4202.35 27 -12 +Manufacturer#4 almond aquamarine yellow dodger mint 4 4 4 1844.92 6047.27 7 -20 +Manufacturer#4 almond azure aquamarine papaya violet 5 5 5 1290.35 7337.620000000001 12 5 +Manufacturer#5 almond antique blue firebrick mint 1 1 1 1789.69 1789.69 31 0 +Manufacturer#5 almond antique medium spring khaki 2 2 2 1611.66 3401.3500000000004 6 -25 +Manufacturer#5 almond antique sky peru orange 3 3 3 1788.73 5190.08 2 -4 +Manufacturer#5 almond aquamarine dodger light gainsboro 4 4 4 1018.1 6208.18 46 44 +Manufacturer#5 almond azure blanched chiffon midnight 5 5 5 1464.48 7672.66 23 -23 +PREHOOK: query: -- 18. testMulti2OperatorsFunctionChainWithMap +select p_mfgr, p_name, +rank() over (partition by p_mfgr,p_name) as r, +dense_rank() over (partition by p_mfgr,p_name) as dr, +p_size, sum(p_size) over (partition by p_mfgr,p_name rows between unbounded preceding and current row) as s1 +from noopstreaming(on + noopwithmap(on + noop(on + noopstreaming(on part + partition by p_mfgr + order by p_mfgr) + ) + partition by p_mfgr,p_name + order by p_mfgr,p_name) + partition by p_mfgr,p_name + order by p_mfgr,p_name) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 18. testMulti2OperatorsFunctionChainWithMap +select p_mfgr, p_name, +rank() over (partition by p_mfgr,p_name) as r, +dense_rank() over (partition by p_mfgr,p_name) as dr, +p_size, sum(p_size) over (partition by p_mfgr,p_name rows between unbounded preceding and current row) as s1 +from noopstreaming(on + noopwithmap(on + noop(on + noopstreaming(on part + partition by p_mfgr + order by p_mfgr) + ) + partition by p_mfgr,p_name + order by p_mfgr,p_name) + partition by p_mfgr,p_name + order by p_mfgr,p_name) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 1 1 2 2 +Manufacturer#1 almond antique burnished rose metallic 1 1 2 4 +Manufacturer#1 almond antique chartreuse lavender yellow 1 1 34 34 +Manufacturer#1 almond antique salmon chartreuse burlywood 1 1 6 6 +Manufacturer#1 almond aquamarine burnished black steel 1 1 28 28 +Manufacturer#1 almond aquamarine pink moccasin thistle 1 1 42 42 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 14 14 +Manufacturer#2 almond antique violet turquoise frosted 1 1 40 40 +Manufacturer#2 almond aquamarine midnight light salmon 1 1 2 2 +Manufacturer#2 almond aquamarine rose maroon antique 1 1 25 25 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 1 1 18 18 +Manufacturer#3 almond antique chartreuse khaki white 1 1 17 17 +Manufacturer#3 almond antique forest lavender goldenrod 1 1 14 14 +Manufacturer#3 almond antique metallic orange dim 1 1 19 19 +Manufacturer#3 almond antique misty red olive 1 1 1 1 +Manufacturer#3 almond antique olive coral navajo 1 1 45 45 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 10 10 +Manufacturer#4 almond antique violet mint lemon 1 1 39 39 +Manufacturer#4 almond aquamarine floral ivory bisque 1 1 27 27 +Manufacturer#4 almond aquamarine yellow dodger mint 1 1 7 7 +Manufacturer#4 almond azure aquamarine papaya violet 1 1 12 12 +Manufacturer#5 almond antique blue firebrick mint 1 1 31 31 +Manufacturer#5 almond antique medium spring khaki 1 1 6 6 +Manufacturer#5 almond antique sky peru orange 1 1 2 2 +Manufacturer#5 almond aquamarine dodger light gainsboro 1 1 46 46 +Manufacturer#5 almond azure blanched chiffon midnight 1 1 23 23 +PREHOOK: query: -- 19. testMulti3OperatorsFunctionChain +select p_mfgr, p_name, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +p_size, sum(p_size) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on + noopstreaming(on + noop(on + noopstreaming(on part + partition by p_mfgr + order by p_mfgr) + ) + partition by p_mfgr,p_name + order by p_mfgr,p_name) + partition by p_mfgr + order by p_mfgr ) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 19. testMulti3OperatorsFunctionChain +select p_mfgr, p_name, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +p_size, sum(p_size) over (partition by p_mfgr order by p_name rows between unbounded preceding and current row) as s1 +from noop(on + noopstreaming(on + noop(on + noopstreaming(on part + partition by p_mfgr + order by p_mfgr) + ) + partition by p_mfgr,p_name + order by p_mfgr,p_name) + partition by p_mfgr + order by p_mfgr ) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 1 1 2 2 +Manufacturer#1 almond antique burnished rose metallic 1 1 2 4 +Manufacturer#1 almond antique chartreuse lavender yellow 3 2 34 38 +Manufacturer#1 almond antique salmon chartreuse burlywood 4 3 6 44 +Manufacturer#1 almond aquamarine burnished black steel 5 4 28 72 +Manufacturer#1 almond aquamarine pink moccasin thistle 6 5 42 114 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 14 14 +Manufacturer#2 almond antique violet turquoise frosted 2 2 40 54 +Manufacturer#2 almond aquamarine midnight light salmon 3 3 2 56 +Manufacturer#2 almond aquamarine rose maroon antique 4 4 25 81 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 5 5 18 99 +Manufacturer#3 almond antique chartreuse khaki white 1 1 17 17 +Manufacturer#3 almond antique forest lavender goldenrod 2 2 14 31 +Manufacturer#3 almond antique metallic orange dim 3 3 19 50 +Manufacturer#3 almond antique misty red olive 4 4 1 51 +Manufacturer#3 almond antique olive coral navajo 5 5 45 96 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 10 10 +Manufacturer#4 almond antique violet mint lemon 2 2 39 49 +Manufacturer#4 almond aquamarine floral ivory bisque 3 3 27 76 +Manufacturer#4 almond aquamarine yellow dodger mint 4 4 7 83 +Manufacturer#4 almond azure aquamarine papaya violet 5 5 12 95 +Manufacturer#5 almond antique blue firebrick mint 1 1 31 31 +Manufacturer#5 almond antique medium spring khaki 2 2 6 37 +Manufacturer#5 almond antique sky peru orange 3 3 2 39 +Manufacturer#5 almond aquamarine dodger light gainsboro 4 4 46 85 +Manufacturer#5 almond azure blanched chiffon midnight 5 5 23 108 +PREHOOK: query: -- 23. testMultiOperatorChainWithDiffPartitionForWindow2 +select p_mfgr, p_name, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +p_size, +sum(p_size) over (partition by p_mfgr order by p_name range between unbounded preceding and current row) as s1, +sum(p_size) over (partition by p_mfgr order by p_name range between unbounded preceding and current row) as s2 +from noopwithmapstreaming(on + noop(on + noopstreaming(on part + partition by p_mfgr, p_name + order by p_mfgr, p_name) + )) +PREHOOK: type: QUERY +PREHOOK: Input: default@part +#### A masked pattern was here #### +POSTHOOK: query: -- 23. testMultiOperatorChainWithDiffPartitionForWindow2 +select p_mfgr, p_name, +rank() over (partition by p_mfgr order by p_name) as r, +dense_rank() over (partition by p_mfgr order by p_name) as dr, +p_size, +sum(p_size) over (partition by p_mfgr order by p_name range between unbounded preceding and current row) as s1, +sum(p_size) over (partition by p_mfgr order by p_name range between unbounded preceding and current row) as s2 +from noopwithmapstreaming(on + noop(on + noopstreaming(on part + partition by p_mfgr, p_name + order by p_mfgr, p_name) + )) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@part +#### A masked pattern was here #### +Manufacturer#1 almond antique burnished rose metallic 1 1 2 4 4 +Manufacturer#1 almond antique burnished rose metallic 1 1 2 4 4 +Manufacturer#1 almond antique chartreuse lavender yellow 3 2 34 38 38 +Manufacturer#1 almond antique salmon chartreuse burlywood 4 3 6 44 44 +Manufacturer#1 almond aquamarine burnished black steel 5 4 28 72 72 +Manufacturer#1 almond aquamarine pink moccasin thistle 6 5 42 114 114 +Manufacturer#2 almond antique violet chocolate turquoise 1 1 14 14 14 +Manufacturer#2 almond antique violet turquoise frosted 2 2 40 54 54 +Manufacturer#2 almond aquamarine midnight light salmon 3 3 2 56 56 +Manufacturer#2 almond aquamarine rose maroon antique 4 4 25 81 81 +Manufacturer#2 almond aquamarine sandy cyan gainsboro 5 5 18 99 99 +Manufacturer#3 almond antique chartreuse khaki white 1 1 17 17 17 +Manufacturer#3 almond antique forest lavender goldenrod 2 2 14 31 31 +Manufacturer#3 almond antique metallic orange dim 3 3 19 50 50 +Manufacturer#3 almond antique misty red olive 4 4 1 51 51 +Manufacturer#3 almond antique olive coral navajo 5 5 45 96 96 +Manufacturer#4 almond antique gainsboro frosted violet 1 1 10 10 10 +Manufacturer#4 almond antique violet mint lemon 2 2 39 49 49 +Manufacturer#4 almond aquamarine floral ivory bisque 3 3 27 76 76 +Manufacturer#4 almond aquamarine yellow dodger mint 4 4 7 83 83 +Manufacturer#4 almond azure aquamarine papaya violet 5 5 12 95 95 +Manufacturer#5 almond antique blue firebrick mint 1 1 31 31 31 +Manufacturer#5 almond antique medium spring khaki 2 2 6 37 37 +Manufacturer#5 almond antique sky peru orange 3 3 2 39 39 +Manufacturer#5 almond aquamarine dodger light gainsboro 4 4 46 85 85 +Manufacturer#5 almond azure blanched chiffon midnight 5 5 23 108 108 diff --git ql/src/test/results/clientpositive/show_functions.q.out ql/src/test/results/clientpositive/show_functions.q.out index 77a05b2..775cc74 100644 --- ql/src/test/results/clientpositive/show_functions.q.out +++ ql/src/test/results/clientpositive/show_functions.q.out @@ -118,7 +118,9 @@ named_struct negative ngrams noop +noopstreaming noopwithmap +noopwithmapstreaming not ntile nvl