diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index acee377..de33830 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFAvgDecimal; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFCount; +import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFCountMerge; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFCountStar; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFSumDecimal; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUDAFAvgDouble; @@ -1898,7 +1899,7 @@ static String getUndecoratedName(String hiveTypeName) { add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.DECIMAL, null, VectorUDAFMaxDecimal.class)); add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.NONE, GroupByDesc.Mode.HASH, VectorUDAFCountStar.class)); add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFCount.class)); - add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.MERGEPARTIAL, VectorUDAFSumLong.class)); + add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.MERGEPARTIAL, VectorUDAFCountMerge.class)); add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFCount.class)); add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFCount.class)); add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH, VectorUDAFCount.class)); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java index 5aa4d4c..4e43905 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCount.java @@ -46,15 +46,7 @@ private static final long serialVersionUID = 1L; - transient private long value; - transient private boolean isNull; - - public void initIfNull() { - if (isNull) { - isNull = false; - value = 0; - } - } + transient private long count; @Override public int getVariableSize() { @@ -63,8 +55,7 @@ public int getVariableSize() { @Override public void reset() { - isNull = true; - value = 0L; + count = 0L; } } @@ -131,8 +122,7 @@ private void iterateNoNullsWithAggregationSelection( aggregationBufferSets, aggregateIndex, i); - myagg.initIfNull(); - myagg.value++; + myagg.count++; } } @@ -148,8 +138,7 @@ private void iterateHasNullsWithAggregationSelection( aggregationBufferSets, aggregateIndex, i); - myagg.initIfNull(); - myagg.value++; + myagg.count++; } } } @@ -168,8 +157,7 @@ private void iterateHasNullsSelectionWithAggregationSelection( aggregationBufferSets, aggregateIndex, j); - myagg.initIfNull(); - myagg.value++; + myagg.count++; } } } @@ -191,17 +179,15 @@ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) Aggregation myagg = (Aggregation)agg; - myagg.initIfNull(); - if (inputVector.isRepeating) { if (inputVector.noNulls || !inputVector.isNull[0]) { - myagg.value += batchSize; + myagg.count += batchSize; } return; } if (inputVector.noNulls) { - myagg.value += batchSize; + myagg.count += batchSize; return; } else if (!batch.selectedInUse) { @@ -221,7 +207,7 @@ private void iterateSelectionHasNulls( for (int j=0; j< batchSize; ++j) { int i = selected[j]; if (!isNull[i]) { - myagg.value += 1; + myagg.count += 1; } } } @@ -233,7 +219,7 @@ private void iterateNoSelectionHasNulls( for (int i=0; i< batchSize; ++i) { if (!isNull[i]) { - myagg.value += 1; + myagg.count += 1; } } } @@ -251,14 +237,9 @@ public void reset(AggregationBuffer agg) throws HiveException { @Override public Object evaluateOutput(AggregationBuffer agg) throws HiveException { - Aggregation myagg = (Aggregation) agg; - if (myagg.isNull) { - return null; - } - else { - result.set (myagg.value); + Aggregation myagg = (Aggregation) agg; + result.set (myagg.count); return result; - } } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java new file mode 100644 index 0000000..7dabbd8 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFCountMerge.java @@ -0,0 +1,400 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorAggregationBufferRow; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.AggregationDesc; +import org.apache.hadoop.hive.ql.util.JavaDataModel; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.LongWritable; + + +/** + * VectorUDAFCountMerge. Vectorized implementation for COUNT aggregate on reduce-side (merge). + */ +@Description(name = "count", value = "_FUNC_(expr) - Returns the merged sum value of expr (vectorized, type: long)") + +public class VectorUDAFCountMerge extends VectorAggregateExpression { + + private static final long serialVersionUID = 1L; + + /** + * class for storing the current aggregate value. + */ + static class Aggregation implements AggregationBuffer { + + private static final long serialVersionUID = 1L; + + transient private long value; + + @Override + public int getVariableSize() { + throw new UnsupportedOperationException(); + } + + @Override + public void reset() { + value = 0L; + } + } + + private VectorExpression inputExpression = null; + transient private final LongWritable result; + + public VectorUDAFCountMerge(VectorExpression inputExpression) { + this(); + this.inputExpression = inputExpression; + } + + public VectorUDAFCountMerge() { + super(); + result = new LongWritable(0); + } + + private Aggregation getCurrentAggregationBuffer( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + int row) { + VectorAggregationBufferRow mySet = aggregationBufferSets[row]; + Aggregation myagg = (Aggregation) mySet.getAggregationBuffer(aggregateIndex); + return myagg; + } + + @Override + public void aggregateInputSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + VectorizedRowBatch batch) throws HiveException { + + int batchSize = batch.size; + + if (batchSize == 0) { + return; + } + + inputExpression.evaluate(batch); + + LongColumnVector inputVector = (LongColumnVector)batch. + cols[this.inputExpression.getOutputColumn()]; + long[] vector = inputVector.vector; + + if (inputVector.noNulls) { + if (inputVector.isRepeating) { + iterateNoNullsRepeatingWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + vector[0], batchSize); + } else { + if (batch.selectedInUse) { + iterateNoNullsSelectionWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + vector, batch.selected, batchSize); + } else { + iterateNoNullsWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + vector, batchSize); + } + } + } else { + if (inputVector.isRepeating) { + if (batch.selectedInUse) { + iterateHasNullsRepeatingSelectionWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + vector[0], batchSize, batch.selected, inputVector.isNull); + } else { + iterateHasNullsRepeatingWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + vector[0], batchSize, inputVector.isNull); + } + } else { + if (batch.selectedInUse) { + iterateHasNullsSelectionWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + vector, batchSize, batch.selected, inputVector.isNull); + } else { + iterateHasNullsWithAggregationSelection( + aggregationBufferSets, aggregateIndex, + vector, batchSize, inputVector.isNull); + } + } + } + } + + private void iterateNoNullsRepeatingWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + long value, + int batchSize) { + + for (int i=0; i < batchSize; ++i) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + myagg.value += value; + } + } + + private void iterateNoNullsSelectionWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + long[] values, + int[] selection, + int batchSize) { + + for (int i=0; i < batchSize; ++i) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + myagg.value += values[selection[i]]; + } + } + + private void iterateNoNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + long[] values, + int batchSize) { + for (int i=0; i < batchSize; ++i) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + myagg.value += values[i]; + } + } + + private void iterateHasNullsRepeatingSelectionWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + long value, + int batchSize, + int[] selection, + boolean[] isNull) { + + for (int i=0; i < batchSize; ++i) { + if (!isNull[selection[i]]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + myagg.value += value; + } + } + + } + + private void iterateHasNullsRepeatingWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + long value, + int batchSize, + boolean[] isNull) { + + for (int i=0; i < batchSize; ++i) { + if (!isNull[i]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + myagg.value += value; + } + } + } + + private void iterateHasNullsSelectionWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + long[] values, + int batchSize, + int[] selection, + boolean[] isNull) { + + for (int j=0; j < batchSize; ++j) { + int i = selection[j]; + if (!isNull[i]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + j); + myagg.value += values[i]; + } + } + } + + private void iterateHasNullsWithAggregationSelection( + VectorAggregationBufferRow[] aggregationBufferSets, + int aggregateIndex, + long[] values, + int batchSize, + boolean[] isNull) { + + for (int i=0; i < batchSize; ++i) { + if (!isNull[i]) { + Aggregation myagg = getCurrentAggregationBuffer( + aggregationBufferSets, + aggregateIndex, + i); + myagg.value += values[i]; + } + } + } + + @Override + public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) + throws HiveException { + + inputExpression.evaluate(batch); + + LongColumnVector inputVector = (LongColumnVector)batch. + cols[this.inputExpression.getOutputColumn()]; + + int batchSize = batch.size; + + if (batchSize == 0) { + return; + } + + Aggregation myagg = (Aggregation)agg; + + long[] vector = inputVector.vector; + + if (inputVector.isRepeating) { + if (inputVector.noNulls) { + myagg.value += vector[0]*batchSize; + } + return; + } + + if (!batch.selectedInUse && inputVector.noNulls) { + iterateNoSelectionNoNulls(myagg, vector, batchSize); + } + else if (!batch.selectedInUse) { + iterateNoSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull); + } + else if (inputVector.noNulls){ + iterateSelectionNoNulls(myagg, vector, batchSize, batch.selected); + } + else { + iterateSelectionHasNulls(myagg, vector, batchSize, inputVector.isNull, batch.selected); + } + } + + private void iterateSelectionHasNulls( + Aggregation myagg, + long[] vector, + int batchSize, + boolean[] isNull, + int[] selected) { + + for (int j=0; j< batchSize; ++j) { + int i = selected[j]; + if (!isNull[i]) { + myagg.value += vector[i]; + } + } + } + + private void iterateSelectionNoNulls( + Aggregation myagg, + long[] vector, + int batchSize, + int[] selected) { + + for (int i=0; i< batchSize; ++i) { + myagg.value += vector[selected[i]]; + } + } + + private void iterateNoSelectionHasNulls( + Aggregation myagg, + long[] vector, + int batchSize, + boolean[] isNull) { + + for(int i=0;i values, + Object expected) throws HiveException { + + @SuppressWarnings("unchecked") + FakeVectorRowBatchFromLongIterables fdr = new FakeVectorRowBatchFromLongIterables(batchSize, + values); + testAggregateCountReduceIterable (fdr, expected); + } + public static interface Validator { void validate (String key, Object expected, Object result); @@ -2223,6 +2259,37 @@ public void testAggregateCountStarIterable ( validator.validate("_total", expected, result); } + public void testAggregateCountReduceIterable ( + Iterable data, + Object expected) throws HiveException { + Map mapColumnNames = new HashMap(); + mapColumnNames.put("A", 0); + VectorizationContext ctx = new VectorizationContext(mapColumnNames, 1); + + GroupByDesc desc = buildGroupByDescType(ctx, "count", "A", TypeInfoFactory.longTypeInfo); + VectorGroupByDesc vectorDesc = desc.getVectorDesc(); + vectorDesc.setIsReduce(true); + + VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc); + + FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(vgo); + vgo.initialize(null, null); + + for (VectorizedRowBatch unit: data) { + vgo.processOp(unit, 0); + } + vgo.close(false); + + List outBatchList = out.getCapturedRows(); + assertNotNull(outBatchList); + assertEquals(1, outBatchList.size()); + + Object result = outBatchList.get(0); + + Validator validator = getValidator("count"); + validator.validate("_total", expected, result); + } + public void testAggregateStringIterable ( String aggregateName, Iterable data, diff --git ql/src/test/queries/clientpositive/vectorization_short_regress.q ql/src/test/queries/clientpositive/vectorization_short_regress.q index 638a31f..d1194cc 100644 --- ql/src/test/queries/clientpositive/vectorization_short_regress.q +++ ql/src/test/queries/clientpositive/vectorization_short_regress.q @@ -850,3 +850,52 @@ WHERE (((cboolean1 IS NOT NULL)) GROUP BY cboolean1 ORDER BY cboolean1; +-- These tests verify COUNT on empty or null colulmns work correctly. +create table test_count(i int) stored as orc; + +explain +select count(*) from test_count; + +select count(*) from test_count; + +explain +select count(i) from test_count; + +select count(i) from test_count; + +create table alltypesnull like alltypesorc; +alter table alltypesnull set fileformat textfile; + +insert into table alltypesnull select null, null, null, null, null, null, null, null, null, null, null, null from alltypesorc; + +create table alltypesnullorc stored as orc as select * from alltypesnull; + +explain +select count(*) from alltypesnullorc; + +select count(*) from alltypesnullorc; + +explain +select count(ctinyint) from alltypesnullorc; + +select count(ctinyint) from alltypesnullorc; + +explain +select count(cint) from alltypesnullorc; + +select count(cint) from alltypesnullorc; + +explain +select count(cfloat) from alltypesnullorc; + +select count(cfloat) from alltypesnullorc; + +explain +select count(cstring1) from alltypesnullorc; + +select count(cstring1) from alltypesnullorc; + +explain +select count(cboolean1) from alltypesnullorc; + +select count(cboolean1) from alltypesnullorc; diff --git ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out index 9fcb4f4..3c1a99f 100644 --- ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out +++ ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out @@ -6618,3 +6618,623 @@ POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### false 11.0 -11.0 -2.389090909090909 -17881597706 -1.7881597716175E10 3.8953387713327066E17 6.0 -0.8249999999999993 -2454.8879999999995 3.8953385925167296E17 -2145884705 1.66288903197104486E18 0.8249999999999993 4.7840233756130287E-17 4.098424268084119E-17 0.8249999999999993 -1051696618 28.692556844886422 2.980633855245E9 -4.032330473245E9 85.79562278396777 4.032330473245E9 -3983699.3106060605 3983699.3106060605 4.1896430920933255E15 true 79.553 -79.553 -0.33034580136836733 -401322621137 -4.01322621147175E11 7.9255373737244976E16 34.727455139160156 -69.3780014038086 4856.6352637899645 7.9254972414623824E16 -2130544867 2.30133924842409523E18 69.3780014038086 3.456813247089758E-17 2.0387240975807185E-18 69.3780014038086 2182477964777 34.654968050508266 2.959326820263E9 2.179518637956737E12 9461.197516216069 -2.179518637956737E12 4.592756659884259E8 -4.592756659884259E8 1.002359020778021E21 +PREHOOK: query: -- These tests verify COUNT on empty or null colulmns work correctly. +create table test_count(i int) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_count +POSTHOOK: query: -- These tests verify COUNT on empty or null colulmns work correctly. +create table test_count(i int) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_count +PREHOOK: query: explain +select count(*) from test_count +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from test_count +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test_count + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from test_count +PREHOOK: type: QUERY +PREHOOK: Input: default@test_count +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from test_count +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_count +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(i) from test_count +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(i) from test_count +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test_count + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: i (type: int) + outputColumnNames: i + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + aggregations: count(i) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(i) from test_count +PREHOOK: type: QUERY +PREHOOK: Input: default@test_count +#### A masked pattern was here #### +POSTHOOK: query: select count(i) from test_count +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_count +#### A masked pattern was here #### +0 +PREHOOK: query: create table alltypesnull like alltypesorc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@alltypesnull +POSTHOOK: query: create table alltypesnull like alltypesorc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alltypesnull +PREHOOK: query: alter table alltypesnull set fileformat textfile +PREHOOK: type: ALTERTABLE_FILEFORMAT +PREHOOK: Input: default@alltypesnull +PREHOOK: Output: default@alltypesnull +POSTHOOK: query: alter table alltypesnull set fileformat textfile +POSTHOOK: type: ALTERTABLE_FILEFORMAT +POSTHOOK: Input: default@alltypesnull +POSTHOOK: Output: default@alltypesnull +PREHOOK: query: insert into table alltypesnull select null, null, null, null, null, null, null, null, null, null, null, null from alltypesorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@alltypesnull +POSTHOOK: query: insert into table alltypesnull select null, null, null, null, null, null, null, null, null, null, null, null from alltypesorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@alltypesnull +POSTHOOK: Lineage: alltypesnull.cbigint EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cboolean1 EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cboolean2 EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cdouble EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cfloat EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cint EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.csmallint EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cstring1 SIMPLE [] +POSTHOOK: Lineage: alltypesnull.cstring2 SIMPLE [] +POSTHOOK: Lineage: alltypesnull.ctimestamp1 EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.ctimestamp2 EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.ctinyint EXPRESSION [] +PREHOOK: query: create table alltypesnullorc stored as orc as select * from alltypesnull +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesnull +PREHOOK: Output: database:default +PREHOOK: Output: default@alltypesnullorc +POSTHOOK: query: create table alltypesnullorc stored as orc as select * from alltypesnull +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesnull +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alltypesnullorc +PREHOOK: query: explain +select count(*) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +12288 +PREHOOK: query: explain +select count(ctinyint) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(ctinyint) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: ctinyint + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(ctinyint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(ctinyint) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(ctinyint) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(cint) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(cint) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: cint + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(cint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(cint) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(cint) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(cfloat) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(cfloat) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cfloat (type: float) + outputColumnNames: cfloat + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(cfloat) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(cfloat) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(cfloat) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(cstring1) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(cstring1) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cstring1 (type: string) + outputColumnNames: cstring1 + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(cstring1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(cstring1) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(cstring1) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(cboolean1) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(cboolean1) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cboolean1 (type: boolean) + outputColumnNames: cboolean1 + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(cboolean1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reducer 2 + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + Execution mode: vectorized + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(cboolean1) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(cboolean1) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0 diff --git ql/src/test/results/clientpositive/vectorization_short_regress.q.out ql/src/test/results/clientpositive/vectorization_short_regress.q.out index 1ca9ab9..8c4d1a4 100644 --- ql/src/test/results/clientpositive/vectorization_short_regress.q.out +++ ql/src/test/results/clientpositive/vectorization_short_regress.q.out @@ -6615,3 +6615,567 @@ POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### false 11.0 -11.0 -2.389090909090909 -17881597706 -1.7881597716175E10 3.8953387713327066E17 6.0 -0.8249999999999993 -2454.8879999999995 3.8953385925167296E17 -2145884705 1.66288903197104486E18 0.8249999999999993 4.7840233756130287E-17 4.098424268084119E-17 0.8249999999999993 -1051696618 28.692556844886422 2.980633855245E9 -4.032330473245E9 85.79562278396777 4.032330473245E9 -3983699.3106060605 3983699.3106060605 4.1896430920933255E15 true 79.553 -79.553 -0.33034580136836733 -401322621137 -4.01322621147175E11 7.9255373737244976E16 34.727455139160156 -69.3780014038086 4856.6352637899645 7.9254972414623824E16 -2130544867 2.30133924842409523E18 69.3780014038086 3.456813247089758E-17 2.0387240975807185E-18 69.3780014038086 2182477964777 34.654968050508266 2.959326820263E9 2.179518637956737E12 9461.197516216069 -2.179518637956737E12 4.592756659884259E8 -4.592756659884259E8 1.002359020778021E21 +PREHOOK: query: -- These tests verify COUNT on empty or null colulmns work correctly. +create table test_count(i int) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test_count +POSTHOOK: query: -- These tests verify COUNT on empty or null colulmns work correctly. +create table test_count(i int) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_count +PREHOOK: query: explain +select count(*) from test_count +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from test_count +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: test_count + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from test_count +PREHOOK: type: QUERY +PREHOOK: Input: default@test_count +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from test_count +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_count +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(i) from test_count +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(i) from test_count +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: test_count + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Select Operator + expressions: i (type: int) + outputColumnNames: i + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Group By Operator + aggregations: count(i) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(i) from test_count +PREHOOK: type: QUERY +PREHOOK: Input: default@test_count +#### A masked pattern was here #### +POSTHOOK: query: select count(i) from test_count +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_count +#### A masked pattern was here #### +0 +PREHOOK: query: create table alltypesnull like alltypesorc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@alltypesnull +POSTHOOK: query: create table alltypesnull like alltypesorc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alltypesnull +PREHOOK: query: alter table alltypesnull set fileformat textfile +PREHOOK: type: ALTERTABLE_FILEFORMAT +PREHOOK: Input: default@alltypesnull +PREHOOK: Output: default@alltypesnull +POSTHOOK: query: alter table alltypesnull set fileformat textfile +POSTHOOK: type: ALTERTABLE_FILEFORMAT +POSTHOOK: Input: default@alltypesnull +POSTHOOK: Output: default@alltypesnull +PREHOOK: query: insert into table alltypesnull select null, null, null, null, null, null, null, null, null, null, null, null from alltypesorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: default@alltypesnull +POSTHOOK: query: insert into table alltypesnull select null, null, null, null, null, null, null, null, null, null, null, null from alltypesorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: default@alltypesnull +POSTHOOK: Lineage: alltypesnull.cbigint EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cboolean1 EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cboolean2 EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cdouble EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cfloat EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cint EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.csmallint EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.cstring1 SIMPLE [] +POSTHOOK: Lineage: alltypesnull.cstring2 SIMPLE [] +POSTHOOK: Lineage: alltypesnull.ctimestamp1 EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.ctimestamp2 EXPRESSION [] +POSTHOOK: Lineage: alltypesnull.ctinyint EXPRESSION [] +PREHOOK: query: create table alltypesnullorc stored as orc as select * from alltypesnull +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesnull +PREHOOK: Output: database:default +PREHOOK: Output: default@alltypesnullorc +POSTHOOK: query: create table alltypesnullorc stored as orc as select * from alltypesnull +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesnull +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alltypesnullorc +PREHOOK: query: explain +select count(*) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(*) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(*) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +12288 +PREHOOK: query: explain +select count(ctinyint) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(ctinyint) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: ctinyint (type: tinyint) + outputColumnNames: ctinyint + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(ctinyint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(ctinyint) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(ctinyint) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(cint) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(cint) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cint (type: int) + outputColumnNames: cint + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(cint) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(cint) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(cint) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(cfloat) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(cfloat) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cfloat (type: float) + outputColumnNames: cfloat + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(cfloat) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(cfloat) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(cfloat) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(cstring1) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(cstring1) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cstring1 (type: string) + outputColumnNames: cstring1 + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(cstring1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(cstring1) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(cstring1) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0 +PREHOOK: query: explain +select count(cboolean1) from alltypesnullorc +PREHOOK: type: QUERY +POSTHOOK: query: explain +select count(cboolean1) from alltypesnullorc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alltypesnullorc + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: cboolean1 (type: boolean) + outputColumnNames: cboolean1 + Statistics: Num rows: 12288 Data size: 168 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: count(cboolean1) + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: vectorized + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select count(cboolean1) from alltypesnullorc +PREHOOK: type: QUERY +PREHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +POSTHOOK: query: select count(cboolean1) from alltypesnullorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alltypesnullorc +#### A masked pattern was here #### +0