diff --git pom.xml pom.xml
index ba87106..394e26b 100644
--- pom.xml
+++ pom.xml
@@ -195,7 +195,7 @@
2.0.0-M5
4.1.17.Final
3.10.5.Final
- 1.10.0
+ 1.11.0
0.16.0
1.5.6
2.5.0
diff --git ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out
index 48e8405..ec2cd8b 100644
--- ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out
+++ ql/src/test/results/clientpositive/llap/vector_partitioned_date_time.q.out
@@ -4077,7 +4077,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: flights_tiny_parquet_partitioned_date
- Statistics: Num rows: 137 Data size: 13861 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 137 Data size: 11618 Basic stats: COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
Select Operator
@@ -4087,7 +4087,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [5]
- Statistics: Num rows: 137 Data size: 13861 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 137 Data size: 11618 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
Group By Vectorization:
@@ -5120,7 +5120,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: flights_tiny_parquet_partitioned_timestamp
- Statistics: Num rows: 137 Data size: 11189 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 137 Data size: 9330 Basic stats: COMPLETE Column stats: COMPLETE
TableScan Vectorization:
native: true
Select Operator
@@ -5130,7 +5130,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [5]
- Statistics: Num rows: 137 Data size: 11189 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 137 Data size: 9330 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
Group By Vectorization:
diff --git ql/src/test/results/clientpositive/parquet_stats.q.out ql/src/test/results/clientpositive/parquet_stats.q.out
index ea15e91..01aa206 100644
--- ql/src/test/results/clientpositive/parquet_stats.q.out
+++ ql/src/test/results/clientpositive/parquet_stats.q.out
@@ -47,8 +47,8 @@ Table Parameters:
bucketing_version 2
numFiles 1
numRows 2
- rawDataSize 146
- totalSize 500
+ rawDataSize 86
+ totalSize 544
#### A masked pattern was here ####
# Storage Information
diff --git ql/src/test/results/clientpositive/spark/parquet_join.q.out ql/src/test/results/clientpositive/spark/parquet_join.q.out
index e86c5f6..cd2131e 100644
--- ql/src/test/results/clientpositive/spark/parquet_join.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_join.q.out
@@ -87,39 +87,39 @@ STAGE PLANS:
TableScan
alias: p1
filterExpr: key is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 73 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 73 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 73 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 2 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 73 Basic stats: COMPLETE Column stats: NONE
Map 3
Map Operator Tree:
TableScan
alias: p2
filterExpr: key is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 199 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 114 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 199 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 114 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), myvalue (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 199 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 114 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 2 Data size: 199 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 114 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
Reducer 2
Reduce Operator Tree:
@@ -130,14 +130,14 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col2
- Statistics: Num rows: 2 Data size: 132 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 132 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 132 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -186,14 +186,14 @@ STAGE PLANS:
TableScan
alias: p1
filterExpr: key is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 73 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 73 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 120 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 73 Basic stats: COMPLETE Column stats: NONE
Spark HashTable Sink Operator
keys:
0 _col0 (type: int)
@@ -210,14 +210,14 @@ STAGE PLANS:
TableScan
alias: p2
filterExpr: key is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 199 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 114 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 199 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 114 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), myvalue (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 199 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 114 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -227,14 +227,14 @@ STAGE PLANS:
outputColumnNames: _col2
input vertices:
0 Map 1
- Statistics: Num rows: 2 Data size: 132 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col2 (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 2 Data size: 132 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 132 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 80 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -321,14 +321,14 @@ STAGE PLANS:
TableScan
alias: p2
filterExpr: key is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 245 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 147 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: key is not null (type: boolean)
- Statistics: Num rows: 2 Data size: 245 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 147 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), value2 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 245 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 147 Basic stats: COMPLETE Column stats: NONE
Sorted Merge Bucket Map Join Operator
condition map:
Inner Join 0 to 1
@@ -336,14 +336,14 @@ STAGE PLANS:
0 _col0 (type: int)
1 _col0 (type: int)
outputColumnNames: _col1, _col3
- Statistics: Num rows: 2 Data size: 158 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 93 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: string), _col3 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 158 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 93 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 158 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 93 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_0.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_0.q.out
index a830f95..013ca68 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_0.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_0.q.out
@@ -84,7 +84,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesparquet
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Select Operator
@@ -94,7 +94,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0]
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(ctinyint)
Group By Vectorization:
@@ -227,11 +227,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesparquet
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col0), count(_col0), sum(_col2), sum(_col1)
minReductionHashAggr: 0.99
@@ -400,7 +400,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesparquet
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Select Operator
@@ -410,7 +410,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [3]
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(cbigint)
Group By Vectorization:
@@ -543,11 +543,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesparquet
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col0), count(_col0), sum(_col2), sum(_col1)
minReductionHashAggr: 0.99
@@ -716,7 +716,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesparquet
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Select Operator
@@ -726,7 +726,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [4]
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(cfloat)
Group By Vectorization:
@@ -859,11 +859,11 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesparquet
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cfloat (type: float), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col0), count(_col0), sum(_col2), sum(_col1)
minReductionHashAggr: 0.99
@@ -1025,7 +1025,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((CAST( cint AS decimal(13,3)) <> 79.553) or (UDFToDouble(cbigint) < cdouble) or (cstring2 like '%b%')) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -1034,7 +1034,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprOrExpr(children: FilterDecimalColNotEqualDecimalScalar(col 13:decimal(13,3), val 79.553)(children: CastLongToDecimal(col 2:int) -> 13:decimal(13,3)), FilterDoubleColLessDoubleColumn(col 14:double, col 5:double)(children: CastLongToDouble(col 3:bigint) -> 14:double), FilterStringColLikeStringScalar(col 7:string, pattern %b%))
predicate: ((CAST( cint AS decimal(13,3)) <> 79.553) or (UDFToDouble(cbigint) < cdouble) or (cstring2 like '%b%')) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint), cfloat (type: float), ctinyint (type: tinyint), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4
@@ -1043,7 +1043,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [3, 4, 0, 15, 18]
selectExpressions: CastLongToDouble(col 3:bigint) -> 15:double, DoubleColMultiplyDoubleColumn(col 16:double, col 17:double)(children: CastLongToDouble(col 3:bigint) -> 16:double, CastLongToDouble(col 3:bigint) -> 17:double) -> 18:double
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col0), count(_col0), sum(_col4), sum(_col3), count(), sum(_col1), min(_col2)
Group By Vectorization:
@@ -1214,14 +1214,14 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((cstring1 like 'a%') or (cstring1 like 'b%') or (cstring1 like 'c%') or ((length(cstring1) < 50) and (cstring1 like '%n') and (length(cstring1) > 0))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: ((cstring1 like 'a%') or (cstring1 like 'b%') or (cstring1 like 'c%') or ((length(cstring1) < 50) and (cstring1 like '%n') and (length(cstring1) > 0))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Select Operator
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
minReductionHashAggr: 0.99
@@ -30033,22 +30033,22 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: (((cint = 49) and (cfloat = 3.5)) or ((cint = 47) and (cfloat = 2.09)) or ((cint = 45) and (cfloat = 3.02))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: (((cint = 49) and (cfloat = 3.5)) or ((cint = 47) and (cfloat = 2.09)) or ((cint = 45) and (cfloat = 3.02))) (type: boolean)
- Statistics: Num rows: 9216 Data size: 445313 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9216 Data size: 445074 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 9216 Data size: 445313 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9216 Data size: 445074 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 9216 Data size: 445313 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9216 Data size: 445074 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -30157,22 +30157,22 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: (struct(cint,cfloat)) IN (const struct(49,3.5), const struct(47,2.09), const struct(45,3.02)) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: (struct(cint,cfloat)) IN (const struct(49,3.5), const struct(47,2.09), const struct(45,3.02)) (type: boolean)
- Statistics: Num rows: 9216 Data size: 445313 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9216 Data size: 445074 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 9216 Data size: 445313 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9216 Data size: 445074 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 9216 Data size: 445313 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9216 Data size: 445074 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -30280,22 +30280,22 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: (((cint = 49) or (cfloat = 3.5)) and ((cint = 47) or (cfloat = 2.09)) and ((cint = 45) or (cfloat = 3.02))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: (((cint = 49) or (cfloat = 3.5)) and ((cint = 47) or (cfloat = 2.09)) and ((cint = 45) or (cfloat = 3.02))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
@@ -30407,25 +30407,25 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: (cstring1) IN ('biology', 'history', 'topology') (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
GatherStats: false
Filter Operator
isSamplingPred: false
predicate: (cstring1) IN ('biology', 'history', 'topology') (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
keys: cstring1 (type: string)
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col1 (type: bigint)
auto parallelism: false
@@ -30492,16 +30492,16 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: bigint), _col0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: string)
null sort order: z
sort order: +
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
tag: -1
value expressions: _col0 (type: bigint)
auto parallelism: false
@@ -30512,13 +30512,13 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: bigint), KEY.reducesinkkey0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
GlobalTableId: 0
#### A masked pattern was here ####
NumFilesPerFileSink: 1
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
#### A masked pattern was here ####
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_1.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_1.q.out
index c57a010..095a701 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_1.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_1.q.out
@@ -64,7 +64,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((cboolean1 < 0) or (cbigint < UDFToLong(ctinyint)) or (UDFToLong(cint) > cbigint) or ((cdouble > UDFToDouble(ctinyint)) and (cboolean2 > 0))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -73,7 +73,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprOrExpr(children: FilterLongColLessLongScalar(col 10:boolean, val 0), FilterLongColLessLongColumn(col 3:bigint, col 0:bigint)(children: col 0:tinyint), FilterLongColGreaterLongColumn(col 2:bigint, col 3:bigint)(children: col 2:int), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleColumn(col 5:double, col 13:double)(children: CastLongToDouble(col 0:tinyint) -> 13:double), FilterLongColGreaterLongScalar(col 11:boolean, val 0)))
predicate: ((cboolean1 < 0) or (cbigint < UDFToLong(ctinyint)) or (UDFToLong(cint) > cbigint) or ((cdouble > UDFToDouble(ctinyint)) and (cboolean2 > 0))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), cfloat (type: float), cint (type: int), cdouble (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double), (cdouble * cdouble) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
@@ -82,7 +82,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [0, 4, 2, 5, 14, 17, 18]
selectExpressions: CastLongToDouble(col 0:tinyint) -> 14:double, DoubleColMultiplyDoubleColumn(col 15:double, col 16:double)(children: CastLongToDouble(col 0:tinyint) -> 15:double, CastLongToDouble(col 0:tinyint) -> 16:double) -> 17:double, DoubleColMultiplyDoubleColumn(col 5:double, col 5:double) -> 18:double
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col5), sum(_col4), count(_col0), sum(_col1), max(_col0), max(_col2), sum(_col6), sum(_col3), count(_col3), count(_col2)
Group By Vectorization:
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_10.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_10.q.out
index 65fa5c1..9ce20c8 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_10.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_10.q.out
@@ -68,7 +68,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((cstring2 <= '10') or ((UDFToDouble(ctinyint) > cdouble) and (CAST( ctinyint AS decimal(6,2)) <= -5638.15)) or ((cdouble > 6981.0D) and ((CAST( csmallint AS decimal(11,4)) = 9763215.5639) or (cstring1 like '%a')))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -77,7 +77,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprOrExpr(children: FilterStringGroupColLessEqualStringScalar(col 7:string, val 10), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleColumn(col 13:double, col 5:double)(children: CastLongToDouble(col 0:tinyint) -> 13:double), FilterDecimalColLessEqualDecimalScalar(col 14:decimal(6,2), val -5638.15)(children: CastLongToDecimal(col 0:tinyint) -> 14:decimal(6,2))), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 5:double, val 6981.0), FilterExprOrExpr(children: FilterDecimalColEqualDecimalScalar(col 15:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 1:smallint) -> 15:decimal(11,4)), FilterStringColLikeStringScalar(col 6:string, pattern %a))))
predicate: ((cstring2 <= '10') or ((UDFToDouble(ctinyint) > cdouble) and (CAST( ctinyint AS decimal(6,2)) <= -5638.15)) or ((cdouble > 6981.0D) and ((CAST( csmallint AS decimal(11,4)) = 9763215.5639) or (cstring1 like '%a')))) (type: boolean)
- Statistics: Num rows: 9557 Data size: 461790 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9557 Data size: 461542 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdouble (type: double), ctimestamp1 (type: timestamp), ctinyint (type: tinyint), cboolean1 (type: boolean), cstring1 (type: string), (- cdouble) (type: double), (cdouble + UDFToDouble(csmallint)) (type: double), ((cdouble + UDFToDouble(csmallint)) % 33.0D) (type: double), (- cdouble) (type: double), (UDFToDouble(ctinyint) % cdouble) (type: double), (UDFToShort(ctinyint) % csmallint) (type: smallint), (- cdouble) (type: double), (cbigint * UDFToLong((UDFToShort(ctinyint) % csmallint))) (type: bigint), (9763215.5639D - (cdouble + UDFToDouble(csmallint))) (type: double), (- (- cdouble)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
@@ -86,13 +86,13 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [5, 8, 0, 10, 6, 16, 18, 21, 22, 24, 25, 26, 28, 31, 33]
selectExpressions: DoubleColUnaryMinus(col 5:double) -> 16:double, DoubleColAddDoubleColumn(col 5:double, col 17:double)(children: CastLongToDouble(col 1:smallint) -> 17:double) -> 18:double, DoubleColModuloDoubleScalar(col 20:double, val 33.0)(children: DoubleColAddDoubleColumn(col 5:double, col 19:double)(children: CastLongToDouble(col 1:smallint) -> 19:double) -> 20:double) -> 21:double, DoubleColUnaryMinus(col 5:double) -> 22:double, DoubleColModuloDoubleColumn(col 23:double, col 5:double)(children: CastLongToDouble(col 0:tinyint) -> 23:double) -> 24:double, LongColModuloLongColumn(col 0:smallint, col 1:smallint)(children: col 0:tinyint) -> 25:smallint, DoubleColUnaryMinus(col 5:double) -> 26:double, LongColMultiplyLongColumn(col 3:bigint, col 27:bigint)(children: LongColModuloLongColumn(col 0:smallint, col 1:smallint)(children: col 0:tinyint) -> 27:smallint) -> 28:bigint, DoubleScalarSubtractDoubleColumn(val 9763215.5639, col 30:double)(children: DoubleColAddDoubleColumn(col 5:double, col 29:double)(children: CastLongToDouble(col 1:smallint) -> 29:double) -> 30:double) -> 31:double, DoubleColUnaryMinus(col 32:double)(children: DoubleColUnaryMinus(col 5:double) -> 32:double) -> 33:double
- Statistics: Num rows: 9557 Data size: 461790 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9557 Data size: 461542 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 9557 Data size: 461790 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9557 Data size: 461542 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_11.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_11.q.out
index 61782ad..5f1aac5 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_11.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_11.q.out
@@ -50,7 +50,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((cstring2 = cstring1) or (ctimestamp1 is null and (cstring1 like '%a'))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -59,7 +59,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprOrExpr(children: FilterStringGroupColEqualStringGroupColumn(col 7:string, col 6:string), FilterExprAndExpr(children: SelectColumnIsNull(col 8:timestamp), FilterStringColLikeStringScalar(col 6:string, pattern %a)))
predicate: ((cstring2 = cstring1) or (ctimestamp1 is null and (cstring1 like '%a'))) (type: boolean)
- Statistics: Num rows: 9216 Data size: 445313 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9216 Data size: 445074 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string), cboolean1 (type: boolean), cdouble (type: double), ctimestamp1 (type: timestamp), (-3728 * UDFToInteger(csmallint)) (type: int), (cdouble - 9763215.5639D) (type: double), (- cdouble) (type: double), ((- cdouble) + 6981.0D) (type: double), (cdouble * -5638.15D) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
@@ -68,13 +68,13 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [6, 10, 5, 8, 13, 14, 15, 17, 18]
selectExpressions: LongScalarMultiplyLongColumn(val -3728, col 1:int)(children: col 1:smallint) -> 13:int, DoubleColSubtractDoubleScalar(col 5:double, val 9763215.5639) -> 14:double, DoubleColUnaryMinus(col 5:double) -> 15:double, DoubleColAddDoubleScalar(col 16:double, val 6981.0)(children: DoubleColUnaryMinus(col 5:double) -> 16:double) -> 17:double, DoubleColMultiplyDoubleScalar(col 5:double, val -5638.15) -> 18:double
- Statistics: Num rows: 9216 Data size: 445313 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9216 Data size: 445074 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 9216 Data size: 445313 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9216 Data size: 445074 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_12.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_12.q.out
index 30b6842..9b0560a 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_12.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_12.q.out
@@ -87,7 +87,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: (ctimestamp1 is null and ((cstring1 like '%a') or ((cboolean2 <= 1) and (cbigint >= UDFToLong(csmallint)))) and ((cboolean1 >= cboolean2) or (UDFToShort(ctinyint) <> csmallint))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -96,7 +96,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: SelectColumnIsNull(col 8:timestamp), FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 6:string, pattern %a), FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 11:boolean, val 1), FilterLongColGreaterEqualLongColumn(col 3:bigint, col 1:bigint)(children: col 1:smallint))), FilterExprOrExpr(children: FilterLongColGreaterEqualLongColumn(col 10:boolean, col 11:boolean), FilterLongColNotEqualLongColumn(col 0:smallint, col 1:smallint)(children: col 0:tinyint)))
predicate: (ctimestamp1 is null and ((cstring1 like '%a') or ((cboolean2 <= 1) and (cbigint >= UDFToLong(csmallint)))) and ((cboolean1 >= cboolean2) or (UDFToShort(ctinyint) <> csmallint))) (type: boolean)
- Statistics: Num rows: 3754 Data size: 181391 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3754 Data size: 181294 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint), cboolean1 (type: boolean), cstring1 (type: string), cdouble (type: double), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double), (cdouble * cdouble) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
@@ -105,7 +105,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [3, 10, 6, 5, 13, 16, 17]
selectExpressions: CastLongToDouble(col 3:bigint) -> 13:double, DoubleColMultiplyDoubleColumn(col 14:double, col 15:double)(children: CastLongToDouble(col 3:bigint) -> 14:double, CastLongToDouble(col 3:bigint) -> 15:double) -> 16:double, DoubleColMultiplyDoubleColumn(col 5:double, col 5:double) -> 17:double
- Statistics: Num rows: 3754 Data size: 181391 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3754 Data size: 181294 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(_col0), sum(_col5), sum(_col4), sum(_col3), count(_col3), sum(_col0), sum(_col6)
Group By Vectorization:
@@ -120,7 +120,7 @@ STAGE PLANS:
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
- Statistics: Num rows: 3754 Data size: 181391 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3754 Data size: 181294 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double), _col1 (type: bigint), _col2 (type: string), _col3 (type: boolean)
null sort order: aaaa
@@ -130,7 +130,7 @@ STAGE PLANS:
className: VectorReduceSinkMultiKeyOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 3754 Data size: 181391 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3754 Data size: 181294 Basic stats: COMPLETE Column stats: NONE
value expressions: _col4 (type: bigint), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: bigint), _col9 (type: bigint), _col10 (type: double)
Execution mode: vectorized
Map Vectorization:
@@ -164,7 +164,7 @@ STAGE PLANS:
keys: KEY._col0 (type: double), KEY._col1 (type: bigint), KEY._col2 (type: string), KEY._col3 (type: boolean)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
- Statistics: Num rows: 1877 Data size: 90695 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1877 Data size: 90647 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: bigint), _col3 (type: boolean), _col2 (type: string), _col0 (type: double), (-6432.0D * _col0) (type: double), (- _col1) (type: bigint), _col4 (type: bigint), (_col1 * _col4) (type: bigint), power(((_col5 - ((_col6 * _col6) / _col4)) / CASE WHEN ((_col4 = 1L)) THEN (null) ELSE ((_col4 - 1)) END), 0.5) (type: double), ((-6432.0D * _col0) / -6432.0D) (type: double), (- ((-6432.0D * _col0) / -6432.0D)) (type: double), (_col7 / _col8) (type: double), (- (-6432.0D * _col0)) (type: double), (-5638.15 + CAST( _col1 AS decimal(19,0))) (type: decimal(22,2)), _col9 (type: bigint), ((_col7 / _col8) / (-6432.0D * _col0)) (type: double), (- (- ((-6432.0D * _col0) / -6432.0D))) (type: double), (((-6432.0D * _col0) / -6432.0D) + (- (-6432.0D * _col0))) (type: double), power(((_col10 - ((_col7 * _col7) / _col8)) / _col8), 0.5) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col17, _col18, _col19
@@ -173,7 +173,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [1, 3, 2, 0, 11, 12, 4, 13, 21, 23, 26, 27, 29, 31, 9, 34, 38, 43, 48]
selectExpressions: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0:double) -> 11:double, LongColUnaryMinus(col 1:bigint) -> 12:bigint, LongColMultiplyLongColumn(col 1:bigint, col 4:bigint) -> 13:bigint, FuncPowerDoubleToDouble(col 20:double)(children: DoubleColDivideLongColumn(col 16:double, col 19:bigint)(children: DoubleColSubtractDoubleColumn(col 5:double, col 15:double)(children: DoubleColDivideLongColumn(col 14:double, col 4:bigint)(children: DoubleColMultiplyDoubleColumn(col 6:double, col 6:double) -> 14:double) -> 15:double) -> 16:double, IfExprNullCondExpr(col 17:boolean, null, col 18:bigint)(children: LongColEqualLongScalar(col 4:bigint, val 1) -> 17:boolean, LongColSubtractLongScalar(col 4:bigint, val 1) -> 18:bigint) -> 19:bigint) -> 20:double) -> 21:double, DoubleColDivideDoubleScalar(col 22:double, val -6432.0)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0:double) -> 22:double) -> 23:double, DoubleColUnaryMinus(col 25:double)(children: DoubleColDivideDoubleScalar(col 24:double, val -6432.0)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0:double) -> 24:double) -> 25:double) -> 26:double, DoubleColDivideLongColumn(col 7:double, col 8:bigint) -> 27:double, DoubleColUnaryMinus(col 28:double)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0:double) -> 28:double) -> 29:double, DecimalScalarAddDecimalColumn(val -5638.15, col 30:decimal(19,0))(children: CastLongToDecimal(col 1:bigint) -> 30:decimal(19,0)) -> 31:decimal(22,2), DoubleColDivideDoubleColumn(col 32:double, col 33:double)(children: DoubleColDivideLongColumn(col 7:double, col 8:bigint) -> 32:double, DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0:double) -> 33:double) -> 34:double, DoubleColUnaryMinus(col 37:double)(children: DoubleColUnaryMinus(col 36:double)(children: DoubleColDivideDoubleScalar(col 35:double, val -6432.0)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0:double) -> 35:double) -> 36:double) -> 37:double) -> 38:double, DoubleColAddDoubleColumn(col 40:double, col 42:double)(children: DoubleColDivideDoubleScalar(col 39:double, val -6432.0)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0:double) -> 39:double) -> 40:double, DoubleColUnaryMinus(col 41:double)(children: DoubleScalarMultiplyDoubleColumn(val -6432.0, col 0:double) -> 41:double) -> 42:double) -> 43:double, FuncPowerDoubleToDouble(col 47:double)(children: DoubleColDivideLongColumn(col 46:double, col 8:bigint)(children: DoubleColSubtractDoubleColumn(col 10:double, col 45:double)(children: DoubleColDivideLongColumn(col 44:double, col 8:bigint)(children: DoubleColMultiplyDoubleColumn(col 7:double, col 7:double) -> 44:double) -> 45:double) -> 46:double) -> 47:double) -> 48:double
- Statistics: Num rows: 1877 Data size: 90695 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1877 Data size: 90647 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col3 (type: double), _col0 (type: bigint), _col2 (type: string)
null sort order: zzz
@@ -182,7 +182,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 1877 Data size: 90695 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1877 Data size: 90647 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: boolean), _col4 (type: double), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: decimal(22,2)), _col14 (type: bigint), _col15 (type: double), _col17 (type: double), _col18 (type: double), _col19 (type: double)
Reducer 3
Execution mode: vectorized
@@ -201,13 +201,13 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [1, 3, 2, 19, 0, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 11, 16, 17, 18]
selectExpressions: ConstantVectorExpression(val null) -> 19:timestamp
- Statistics: Num rows: 1877 Data size: 90695 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1877 Data size: 90647 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 1877 Data size: 90695 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1877 Data size: 90647 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_13.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_13.q.out
index 875348c..921c8b9 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_13.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_13.q.out
@@ -89,7 +89,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -98,7 +98,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28789.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 14:double, val -28788.0)(children: CastTimestampToDouble(col 9:timestamp) -> 14:double), FilterDecimalColLessDecimalScalar(col 15:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 15:decimal(11,4))))
predicate: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28789.0D) and (UDFToDouble(ctimestamp2) <> -28788.0D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean)
- Statistics: Num rows: 2730 Data size: 131912 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2730 Data size: 131841 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
@@ -107,7 +107,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [10, 0, 8, 4, 6, 4, 16, 17, 20]
selectExpressions: DoubleColMultiplyDoubleColumn(col 4:double, col 4:double)(children: col 4:float, col 4:float) -> 16:double, CastLongToDouble(col 0:tinyint) -> 17:double, DoubleColMultiplyDoubleColumn(col 18:double, col 19:double)(children: CastLongToDouble(col 0:tinyint) -> 18:double, CastLongToDouble(col 0:tinyint) -> 19:double) -> 20:double
- Statistics: Num rows: 2730 Data size: 131912 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2730 Data size: 131841 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: max(_col1), sum(_col3), sum(_col6), sum(_col5), count(_col3), sum(_col8), sum(_col7), count(_col1), max(_col3), min(_col1)
Group By Vectorization:
@@ -122,7 +122,7 @@ STAGE PLANS:
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
- Statistics: Num rows: 2730 Data size: 131912 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2730 Data size: 131841 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string)
null sort order: aaaaa
@@ -132,7 +132,7 @@ STAGE PLANS:
className: VectorReduceSinkMultiKeyOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 2730 Data size: 131912 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2730 Data size: 131841 Basic stats: COMPLETE Column stats: NONE
value expressions: _col5 (type: tinyint), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: bigint), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: float), _col14 (type: tinyint)
Execution mode: vectorized
Map Vectorization:
@@ -166,7 +166,7 @@ STAGE PLANS:
keys: KEY._col0 (type: boolean), KEY._col1 (type: tinyint), KEY._col2 (type: timestamp), KEY._col3 (type: float), KEY._col4 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * UDFToDouble(((- _col1) + _col5))) (type: double), (- _col6) (type: double), (79.553 * _col3) (type: float), power(((_col7 - ((_col8 * _col8) / _col9)) / _col9), 0.5) (type: double), (- _col6) (type: double), power(((_col10 - ((_col11 * _col11) / _col12)) / _col12), 0.5) (type: double), (CAST( ((- _col1) + _col5) AS decimal(3,0)) - 10.175) (type: decimal(7,3)), (- (- _col6)) (type: double), (-26.28D / (- (- _col6))) (type: double), _col13 (type: float), ((_col6 * UDFToDouble(((- _col1) + _col5))) / UDFToDouble(_col1)) (type: double), _col14 (type: tinyint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
@@ -175,7 +175,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [0, 1, 2, 3, 4, 15, 5, 17, 6, 21, 22, 23, 28, 29, 34, 38, 40, 43, 13, 49, 14]
selectExpressions: LongColUnaryMinus(col 1:tinyint) -> 15:tinyint, LongColAddLongColumn(col 16:tinyint, col 5:tinyint)(children: LongColUnaryMinus(col 1:tinyint) -> 16:tinyint) -> 17:tinyint, DoubleColMultiplyDoubleColumn(col 6:double, col 20:double)(children: CastLongToDouble(col 19:tinyint)(children: LongColAddLongColumn(col 18:tinyint, col 5:tinyint)(children: LongColUnaryMinus(col 1:tinyint) -> 18:tinyint) -> 19:tinyint) -> 20:double) -> 21:double, DoubleColUnaryMinus(col 6:double) -> 22:double, DoubleScalarMultiplyDoubleColumn(val 79.5530014038086, col 3:float) -> 23:float, FuncPowerDoubleToDouble(col 27:double)(children: DoubleColDivideLongColumn(col 26:double, col 9:bigint)(children: DoubleColSubtractDoubleColumn(col 7:double, col 25:double)(children: DoubleColDivideLongColumn(col 24:double, col 9:bigint)(children: DoubleColMultiplyDoubleColumn(col 8:double, col 8:double) -> 24:double) -> 25:double) -> 26:double) -> 27:double) -> 28:double, DoubleColUnaryMinus(col 6:double) -> 29:double, FuncPowerDoubleToDouble(col 33:double)(children: DoubleColDivideLongColumn(col 32:double, col 12:bigint)(children: DoubleColSubtractDoubleColumn(col 10:double, col 31:double)(children: DoubleColDivideLongColumn(col 30:double, col 12:bigint)(children: DoubleColMultiplyDoubleColumn(col 11:double, col 11:double) -> 30:double) -> 31:double) -> 32:double) -> 33:double) -> 34:double, DecimalColSubtractDecimalScalar(col 37:decimal(3,0), val 10.175)(children: CastLongToDecimal(col 36:tinyint)(children: LongColAddLongColumn(col 35:tinyint, col 5:tinyint)(children: LongColUnaryMinus(col 1:tinyint) -> 35:tinyint) -> 36:tinyint) -> 37:decimal(3,0)) -> 38:decimal(7,3), DoubleColUnaryMinus(col 39:double)(children: DoubleColUnaryMinus(col 6:double) -> 39:double) -> 40:double, DoubleScalarDivideDoubleColumn(val -26.28, col 42:double)(children: DoubleColUnaryMinus(col 41:double)(children: DoubleColUnaryMinus(col 6:double) -> 41:double) -> 42:double) -> 43:double, DoubleColDivideDoubleColumn(col 47:double, col 48:double)(children: DoubleColMultiplyDoubleColumn(col 6:double, col 46:double)(children: CastLongToDouble(col 45:tinyint)(children: LongColAddLongColumn(col 44:tinyint, col 5:tinyint)(children: LongColUnaryMinus(col 1:tinyint) -> 44:tinyint) -> 45:tinyint) -> 46:double) -> 47:double, CastLongToDouble(col 1:tinyint) -> 48:double) -> 49:double
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), _col5 (type: tinyint), _col6 (type: tinyint), _col7 (type: tinyint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: double), _col13 (type: double), _col14 (type: double), _col15 (type: decimal(7,3)), _col16 (type: double), _col17 (type: double), _col18 (type: float), _col19 (type: double), _col20 (type: tinyint)
null sort order: zzzzzzzzzzzzzzzzzzzzz
@@ -184,7 +184,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
Reducer 3
Execution mode: vectorized
@@ -202,7 +202,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 10, 14, 15, 16, 17, 18, 19, 20]
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 40
Limit Vectorization:
@@ -425,7 +425,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -434,7 +434,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val 3569.0), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 10.175), FilterLongColNotEqualLongScalar(col 10:boolean, val 1)), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28801.388)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 14:double, val -28801.336)(children: CastTimestampToDouble(col 9:timestamp) -> 14:double), FilterDecimalColLessDecimalScalar(col 15:decimal(11,4), val 9763215.5639)(children: CastLongToDecimal(col 0:tinyint) -> 15:decimal(11,4))))
predicate: (((cfloat < 3569.0) and (cdouble <= 10.175D) and (cboolean1 <> 1)) or ((UDFToDouble(ctimestamp1) > -28801.388D) and (UDFToDouble(ctimestamp2) <> -28801.336D) and (CAST( ctinyint AS decimal(11,4)) < 9763215.5639))) (type: boolean)
- Statistics: Num rows: 2730 Data size: 131912 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2730 Data size: 131841 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cboolean1 (type: boolean), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
@@ -443,7 +443,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [10, 0, 8, 4, 6, 4, 16, 17, 20]
selectExpressions: DoubleColMultiplyDoubleColumn(col 4:double, col 4:double)(children: col 4:float, col 4:float) -> 16:double, CastLongToDouble(col 0:tinyint) -> 17:double, DoubleColMultiplyDoubleColumn(col 18:double, col 19:double)(children: CastLongToDouble(col 0:tinyint) -> 18:double, CastLongToDouble(col 0:tinyint) -> 19:double) -> 20:double
- Statistics: Num rows: 2730 Data size: 131912 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2730 Data size: 131841 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: max(_col1), sum(_col3), sum(_col6), sum(_col5), count(_col3), sum(_col8), sum(_col7), count(_col1), max(_col3), min(_col1)
Group By Vectorization:
@@ -458,7 +458,7 @@ STAGE PLANS:
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
- Statistics: Num rows: 2730 Data size: 131912 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2730 Data size: 131841 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string)
null sort order: aaaaa
@@ -468,7 +468,7 @@ STAGE PLANS:
className: VectorReduceSinkMultiKeyOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 2730 Data size: 131912 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2730 Data size: 131841 Basic stats: COMPLETE Column stats: NONE
value expressions: _col5 (type: tinyint), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: bigint), _col10 (type: double), _col11 (type: double), _col12 (type: bigint), _col13 (type: float), _col14 (type: tinyint)
Execution mode: vectorized
Map Vectorization:
@@ -502,7 +502,7 @@ STAGE PLANS:
keys: KEY._col0 (type: boolean), KEY._col1 (type: tinyint), KEY._col2 (type: timestamp), KEY._col3 (type: float), KEY._col4 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), (- _col1) (type: tinyint), _col5 (type: tinyint), ((- _col1) + _col5) (type: tinyint), _col6 (type: double), (_col6 * UDFToDouble(((- _col1) + _col5))) (type: double), (- _col6) (type: double), (79.553 * _col3) (type: float), power(((_col7 - ((_col8 * _col8) / _col9)) / _col9), 0.5) (type: double), (- _col6) (type: double), power(((_col10 - ((_col11 * _col11) / _col12)) / _col12), 0.5) (type: double), (CAST( ((- _col1) + _col5) AS decimal(3,0)) - 10.175) (type: decimal(7,3)), (- (- _col6)) (type: double), (-26.28D / (- (- _col6))) (type: double), _col13 (type: float), ((_col6 * UDFToDouble(((- _col1) + _col5))) / UDFToDouble(_col1)) (type: double), _col14 (type: tinyint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
@@ -511,7 +511,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [0, 1, 2, 3, 4, 15, 5, 17, 6, 21, 22, 23, 28, 29, 34, 38, 40, 43, 13, 49, 14]
selectExpressions: LongColUnaryMinus(col 1:tinyint) -> 15:tinyint, LongColAddLongColumn(col 16:tinyint, col 5:tinyint)(children: LongColUnaryMinus(col 1:tinyint) -> 16:tinyint) -> 17:tinyint, DoubleColMultiplyDoubleColumn(col 6:double, col 20:double)(children: CastLongToDouble(col 19:tinyint)(children: LongColAddLongColumn(col 18:tinyint, col 5:tinyint)(children: LongColUnaryMinus(col 1:tinyint) -> 18:tinyint) -> 19:tinyint) -> 20:double) -> 21:double, DoubleColUnaryMinus(col 6:double) -> 22:double, DoubleScalarMultiplyDoubleColumn(val 79.5530014038086, col 3:float) -> 23:float, FuncPowerDoubleToDouble(col 27:double)(children: DoubleColDivideLongColumn(col 26:double, col 9:bigint)(children: DoubleColSubtractDoubleColumn(col 7:double, col 25:double)(children: DoubleColDivideLongColumn(col 24:double, col 9:bigint)(children: DoubleColMultiplyDoubleColumn(col 8:double, col 8:double) -> 24:double) -> 25:double) -> 26:double) -> 27:double) -> 28:double, DoubleColUnaryMinus(col 6:double) -> 29:double, FuncPowerDoubleToDouble(col 33:double)(children: DoubleColDivideLongColumn(col 32:double, col 12:bigint)(children: DoubleColSubtractDoubleColumn(col 10:double, col 31:double)(children: DoubleColDivideLongColumn(col 30:double, col 12:bigint)(children: DoubleColMultiplyDoubleColumn(col 11:double, col 11:double) -> 30:double) -> 31:double) -> 32:double) -> 33:double) -> 34:double, DecimalColSubtractDecimalScalar(col 37:decimal(3,0), val 10.175)(children: CastLongToDecimal(col 36:tinyint)(children: LongColAddLongColumn(col 35:tinyint, col 5:tinyint)(children: LongColUnaryMinus(col 1:tinyint) -> 35:tinyint) -> 36:tinyint) -> 37:decimal(3,0)) -> 38:decimal(7,3), DoubleColUnaryMinus(col 39:double)(children: DoubleColUnaryMinus(col 6:double) -> 39:double) -> 40:double, DoubleScalarDivideDoubleColumn(val -26.28, col 42:double)(children: DoubleColUnaryMinus(col 41:double)(children: DoubleColUnaryMinus(col 6:double) -> 41:double) -> 42:double) -> 43:double, DoubleColDivideDoubleColumn(col 47:double, col 48:double)(children: DoubleColMultiplyDoubleColumn(col 6:double, col 46:double)(children: CastLongToDouble(col 45:tinyint)(children: LongColAddLongColumn(col 44:tinyint, col 5:tinyint)(children: LongColUnaryMinus(col 1:tinyint) -> 44:tinyint) -> 45:tinyint) -> 46:double) -> 47:double, CastLongToDouble(col 1:tinyint) -> 48:double) -> 49:double
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: boolean), _col1 (type: tinyint), _col2 (type: timestamp), _col3 (type: float), _col4 (type: string), _col5 (type: tinyint), _col6 (type: tinyint), _col7 (type: tinyint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: double), _col13 (type: double), _col14 (type: double), _col15 (type: decimal(7,3)), _col16 (type: double), _col17 (type: double), _col18 (type: float), _col19 (type: double), _col20 (type: tinyint)
null sort order: zzzzzzzzzzzzzzzzzzzzz
@@ -520,7 +520,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
Reducer 3
Execution mode: vectorized
@@ -538,7 +538,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 10, 14, 15, 16, 17, 18, 19, 20]
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 40
Limit Vectorization:
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_14.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_14.q.out
index 8ef326c..6fb575d 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_14.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_14.q.out
@@ -89,7 +89,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((UDFToLong(ctinyint) <= cbigint) and (cdouble < UDFToDouble(ctinyint)) and ((cbigint > -257L) or (cfloat < UDFToFloat(cint))) and ((UDFToDouble(cint) <= cdouble) or (ctimestamp2 < ctimestamp1))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -98,7 +98,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: FilterLongColLessEqualLongColumn(col 0:bigint, col 3:bigint)(children: col 0:tinyint), FilterDoubleColLessDoubleColumn(col 5:double, col 13:double)(children: CastLongToDouble(col 0:tinyint) -> 13:double), FilterExprOrExpr(children: FilterLongColGreaterLongScalar(col 3:bigint, val -257), FilterDoubleColLessDoubleColumn(col 4:float, col 14:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 14:float)), FilterExprOrExpr(children: FilterDoubleColLessEqualDoubleColumn(col 15:double, col 5:double)(children: CastLongToDouble(col 2:int) -> 15:double), FilterTimestampColLessTimestampColumn(col 9:timestamp, col 8:timestamp)))
predicate: ((UDFToLong(ctinyint) <= cbigint) and (cdouble < UDFToDouble(ctinyint)) and ((cbigint > -257L) or (cfloat < UDFToFloat(cint))) and ((UDFToDouble(cint) <= cdouble) or (ctimestamp2 < ctimestamp1))) (type: boolean)
- Statistics: Num rows: 606 Data size: 29281 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 606 Data size: 29265 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctimestamp1 (type: timestamp), cfloat (type: float), cstring1 (type: string), cboolean1 (type: boolean), cdouble (type: double), (- (-26.28D + cdouble)) (type: double), ((- (-26.28D + cdouble)) * (- (-26.28D + cdouble))) (type: double), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
@@ -107,7 +107,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [8, 4, 6, 10, 5, 17, 22, 4, 23]
selectExpressions: DoubleColUnaryMinus(col 16:double)(children: DoubleScalarAddDoubleColumn(val -26.28, col 5:double) -> 16:double) -> 17:double, DoubleColMultiplyDoubleColumn(col 19:double, col 21:double)(children: DoubleColUnaryMinus(col 18:double)(children: DoubleScalarAddDoubleColumn(val -26.28, col 5:double) -> 18:double) -> 19:double, DoubleColUnaryMinus(col 20:double)(children: DoubleScalarAddDoubleColumn(val -26.28, col 5:double) -> 20:double) -> 21:double) -> 22:double, DoubleColMultiplyDoubleColumn(col 4:double, col 4:double)(children: col 4:float, col 4:float) -> 23:double
- Statistics: Num rows: 606 Data size: 29281 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 606 Data size: 29265 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col6), sum(_col5), count(_col5), max(_col1), sum(_col8), sum(_col7), count(_col1)
Group By Vectorization:
@@ -122,7 +122,7 @@ STAGE PLANS:
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 606 Data size: 29281 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 606 Data size: 29265 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: float), _col2 (type: double), _col3 (type: timestamp), _col4 (type: boolean)
null sort order: aaaaa
@@ -132,7 +132,7 @@ STAGE PLANS:
className: VectorReduceSinkMultiKeyOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 606 Data size: 29281 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 606 Data size: 29265 Basic stats: COMPLETE Column stats: NONE
value expressions: _col5 (type: double), _col6 (type: double), _col7 (type: bigint), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: bigint)
Execution mode: vectorized
Map Vectorization:
@@ -166,7 +166,7 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: float), KEY._col2 (type: double), KEY._col3 (type: timestamp), KEY._col4 (type: boolean)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
- Statistics: Num rows: 303 Data size: 14640 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 303 Data size: 14632 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col3 (type: timestamp), _col1 (type: float), _col0 (type: string), _col4 (type: boolean), _col2 (type: double), (-26.28D + _col2) (type: double), (- (-26.28D + _col2)) (type: double), power(((_col5 - ((_col6 * _col6) / _col7)) / CASE WHEN ((_col7 = 1L)) THEN (null) ELSE ((_col7 - 1)) END), 0.5) (type: double), (_col1 * -26.28) (type: float), _col8 (type: float), (- _col1) (type: float), (- _col8) (type: float), ((- (-26.28D + _col2)) / 10.175D) (type: double), power(((_col9 - ((_col10 * _col10) / _col11)) / _col11), 0.5) (type: double), _col11 (type: bigint), (- ((- (-26.28D + _col2)) / 10.175D)) (type: double), (-1.389D % power(((_col5 - ((_col6 * _col6) / _col7)) / CASE WHEN ((_col7 = 1L)) THEN (null) ELSE ((_col7 - 1)) END), 0.5)) (type: double), (UDFToDouble(_col1) - _col2) (type: double), ((_col9 - ((_col10 * _col10) / _col11)) / _col11) (type: double), (((_col9 - ((_col10 * _col10) / _col11)) / _col11) % 10.175D) (type: double), ((_col9 - ((_col10 * _col10) / _col11)) / CASE WHEN ((_col11 = 1L)) THEN (null) ELSE ((_col11 - 1)) END) (type: double), (- (UDFToDouble(_col1) - _col2)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21
@@ -175,7 +175,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [3, 1, 0, 4, 2, 12, 14, 22, 23, 8, 24, 25, 28, 33, 11, 37, 46, 47, 51, 56, 63, 65]
selectExpressions: DoubleScalarAddDoubleColumn(val -26.28, col 2:double) -> 12:double, DoubleColUnaryMinus(col 13:double)(children: DoubleScalarAddDoubleColumn(val -26.28, col 2:double) -> 13:double) -> 14:double, FuncPowerDoubleToDouble(col 21:double)(children: DoubleColDivideLongColumn(col 17:double, col 20:bigint)(children: DoubleColSubtractDoubleColumn(col 5:double, col 16:double)(children: DoubleColDivideLongColumn(col 15:double, col 7:bigint)(children: DoubleColMultiplyDoubleColumn(col 6:double, col 6:double) -> 15:double) -> 16:double) -> 17:double, IfExprNullCondExpr(col 18:boolean, null, col 19:bigint)(children: LongColEqualLongScalar(col 7:bigint, val 1) -> 18:boolean, LongColSubtractLongScalar(col 7:bigint, val 1) -> 19:bigint) -> 20:bigint) -> 21:double) -> 22:double, DoubleColMultiplyDoubleScalar(col 1:float, val -26.280000686645508) -> 23:float, DoubleColUnaryMinus(col 1:float) -> 24:float, DoubleColUnaryMinus(col 8:float) -> 25:float, DoubleColDivideDoubleScalar(col 27:double, val 10.175)(children: DoubleColUnaryMinus(col 26:double)(children: DoubleScalarAddDoubleColumn(val -26.28, col 2:double) -> 26:double) -> 27:double) -> 28:double, FuncPowerDoubleToDouble(col 32:double)(children: DoubleColDivideLongColumn(col 31:double, col 11:bigint)(children: DoubleColSubtractDoubleColumn(col 9:double, col 30:double)(children: DoubleColDivideLongColumn(col 29:double, col 11:bigint)(children: DoubleColMultiplyDoubleColumn(col 10:double, col 10:double) -> 29:double) -> 30:double) -> 31:double) -> 32:double) -> 33:double, DoubleColUnaryMinus(col 36:double)(children: DoubleColDivideDoubleScalar(col 35:double, val 10.175)(children: DoubleColUnaryMinus(col 34:double)(children: DoubleScalarAddDoubleColumn(val -26.28, col 2:double) -> 34:double) -> 35:double) -> 36:double) -> 37:double, DoubleScalarModuloDoubleColumn(val -1.389, col 45:double)(children: FuncPowerDoubleToDouble(col 44:double)(children: DoubleColDivideLongColumn(col 40:double, col 43:bigint)(children: DoubleColSubtractDoubleColumn(col 5:double, col 39:double)(children: DoubleColDivideLongColumn(col 38:double, col 7:bigint)(children: DoubleColMultiplyDoubleColumn(col 6:double, col 6:double) -> 38:double) -> 39:double) -> 40:double, IfExprNullCondExpr(col 41:boolean, null, col 42:bigint)(children: LongColEqualLongScalar(col 7:bigint, val 1) -> 41:boolean, LongColSubtractLongScalar(col 7:bigint, val 1) -> 42:bigint) -> 43:bigint) -> 44:double) -> 45:double) -> 46:double, DoubleColSubtractDoubleColumn(col 1:double, col 2:double)(children: col 1:float) -> 47:double, DoubleColDivideLongColumn(col 50:double, col 11:bigint)(children: DoubleColSubtractDoubleColumn(col 9:double, col 49:double)(children: DoubleColDivideLongColumn(col 48:double, col 11:bigint)(children: DoubleColMultiplyDoubleColumn(col 10:double, col 10:double) -> 48:double) -> 49:double) -> 50:double) -> 51:double, DoubleColModuloDoubleScalar(col 55:double, val 10.175)(children: DoubleColDivideLongColumn(col 54:double, col 11:bigint)(children: DoubleColSubtractDoubleColumn(col 9:double, col 53:double)(children: DoubleColDivideLongColumn(col 52:double, col 11:bigint)(children: DoubleColMultiplyDoubleColumn(col 10:double, col 10:double) -> 52:double) -> 53:double) -> 54:double) -> 55:double) -> 56:double, DoubleColDivideLongColumn(col 59:double, col 62:bigint)(children: DoubleColSubtractDoubleColumn(col 9:double, col 58:double)(children: DoubleColDivideLongColumn(col 57:double, col 11:bigint)(children: DoubleColMultiplyDoubleColumn(col 10:double, col 10:double) -> 57:double) -> 58:double) -> 59:double, IfExprNullCondExpr(col 60:boolean, null, col 61:bigint)(children: LongColEqualLongScalar(col 11:bigint, val 1) -> 60:boolean, LongColSubtractLongScalar(col 11:bigint, val 1) -> 61:bigint) -> 62:bigint) -> 63:double, DoubleColUnaryMinus(col 64:double)(children: DoubleColSubtractDoubleColumn(col 1:double, col 2:double)(children: col 1:float) -> 64:double) -> 65:double
- Statistics: Num rows: 303 Data size: 14640 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 303 Data size: 14632 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col2 (type: string), _col1 (type: float), _col4 (type: double), _col0 (type: timestamp)
null sort order: zzzz
@@ -184,7 +184,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 303 Data size: 14640 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 303 Data size: 14632 Basic stats: COMPLETE Column stats: NONE
value expressions: _col3 (type: boolean), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: float), _col10 (type: float), _col11 (type: float), _col12 (type: double), _col13 (type: double), _col14 (type: bigint), _col15 (type: double), _col16 (type: double), _col17 (type: double), _col18 (type: double), _col19 (type: double), _col20 (type: double), _col21 (type: double)
Reducer 3
Execution mode: vectorized
@@ -202,13 +202,13 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [3, 1, 0, 4, 2, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21]
- Statistics: Num rows: 303 Data size: 14640 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 303 Data size: 14632 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 303 Data size: 14640 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 303 Data size: 14632 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_15.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_15.q.out
index f124b67..88095ba 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_15.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_15.q.out
@@ -85,7 +85,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((cstring1 like '10%') or (cstring2 like '%ss%') or ((cint >= -75) and (UDFToShort(ctinyint) = csmallint) and (cdouble >= -3728.0D))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -94,7 +94,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 6:string, pattern 10%), FilterStringColLikeStringScalar(col 7:string, pattern %ss%), FilterExprAndExpr(children: FilterLongColGreaterEqualLongScalar(col 2:int, val -75), FilterLongColEqualLongColumn(col 0:smallint, col 1:smallint)(children: col 0:tinyint), FilterDoubleColGreaterEqualDoubleScalar(col 5:double, val -3728.0)))
predicate: ((cstring1 like '10%') or (cstring2 like '%ss%') or ((cint >= -75) and (UDFToShort(ctinyint) = csmallint) and (cdouble >= -3728.0D))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cfloat (type: float), cboolean1 (type: boolean), cdouble (type: double), cstring1 (type: string), ctinyint (type: tinyint), cint (type: int), ctimestamp1 (type: timestamp), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double), UDFToDouble(cint) (type: double), (UDFToDouble(cint) * UDFToDouble(cint)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
@@ -103,7 +103,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [4, 10, 5, 6, 0, 2, 8, 4, 13, 14, 17, 18, 21]
selectExpressions: DoubleColMultiplyDoubleColumn(col 4:double, col 4:double)(children: col 4:float, col 4:float) -> 13:double, CastLongToDouble(col 0:tinyint) -> 14:double, DoubleColMultiplyDoubleColumn(col 15:double, col 16:double)(children: CastLongToDouble(col 0:tinyint) -> 15:double, CastLongToDouble(col 0:tinyint) -> 16:double) -> 17:double, CastLongToDouble(col 2:int) -> 18:double, DoubleColMultiplyDoubleColumn(col 19:double, col 20:double)(children: CastLongToDouble(col 2:int) -> 19:double, CastLongToDouble(col 2:int) -> 20:double) -> 21:double
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col8), sum(_col7), count(_col0), min(_col2), sum(_col10), sum(_col9), count(_col4), sum(_col12), sum(_col11), count(_col5)
Group By Vectorization:
@@ -118,7 +118,7 @@ STAGE PLANS:
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: float), _col1 (type: boolean), _col2 (type: double), _col3 (type: string), _col4 (type: tinyint), _col5 (type: int), _col6 (type: timestamp)
null sort order: aaaaaaa
@@ -128,7 +128,7 @@ STAGE PLANS:
className: VectorReduceSinkMultiKeyOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
value expressions: _col7 (type: double), _col8 (type: double), _col9 (type: bigint), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: bigint), _col14 (type: double), _col15 (type: double), _col16 (type: bigint)
Execution mode: vectorized
Map Vectorization:
@@ -151,16 +151,16 @@ STAGE PLANS:
keys: KEY._col0 (type: float), KEY._col1 (type: boolean), KEY._col2 (type: double), KEY._col3 (type: string), KEY._col4 (type: tinyint), KEY._col5 (type: int), KEY._col6 (type: timestamp)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: float), _col1 (type: boolean), _col2 (type: double), _col3 (type: string), _col4 (type: tinyint), _col5 (type: int), _col6 (type: timestamp), power(((_col7 - ((_col8 * _col8) / _col9)) / CASE WHEN ((_col9 = 1L)) THEN (null) ELSE ((_col9 - 1)) END), 0.5) (type: double), (-26.28 - CAST( _col5 AS decimal(10,0))) (type: decimal(13,2)), _col10 (type: double), (_col2 * 79.553D) (type: double), (33.0 % _col0) (type: float), power(((_col11 - ((_col12 * _col12) / _col13)) / CASE WHEN ((_col13 = 1L)) THEN (null) ELSE ((_col13 - 1)) END), 0.5) (type: double), ((_col11 - ((_col12 * _col12) / _col13)) / _col13) (type: double), (-23.0D % _col2) (type: double), (- _col4) (type: tinyint), ((_col14 - ((_col15 * _col15) / _col16)) / CASE WHEN ((_col16 = 1L)) THEN (null) ELSE ((_col16 - 1)) END) (type: double), (UDFToFloat(_col5) - _col0) (type: float), (-23 % UDFToInteger(_col4)) (type: int), (- (-26.28 - CAST( _col5 AS decimal(10,0)))) (type: decimal(13,2)), power(((_col14 - ((_col15 * _col15) / _col16)) / _col16), 0.5) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: float), _col1 (type: boolean), _col2 (type: double), _col3 (type: string), _col4 (type: tinyint), _col5 (type: int), _col6 (type: timestamp)
null sort order: zzzzzzz
sort order: +++++++
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
value expressions: _col7 (type: double), _col8 (type: decimal(13,2)), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: double), _col13 (type: double), _col14 (type: double), _col15 (type: tinyint), _col16 (type: double), _col17 (type: float), _col18 (type: int), _col19 (type: decimal(13,2)), _col20 (type: double)
Reducer 3
Reduce Vectorization:
@@ -171,10 +171,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: float), KEY.reducesinkkey1 (type: boolean), KEY.reducesinkkey2 (type: double), KEY.reducesinkkey3 (type: string), KEY.reducesinkkey4 (type: tinyint), KEY.reducesinkkey5 (type: int), KEY.reducesinkkey6 (type: timestamp), VALUE._col0 (type: double), VALUE._col1 (type: decimal(13,2)), VALUE._col2 (type: double), VALUE._col3 (type: double), VALUE._col4 (type: float), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: tinyint), VALUE._col9 (type: double), VALUE._col10 (type: float), VALUE._col11 (type: int), VALUE._col12 (type: decimal(13,2)), VALUE._col13 (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_16.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_16.q.out
index 6658e64..cc5275d 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_16.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_16.q.out
@@ -62,7 +62,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((cstring2 like '%b%') and ((cdouble >= -1.389D) or (cstring1 < 'a'))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -71,7 +71,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: FilterStringColLikeStringScalar(col 7:string, pattern %b%), FilterExprOrExpr(children: FilterDoubleColGreaterEqualDoubleScalar(col 5:double, val -1.389), FilterStringGroupColLessStringScalar(col 6:string, val a)))
predicate: ((cstring2 like '%b%') and ((cdouble >= -1.389D) or (cstring1 < 'a'))) (type: boolean)
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string), cdouble (type: double), ctimestamp1 (type: timestamp), (cdouble * cdouble) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
@@ -80,7 +80,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [6, 5, 8, 13]
selectExpressions: DoubleColMultiplyDoubleColumn(col 5:double, col 5:double) -> 13:double
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(_col1), sum(_col3), sum(_col1), min(_col1)
Group By Vectorization:
@@ -95,7 +95,7 @@ STAGE PLANS:
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: double), _col2 (type: timestamp)
null sort order: aaa
@@ -105,7 +105,7 @@ STAGE PLANS:
className: VectorReduceSinkMultiKeyOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
value expressions: _col3 (type: bigint), _col4 (type: double), _col5 (type: double), _col6 (type: double)
Execution mode: vectorized
Map Vectorization:
@@ -139,7 +139,7 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: double), KEY._col2 (type: timestamp)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 2048 Data size: 98958 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2048 Data size: 98905 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: double), _col2 (type: timestamp), (_col1 - 9763215.5639D) (type: double), (- (_col1 - 9763215.5639D)) (type: double), _col3 (type: bigint), power(((_col4 - ((_col5 * _col5) / _col3)) / CASE WHEN ((_col3 = 1L)) THEN (null) ELSE ((_col3 - 1)) END), 0.5) (type: double), (- power(((_col4 - ((_col5 * _col5) / _col3)) / CASE WHEN ((_col3 = 1L)) THEN (null) ELSE ((_col3 - 1)) END), 0.5)) (type: double), (power(((_col4 - ((_col5 * _col5) / _col3)) / CASE WHEN ((_col3 = 1L)) THEN (null) ELSE ((_col3 - 1)) END), 0.5) * UDFToDouble(_col3)) (type: double), _col6 (type: double), (9763215.5639D / _col1) (type: double), (CAST( _col3 AS decimal(19,0)) / -1.389) (type: decimal(28,6)), power(((_col4 - ((_col5 * _col5) / _col3)) / CASE WHEN ((_col3 = 1L)) THEN (null) ELSE ((_col3 - 1)) END), 0.5) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
@@ -148,13 +148,13 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [0, 1, 2, 7, 9, 3, 17, 26, 36, 6, 37, 39, 47]
selectExpressions: DoubleColSubtractDoubleScalar(col 1:double, val 9763215.5639) -> 7:double, DoubleColUnaryMinus(col 8:double)(children: DoubleColSubtractDoubleScalar(col 1:double, val 9763215.5639) -> 8:double) -> 9:double, FuncPowerDoubleToDouble(col 16:double)(children: DoubleColDivideLongColumn(col 12:double, col 15:bigint)(children: DoubleColSubtractDoubleColumn(col 4:double, col 11:double)(children: DoubleColDivideLongColumn(col 10:double, col 3:bigint)(children: DoubleColMultiplyDoubleColumn(col 5:double, col 5:double) -> 10:double) -> 11:double) -> 12:double, IfExprNullCondExpr(col 13:boolean, null, col 14:bigint)(children: LongColEqualLongScalar(col 3:bigint, val 1) -> 13:boolean, LongColSubtractLongScalar(col 3:bigint, val 1) -> 14:bigint) -> 15:bigint) -> 16:double) -> 17:double, DoubleColUnaryMinus(col 25:double)(children: FuncPowerDoubleToDouble(col 24:double)(children: DoubleColDivideLongColumn(col 20:double, col 23:bigint)(children: DoubleColSubtractDoubleColumn(col 4:double, col 19:double)(children: DoubleColDivideLongColumn(col 18:double, col 3:bigint)(children: DoubleColMultiplyDoubleColumn(col 5:double, col 5:double) -> 18:double) -> 19:double) -> 20:double, IfExprNullCondExpr(col 21:boolean, null, col 22:bigint)(children: LongColEqualLongScalar(col 3:bigint, val 1) -> 21:boolean, LongColSubtractLongScalar(col 3:bigint, val 1) -> 22:bigint) -> 23:bigint) -> 24:double) -> 25:double) -> 26:double, DoubleColMultiplyDoubleColumn(col 34:double, col 35:double)(children: FuncPowerDoubleToDouble(col 33:double)(children: DoubleColDivideLongColumn(col 29:double, col 32:bigint)(children: DoubleColSubtractDoubleColumn(col 4:double, col 28:double)(children: DoubleColDivideLongColumn(col 27:double, col 3:bigint)(children: DoubleColMultiplyDoubleColumn(col 5:double, col 5:double) -> 27:double) -> 28:double) -> 29:double, IfExprNullCondExpr(col 30:boolean, null, col 31:bigint)(children: LongColEqualLongScalar(col 3:bigint, val 1) -> 30:boolean, LongColSubtractLongScalar(col 3:bigint, val 1) -> 31:bigint) -> 32:bigint) -> 33:double) -> 34:double, CastLongToDouble(col 3:bigint) -> 35:double) -> 36:double, DoubleScalarDivideDoubleColumn(val 9763215.5639, col 1:double) -> 37:double, DecimalColDivideDecimalScalar(col 38:decimal(19,0), val -1.389)(children: CastLongToDecimal(col 3:bigint) -> 38:decimal(19,0)) -> 39:decimal(28,6), FuncPowerDoubleToDouble(col 46:double)(children: DoubleColDivideLongColumn(col 42:double, col 45:bigint)(children: DoubleColSubtractDoubleColumn(col 4:double, col 41:double)(children: DoubleColDivideLongColumn(col 40:double, col 3:bigint)(children: DoubleColMultiplyDoubleColumn(col 5:double, col 5:double) -> 40:double) -> 41:double) -> 42:double, IfExprNullCondExpr(col 43:boolean, null, col 44:bigint)(children: LongColEqualLongScalar(col 3:bigint, val 1) -> 43:boolean, LongColSubtractLongScalar(col 3:bigint, val 1) -> 44:bigint) -> 45:bigint) -> 46:double) -> 47:double
- Statistics: Num rows: 2048 Data size: 98958 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2048 Data size: 98905 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 2048 Data size: 98958 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2048 Data size: 98905 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_17.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_17.q.out
index f431947..8829f80 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_17.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_17.q.out
@@ -70,7 +70,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((cbigint > -23L) and ((ctinyint >= 33Y) or (UDFToLong(csmallint) >= cbigint) or (UDFToDouble(cfloat) = cdouble)) and ((cdouble <> 988888.0D) or (CAST( cint AS decimal(13,3)) > -863.257))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -79,7 +79,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: FilterLongColGreaterLongScalar(col 3:bigint, val -23), FilterExprOrExpr(children: FilterLongColGreaterEqualLongScalar(col 0:tinyint, val 33), FilterLongColGreaterEqualLongColumn(col 1:bigint, col 3:bigint)(children: col 1:smallint), FilterDoubleColEqualDoubleColumn(col 4:double, col 5:double)(children: col 4:float)), FilterExprOrExpr(children: FilterDoubleColNotEqualDoubleScalar(col 5:double, val 988888.0), FilterDecimalColGreaterDecimalScalar(col 13:decimal(13,3), val -863.257)(children: CastLongToDecimal(col 2:int) -> 13:decimal(13,3))))
predicate: ((cbigint > -23L) and ((ctinyint >= 33Y) or (UDFToLong(csmallint) >= cbigint) or (UDFToDouble(cfloat) = cdouble)) and ((cdouble <> 988888.0D) or (CAST( cint AS decimal(13,3)) > -863.257))) (type: boolean)
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cfloat (type: float), cstring1 (type: string), cint (type: int), ctimestamp1 (type: timestamp), cdouble (type: double), cbigint (type: bigint), (UDFToDouble(cfloat) / UDFToDouble(ctinyint)) (type: double), (UDFToLong(cint) % cbigint) (type: bigint), (- cdouble) (type: double), (cdouble + (UDFToDouble(cfloat) / UDFToDouble(ctinyint))) (type: double), (cdouble / UDFToDouble(cint)) (type: double), (- (- cdouble)) (type: double), (9763215.5639 % CAST( cbigint AS decimal(19,0))) (type: decimal(11,4)), (2563.58D + (- (- cdouble))) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
@@ -88,7 +88,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [4, 6, 2, 8, 5, 3, 15, 16, 17, 20, 22, 24, 26, 29]
selectExpressions: DoubleColDivideDoubleColumn(col 4:double, col 14:double)(children: col 4:float, CastLongToDouble(col 0:tinyint) -> 14:double) -> 15:double, LongColModuloLongColumn(col 2:bigint, col 3:bigint)(children: col 2:int) -> 16:bigint, DoubleColUnaryMinus(col 5:double) -> 17:double, DoubleColAddDoubleColumn(col 5:double, col 19:double)(children: DoubleColDivideDoubleColumn(col 4:double, col 18:double)(children: col 4:float, CastLongToDouble(col 0:tinyint) -> 18:double) -> 19:double) -> 20:double, DoubleColDivideDoubleColumn(col 5:double, col 21:double)(children: CastLongToDouble(col 2:int) -> 21:double) -> 22:double, DoubleColUnaryMinus(col 23:double)(children: DoubleColUnaryMinus(col 5:double) -> 23:double) -> 24:double, DecimalScalarModuloDecimalColumn(val 9763215.5639, col 25:decimal(19,0))(children: CastLongToDecimal(col 3:bigint) -> 25:decimal(19,0)) -> 26:decimal(11,4), DoubleScalarAddDoubleColumn(val 2563.58, col 28:double)(children: DoubleColUnaryMinus(col 27:double)(children: DoubleColUnaryMinus(col 5:double) -> 27:double) -> 28:double) -> 29:double
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col5 (type: bigint), _col0 (type: float)
null sort order: zz
@@ -97,7 +97,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string), _col2 (type: int), _col3 (type: timestamp), _col4 (type: double), _col6 (type: double), _col7 (type: bigint), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: decimal(11,4)), _col13 (type: double)
Execution mode: vectorized
Map Vectorization:
@@ -125,13 +125,13 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [1, 2, 3, 4, 5, 0, 6, 7, 8, 9, 10, 11, 12, 13]
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_2.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_2.q.out
index 1a2ffd0..ae8bf58 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_2.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_2.q.out
@@ -68,7 +68,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: (((cdouble < UDFToDouble(ctinyint)) and ((UDFToDouble(ctimestamp2) <> -10669.0D) or (cint < 359))) or ((ctimestamp1 < ctimestamp2) and (cstring2 like 'b%') and (cfloat <= -5638.15))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -77,7 +77,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColLessDoubleColumn(col 5:double, col 13:double)(children: CastLongToDouble(col 0:tinyint) -> 13:double), FilterExprOrExpr(children: FilterDoubleColNotEqualDoubleScalar(col 14:double, val -10669.0)(children: CastTimestampToDouble(col 9:timestamp) -> 14:double), FilterLongColLessLongScalar(col 2:int, val 359))), FilterExprAndExpr(children: FilterTimestampColLessTimestampColumn(col 8:timestamp, col 9:timestamp), FilterStringColLikeStringScalar(col 7:string, pattern b%), FilterDoubleColLessEqualDoubleScalar(col 4:float, val -5638.14990234375)))
predicate: (((cdouble < UDFToDouble(ctinyint)) and ((UDFToDouble(ctimestamp2) <> -10669.0D) or (cint < 359))) or ((ctimestamp1 < ctimestamp2) and (cstring2 like 'b%') and (cfloat <= -5638.15))) (type: boolean)
- Statistics: Num rows: 4778 Data size: 230870 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4778 Data size: 230747 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: csmallint (type: smallint), cfloat (type: float), cbigint (type: bigint), ctinyint (type: tinyint), cdouble (type: double), UDFToDouble(cbigint) (type: double), (UDFToDouble(cbigint) * UDFToDouble(cbigint)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
@@ -86,7 +86,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [1, 4, 3, 0, 5, 15, 18]
selectExpressions: CastLongToDouble(col 3:bigint) -> 15:double, DoubleColMultiplyDoubleColumn(col 16:double, col 17:double)(children: CastLongToDouble(col 3:bigint) -> 16:double, CastLongToDouble(col 3:bigint) -> 17:double) -> 18:double
- Statistics: Num rows: 4778 Data size: 230870 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4778 Data size: 230747 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col0), count(_col0), sum(_col1), sum(_col6), sum(_col5), count(_col2), count(), min(_col3), sum(_col4), count(_col4)
Group By Vectorization:
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_3.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_3.q.out
index 64be366..e96c587 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_3.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_3.q.out
@@ -73,7 +73,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: (((UDFToDouble(cbigint) > cdouble) and (CAST( csmallint AS decimal(8,3)) >= 79.553) and (ctimestamp1 > ctimestamp2)) or ((UDFToFloat(cint) <= cfloat) and (CAST( cbigint AS decimal(22,3)) <> 79.553) and (UDFToDouble(ctimestamp2) = -29071.0D))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -82,7 +82,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterDoubleColGreaterDoubleColumn(col 13:double, col 5:double)(children: CastLongToDouble(col 3:bigint) -> 13:double), FilterDecimalColGreaterEqualDecimalScalar(col 14:decimal(8,3), val 79.553)(children: CastLongToDecimal(col 1:smallint) -> 14:decimal(8,3)), FilterTimestampColGreaterTimestampColumn(col 8:timestamp, col 9:timestamp)), FilterExprAndExpr(children: FilterDoubleColLessEqualDoubleColumn(col 15:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 15:float), FilterDecimalColNotEqualDecimalScalar(col 16:decimal(22,3), val 79.553)(children: CastLongToDecimal(col 3:bigint) -> 16:decimal(22,3)), FilterDoubleColEqualDoubleScalar(col 17:double, val -29071.0)(children: CastTimestampToDouble(col 9:timestamp) -> 17:double)))
predicate: (((UDFToDouble(cbigint) > cdouble) and (CAST( csmallint AS decimal(8,3)) >= 79.553) and (ctimestamp1 > ctimestamp2)) or ((UDFToFloat(cint) <= cfloat) and (CAST( cbigint AS decimal(22,3)) <> 79.553) and (UDFToDouble(ctimestamp2) = -29071.0D))) (type: boolean)
- Statistics: Num rows: 2503 Data size: 120943 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2503 Data size: 120879 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: csmallint (type: smallint), ctinyint (type: tinyint), cfloat (type: float), cint (type: int), UDFToDouble(csmallint) (type: double), (UDFToDouble(csmallint) * UDFToDouble(csmallint)) (type: double), UDFToDouble(ctinyint) (type: double), (UDFToDouble(ctinyint) * UDFToDouble(ctinyint)) (type: double), UDFToDouble(cfloat) (type: double), (UDFToDouble(cfloat) * UDFToDouble(cfloat)) (type: double), UDFToDouble(cint) (type: double), (UDFToDouble(cint) * UDFToDouble(cint)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
@@ -91,7 +91,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [1, 0, 4, 2, 18, 21, 22, 25, 4, 26, 27, 30]
selectExpressions: CastLongToDouble(col 1:smallint) -> 18:double, DoubleColMultiplyDoubleColumn(col 19:double, col 20:double)(children: CastLongToDouble(col 1:smallint) -> 19:double, CastLongToDouble(col 1:smallint) -> 20:double) -> 21:double, CastLongToDouble(col 0:tinyint) -> 22:double, DoubleColMultiplyDoubleColumn(col 23:double, col 24:double)(children: CastLongToDouble(col 0:tinyint) -> 23:double, CastLongToDouble(col 0:tinyint) -> 24:double) -> 25:double, DoubleColMultiplyDoubleColumn(col 4:double, col 4:double)(children: col 4:float, col 4:float) -> 26:double, CastLongToDouble(col 2:int) -> 27:double, DoubleColMultiplyDoubleColumn(col 28:double, col 29:double)(children: CastLongToDouble(col 2:int) -> 28:double, CastLongToDouble(col 2:int) -> 29:double) -> 30:double
- Statistics: Num rows: 2503 Data size: 120943 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2503 Data size: 120879 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col5), sum(_col4), count(_col0), sum(_col7), sum(_col6), count(_col1), sum(_col9), sum(_col8), count(_col2), sum(_col2), sum(_col3), count(_col3), sum(_col11), sum(_col10)
Group By Vectorization:
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_4.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_4.q.out
index 1196b71..255f0ad 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_4.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_4.q.out
@@ -68,7 +68,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((UDFToInteger(csmallint) >= cint) or ((cbigint <> -563L) and ((UDFToLong(ctinyint) <> cbigint) or (cdouble <= -3728.0D))) or ((UDFToInteger(ctinyint) <= -89010) and (cdouble > 79.553D))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -77,7 +77,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprOrExpr(children: FilterLongColGreaterEqualLongColumn(col 1:int, col 2:int)(children: col 1:smallint), FilterExprAndExpr(children: FilterLongColNotEqualLongScalar(col 3:bigint, val -563), FilterExprOrExpr(children: FilterLongColNotEqualLongColumn(col 0:bigint, col 3:bigint)(children: col 0:tinyint), FilterDoubleColLessEqualDoubleScalar(col 5:double, val -3728.0))), FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 0:int, val -89010)(children: col 0:tinyint), FilterDoubleColGreaterDoubleScalar(col 5:double, val 79.553)))
predicate: ((UDFToInteger(csmallint) >= cint) or ((cbigint <> -563L) and ((UDFToLong(ctinyint) <> cbigint) or (cdouble <= -3728.0D))) or ((UDFToInteger(ctinyint) <= -89010) and (cdouble > 79.553D))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cdouble (type: double), ctinyint (type: tinyint), (cdouble * cdouble) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
@@ -86,7 +86,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [2, 5, 0, 13]
selectExpressions: DoubleColMultiplyDoubleColumn(col 5:double, col 5:double) -> 13:double
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col0), sum(_col3), sum(_col1), count(_col1), min(_col2)
Group By Vectorization:
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_5.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_5.q.out
index 497367c..ceff0e8 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_5.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_5.q.out
@@ -62,7 +62,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: (((cstring1 like '%b%') and cboolean2 is not null) or ((UDFToDouble(ctinyint) = cdouble) and (cstring2 like 'a') and ctimestamp2 is not null)) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -71,7 +71,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterStringColLikeStringScalar(col 6:string, pattern %b%), SelectColumnIsNotNull(col 11:boolean)), FilterExprAndExpr(children: FilterDoubleColEqualDoubleColumn(col 13:double, col 5:double)(children: CastLongToDouble(col 0:tinyint) -> 13:double), FilterStringColLikeStringScalar(col 7:string, pattern a), SelectColumnIsNotNull(col 9:timestamp)))
predicate: (((cstring1 like '%b%') and cboolean2 is not null) or ((UDFToDouble(ctinyint) = cdouble) and (cstring2 like 'a') and ctimestamp2 is not null)) (type: boolean)
- Statistics: Num rows: 9216 Data size: 445313 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9216 Data size: 445074 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int)
outputColumnNames: ctinyint, csmallint, cint
@@ -79,7 +79,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1, 2]
- Statistics: Num rows: 9216 Data size: 445313 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9216 Data size: 445074 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: max(csmallint), count(), min(csmallint), sum(cint), max(ctinyint)
Group By Vectorization:
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_6.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_6.q.out
index 2f00044..a9b48fc 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_6.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_6.q.out
@@ -62,7 +62,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (((cstring2 like '%a') or (cfloat <= -257.0)) and cbigint is not null)) and (ctinyint <> 0Y)) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -71,7 +71,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: FilterExprOrExpr(children: FilterExprAndExpr(children: FilterLongColLessEqualLongScalar(col 10:boolean, val 0), FilterLongColGreaterEqualLongColumn(col 11:boolean, col 10:boolean)), FilterExprAndExpr(children: FilterExprOrExpr(children: FilterStringColLikeStringScalar(col 7:string, pattern %a), FilterDoubleColLessEqualDoubleScalar(col 4:float, val -257.0)), SelectColumnIsNotNull(col 3:bigint))), FilterLongColNotEqualLongScalar(col 0:tinyint, val 0))
predicate: ((((cboolean1 <= 0) and (cboolean2 >= cboolean1)) or (((cstring2 like '%a') or (cfloat <= -257.0)) and cbigint is not null)) and (ctinyint <> 0Y)) (type: boolean)
- Statistics: Num rows: 11605 Data size: 560748 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11605 Data size: 560448 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cboolean1 (type: boolean), cfloat (type: float), cstring1 (type: string), (988888 * UDFToInteger(csmallint)) (type: int), (- csmallint) (type: smallint), (- cfloat) (type: float), (-26.28D / UDFToDouble(cfloat)) (type: double), (cfloat * 359.0) (type: float), (cint % UDFToInteger(ctinyint)) (type: int), (- cdouble) (type: double), (UDFToInteger(ctinyint) - -75) (type: int), (762 * (cint % UDFToInteger(ctinyint))) (type: int)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
@@ -80,13 +80,13 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [10, 4, 6, 13, 14, 15, 16, 17, 18, 19, 20, 22]
selectExpressions: LongScalarMultiplyLongColumn(val 988888, col 1:int)(children: col 1:smallint) -> 13:int, LongColUnaryMinus(col 1:smallint) -> 14:smallint, DoubleColUnaryMinus(col 4:float) -> 15:float, DoubleScalarDivideDoubleColumn(val -26.28, col 4:double)(children: col 4:float) -> 16:double, DoubleColMultiplyDoubleScalar(col 4:float, val 359.0) -> 17:float, LongColModuloLongColumn(col 2:int, col 0:int)(children: col 0:tinyint) -> 18:int, DoubleColUnaryMinus(col 5:double) -> 19:double, LongColSubtractLongScalar(col 0:int, val -75)(children: col 0:tinyint) -> 20:int, LongScalarMultiplyLongColumn(val 762, col 21:int)(children: LongColModuloLongColumn(col 2:int, col 0:int)(children: col 0:tinyint) -> 21:int) -> 22:int
- Statistics: Num rows: 11605 Data size: 560748 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11605 Data size: 560448 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 11605 Data size: 560748 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11605 Data size: 560448 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_7.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_7.q.out
index c9ae0c9..9cd2f50 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_7.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_7.q.out
@@ -76,7 +76,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: (((cdouble > 988888.0D) or ((UDFToDouble(ctimestamp2) > -28815.0D) and (cdouble <= 3569.0D))) and ((UDFToDouble(ctimestamp1) <= -28800.0D) or (UDFToInteger(ctinyint) = cint) or (cstring2 like 'ss')) and (ctinyint <> 0Y)) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -85,7 +85,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: FilterExprOrExpr(children: FilterDoubleColGreaterDoubleScalar(col 5:double, val 988888.0), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28815.0)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 3569.0))), FilterExprOrExpr(children: FilterDoubleColLessEqualDoubleScalar(col 14:double, val -28800.0)(children: CastTimestampToDouble(col 8:timestamp) -> 14:double), FilterLongColEqualLongColumn(col 0:int, col 2:int)(children: col 0:tinyint), FilterStringColLikeStringScalar(col 7:string, pattern ss)), FilterLongColNotEqualLongScalar(col 0:tinyint, val 0))
predicate: (((cdouble > 988888.0D) or ((UDFToDouble(ctimestamp2) > -28815.0D) and (cdouble <= 3569.0D))) and ((UDFToDouble(ctimestamp1) <= -28800.0D) or (UDFToInteger(ctinyint) = cint) or (cstring2 like 'ss')) and (ctinyint <> 0Y)) (type: boolean)
- Statistics: Num rows: 5461 Data size: 263873 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5461 Data size: 263731 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
@@ -94,7 +94,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [10, 3, 1, 0, 8, 6, 15, 16, 17, 18, 20, 22, 23, 24, 26]
selectExpressions: LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 15:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 16:int, LongColUnaryMinus(col 1:smallint) -> 17:smallint, LongColUnaryMinus(col 0:tinyint) -> 18:tinyint, LongColAddLongScalar(col 19:int, val 17)(children: LongColUnaryMinus(col 0:tinyint) -> 19:tinyint) -> 20:int, LongColMultiplyLongColumn(col 3:bigint, col 21:bigint)(children: LongColUnaryMinus(col 1:smallint) -> 21:smallint) -> 22:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 23:int, LongColUnaryMinus(col 0:tinyint) -> 24:tinyint, LongColModuloLongColumn(col 25:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 25:tinyint) -> 26:tinyint
- Statistics: Num rows: 5461 Data size: 263873 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5461 Data size: 263731 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: boolean), _col1 (type: bigint), _col2 (type: smallint), _col3 (type: tinyint), _col4 (type: timestamp), _col5 (type: string), _col6 (type: bigint), _col7 (type: int), _col8 (type: smallint), _col9 (type: tinyint), _col10 (type: int), _col11 (type: bigint), _col12 (type: int), _col13 (type: tinyint), _col14 (type: tinyint)
null sort order: zzzzzzzzzzzzzzz
@@ -103,7 +103,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 5461 Data size: 263873 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5461 Data size: 263731 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Map Vectorization:
@@ -131,7 +131,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 9, 14]
- Statistics: Num rows: 5461 Data size: 263873 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5461 Data size: 263731 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 25
Limit Vectorization:
@@ -314,7 +314,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: (((cdouble > 988888.0D) or ((UDFToDouble(ctimestamp2) > -28792.315D) and (cdouble <= 3569.0D))) and ((UDFToDouble(ctimestamp1) <= -28800.0D) or (UDFToInteger(ctinyint) = cint) or (cstring2 like 'ss')) and (ctinyint <> 0Y)) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -323,7 +323,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: FilterExprOrExpr(children: FilterDoubleColGreaterDoubleScalar(col 5:double, val 988888.0), FilterExprAndExpr(children: FilterDoubleColGreaterDoubleScalar(col 13:double, val -28792.315)(children: CastTimestampToDouble(col 9:timestamp) -> 13:double), FilterDoubleColLessEqualDoubleScalar(col 5:double, val 3569.0))), FilterExprOrExpr(children: FilterDoubleColLessEqualDoubleScalar(col 14:double, val -28800.0)(children: CastTimestampToDouble(col 8:timestamp) -> 14:double), FilterLongColEqualLongColumn(col 0:int, col 2:int)(children: col 0:tinyint), FilterStringColLikeStringScalar(col 7:string, pattern ss)), FilterLongColNotEqualLongScalar(col 0:tinyint, val 0))
predicate: (((cdouble > 988888.0D) or ((UDFToDouble(ctimestamp2) > -28792.315D) and (cdouble <= 3569.0D))) and ((UDFToDouble(ctimestamp1) <= -28800.0D) or (UDFToInteger(ctinyint) = cint) or (cstring2 like 'ss')) and (ctinyint <> 0Y)) (type: boolean)
- Statistics: Num rows: 5461 Data size: 263873 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5461 Data size: 263731 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cboolean1 (type: boolean), cbigint (type: bigint), csmallint (type: smallint), ctinyint (type: tinyint), ctimestamp1 (type: timestamp), cstring1 (type: string), (cbigint + cbigint) (type: bigint), (UDFToInteger(csmallint) % -257) (type: int), (- csmallint) (type: smallint), (- ctinyint) (type: tinyint), (UDFToInteger((- ctinyint)) + 17) (type: int), (cbigint * UDFToLong((- csmallint))) (type: bigint), (cint % UDFToInteger(csmallint)) (type: int), (- ctinyint) (type: tinyint), ((- ctinyint) % ctinyint) (type: tinyint)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14
@@ -332,7 +332,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [10, 3, 1, 0, 8, 6, 15, 16, 17, 18, 20, 22, 23, 24, 26]
selectExpressions: LongColAddLongColumn(col 3:bigint, col 3:bigint) -> 15:bigint, LongColModuloLongScalar(col 1:int, val -257)(children: col 1:smallint) -> 16:int, LongColUnaryMinus(col 1:smallint) -> 17:smallint, LongColUnaryMinus(col 0:tinyint) -> 18:tinyint, LongColAddLongScalar(col 19:int, val 17)(children: LongColUnaryMinus(col 0:tinyint) -> 19:tinyint) -> 20:int, LongColMultiplyLongColumn(col 3:bigint, col 21:bigint)(children: LongColUnaryMinus(col 1:smallint) -> 21:smallint) -> 22:bigint, LongColModuloLongColumn(col 2:int, col 1:int)(children: col 1:smallint) -> 23:int, LongColUnaryMinus(col 0:tinyint) -> 24:tinyint, LongColModuloLongColumn(col 25:tinyint, col 0:tinyint)(children: LongColUnaryMinus(col 0:tinyint) -> 25:tinyint) -> 26:tinyint
- Statistics: Num rows: 5461 Data size: 263873 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5461 Data size: 263731 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: boolean), _col1 (type: bigint), _col2 (type: smallint), _col3 (type: tinyint), _col4 (type: timestamp), _col5 (type: string), _col6 (type: bigint), _col7 (type: int), _col8 (type: smallint), _col9 (type: tinyint), _col10 (type: int), _col11 (type: bigint), _col12 (type: int), _col13 (type: tinyint), _col14 (type: tinyint)
null sort order: zzzzzzzzzzzzzzz
@@ -341,7 +341,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 5461 Data size: 263873 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5461 Data size: 263731 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Map Vectorization:
@@ -369,7 +369,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 9, 14]
- Statistics: Num rows: 5461 Data size: 263873 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 5461 Data size: 263731 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 25
Limit Vectorization:
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_8.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_8.q.out
index 07dab14..62d6071 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_8.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_8.q.out
@@ -72,7 +72,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((cfloat < -6432.0) or ((UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D) and cstring2 is not null) or ((cdouble = 988888.0D) and cboolean1 is not null)) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -81,7 +81,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprOrExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: FilterDoubleColLessEqualDoubleScalar(col 13:double, val 10.0)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 14:double, val 16.0)(children: CastTimestampToDouble(col 9:timestamp) -> 14:double), SelectColumnIsNotNull(col 7:string)), FilterExprAndExpr(children: FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0), SelectColumnIsNotNull(col 10:boolean)))
predicate: ((cfloat < -6432.0) or ((UDFToDouble(ctimestamp1) <= 10.0D) and (UDFToDouble(ctimestamp2) <> 16.0D) and cstring2 is not null) or ((cdouble = 988888.0D) and cboolean1 is not null)) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
@@ -90,7 +90,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [8, 5, 10, 6, 4, 15, 16, 17, 19, 22, 23, 24, 25, 29]
selectExpressions: DoubleColUnaryMinus(col 5:double) -> 15:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 16:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 17:double, DoubleColAddDoubleColumn(col 18:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 18:float) -> 19:float, DoubleColAddDoubleColumn(col 20:double, col 21:double)(children: DoubleColUnaryMinus(col 5:double) -> 20:double, CastLongToDouble(col 3:bigint) -> 21:double) -> 22:double, DoubleColUnaryMinus(col 5:double) -> 23:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 24:float, DoubleColUnaryMinus(col 4:float) -> 25:float, DoubleColAddDoubleColumn(col 26:double, col 28:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 26:double, DoubleColAddDoubleColumn(col 27:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 27:float) -> 28:float) -> 29:double
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: timestamp), _col1 (type: double), _col2 (type: boolean), _col3 (type: string), _col4 (type: float), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: float), _col13 (type: double)
null sort order: zzzzzzzzzzzzzz
@@ -99,7 +99,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Map Vectorization:
@@ -127,7 +127,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 5, 11, 12, 13]
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 20
Limit Vectorization:
@@ -297,7 +297,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((cfloat < -6432.0) or ((UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D) and cstring2 is not null) or ((cdouble = 988888.0D) and cboolean1 is not null)) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -306,7 +306,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprOrExpr(children: FilterDoubleColLessDoubleScalar(col 4:float, val -6432.0), FilterExprAndExpr(children: FilterDoubleColLessEqualDoubleScalar(col 13:double, val 12.503)(children: CastTimestampToDouble(col 8:timestamp) -> 13:double), FilterDoubleColNotEqualDoubleScalar(col 14:double, val 11.998)(children: CastTimestampToDouble(col 9:timestamp) -> 14:double), SelectColumnIsNotNull(col 7:string)), FilterExprAndExpr(children: FilterDoubleColEqualDoubleScalar(col 5:double, val 988888.0), SelectColumnIsNotNull(col 10:boolean)))
predicate: ((cfloat < -6432.0) or ((UDFToDouble(ctimestamp1) <= 12.503D) and (UDFToDouble(ctimestamp2) <> 11.998D) and cstring2 is not null) or ((cdouble = 988888.0D) and cboolean1 is not null)) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctimestamp1 (type: timestamp), cdouble (type: double), cboolean1 (type: boolean), cstring1 (type: string), cfloat (type: float), (- cdouble) (type: double), (-5638.15D - cdouble) (type: double), (cdouble * -257.0D) (type: double), (UDFToFloat(cint) + cfloat) (type: float), ((- cdouble) + UDFToDouble(cbigint)) (type: double), (- cdouble) (type: double), (-1.389 - cfloat) (type: float), (- cfloat) (type: float), ((-5638.15D - cdouble) + UDFToDouble((UDFToFloat(cint) + cfloat))) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
@@ -315,7 +315,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [8, 5, 10, 6, 4, 15, 16, 17, 19, 22, 23, 24, 25, 29]
selectExpressions: DoubleColUnaryMinus(col 5:double) -> 15:double, DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 16:double, DoubleColMultiplyDoubleScalar(col 5:double, val -257.0) -> 17:double, DoubleColAddDoubleColumn(col 18:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 18:float) -> 19:float, DoubleColAddDoubleColumn(col 20:double, col 21:double)(children: DoubleColUnaryMinus(col 5:double) -> 20:double, CastLongToDouble(col 3:bigint) -> 21:double) -> 22:double, DoubleColUnaryMinus(col 5:double) -> 23:double, DoubleScalarSubtractDoubleColumn(val -1.3890000581741333, col 4:float) -> 24:float, DoubleColUnaryMinus(col 4:float) -> 25:float, DoubleColAddDoubleColumn(col 26:double, col 28:double)(children: DoubleScalarSubtractDoubleColumn(val -5638.15, col 5:double) -> 26:double, DoubleColAddDoubleColumn(col 27:float, col 4:float)(children: CastLongToFloatViaLongToDouble(col 2:int) -> 27:float) -> 28:float) -> 29:double
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: timestamp), _col1 (type: double), _col2 (type: boolean), _col3 (type: string), _col4 (type: float), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: float), _col9 (type: double), _col10 (type: double), _col11 (type: float), _col12 (type: float), _col13 (type: double)
null sort order: zzzzzzzzzzzzzz
@@ -324,7 +324,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Map Vectorization:
@@ -352,7 +352,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 5, 11, 12, 13]
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 20
Limit Vectorization:
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_9.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_9.q.out
index 6658e64..cc5275d 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_9.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_9.q.out
@@ -62,7 +62,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((cstring2 like '%b%') and ((cdouble >= -1.389D) or (cstring1 < 'a'))) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -71,7 +71,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: FilterStringColLikeStringScalar(col 7:string, pattern %b%), FilterExprOrExpr(children: FilterDoubleColGreaterEqualDoubleScalar(col 5:double, val -1.389), FilterStringGroupColLessStringScalar(col 6:string, val a)))
predicate: ((cstring2 like '%b%') and ((cdouble >= -1.389D) or (cstring1 < 'a'))) (type: boolean)
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cstring1 (type: string), cdouble (type: double), ctimestamp1 (type: timestamp), (cdouble * cdouble) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
@@ -80,7 +80,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [6, 5, 8, 13]
selectExpressions: DoubleColMultiplyDoubleColumn(col 5:double, col 5:double) -> 13:double
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(_col1), sum(_col3), sum(_col1), min(_col1)
Group By Vectorization:
@@ -95,7 +95,7 @@ STAGE PLANS:
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: double), _col2 (type: timestamp)
null sort order: aaa
@@ -105,7 +105,7 @@ STAGE PLANS:
className: VectorReduceSinkMultiKeyOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
value expressions: _col3 (type: bigint), _col4 (type: double), _col5 (type: double), _col6 (type: double)
Execution mode: vectorized
Map Vectorization:
@@ -139,7 +139,7 @@ STAGE PLANS:
keys: KEY._col0 (type: string), KEY._col1 (type: double), KEY._col2 (type: timestamp)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 2048 Data size: 98958 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2048 Data size: 98905 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: double), _col2 (type: timestamp), (_col1 - 9763215.5639D) (type: double), (- (_col1 - 9763215.5639D)) (type: double), _col3 (type: bigint), power(((_col4 - ((_col5 * _col5) / _col3)) / CASE WHEN ((_col3 = 1L)) THEN (null) ELSE ((_col3 - 1)) END), 0.5) (type: double), (- power(((_col4 - ((_col5 * _col5) / _col3)) / CASE WHEN ((_col3 = 1L)) THEN (null) ELSE ((_col3 - 1)) END), 0.5)) (type: double), (power(((_col4 - ((_col5 * _col5) / _col3)) / CASE WHEN ((_col3 = 1L)) THEN (null) ELSE ((_col3 - 1)) END), 0.5) * UDFToDouble(_col3)) (type: double), _col6 (type: double), (9763215.5639D / _col1) (type: double), (CAST( _col3 AS decimal(19,0)) / -1.389) (type: decimal(28,6)), power(((_col4 - ((_col5 * _col5) / _col3)) / CASE WHEN ((_col3 = 1L)) THEN (null) ELSE ((_col3 - 1)) END), 0.5) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
@@ -148,13 +148,13 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [0, 1, 2, 7, 9, 3, 17, 26, 36, 6, 37, 39, 47]
selectExpressions: DoubleColSubtractDoubleScalar(col 1:double, val 9763215.5639) -> 7:double, DoubleColUnaryMinus(col 8:double)(children: DoubleColSubtractDoubleScalar(col 1:double, val 9763215.5639) -> 8:double) -> 9:double, FuncPowerDoubleToDouble(col 16:double)(children: DoubleColDivideLongColumn(col 12:double, col 15:bigint)(children: DoubleColSubtractDoubleColumn(col 4:double, col 11:double)(children: DoubleColDivideLongColumn(col 10:double, col 3:bigint)(children: DoubleColMultiplyDoubleColumn(col 5:double, col 5:double) -> 10:double) -> 11:double) -> 12:double, IfExprNullCondExpr(col 13:boolean, null, col 14:bigint)(children: LongColEqualLongScalar(col 3:bigint, val 1) -> 13:boolean, LongColSubtractLongScalar(col 3:bigint, val 1) -> 14:bigint) -> 15:bigint) -> 16:double) -> 17:double, DoubleColUnaryMinus(col 25:double)(children: FuncPowerDoubleToDouble(col 24:double)(children: DoubleColDivideLongColumn(col 20:double, col 23:bigint)(children: DoubleColSubtractDoubleColumn(col 4:double, col 19:double)(children: DoubleColDivideLongColumn(col 18:double, col 3:bigint)(children: DoubleColMultiplyDoubleColumn(col 5:double, col 5:double) -> 18:double) -> 19:double) -> 20:double, IfExprNullCondExpr(col 21:boolean, null, col 22:bigint)(children: LongColEqualLongScalar(col 3:bigint, val 1) -> 21:boolean, LongColSubtractLongScalar(col 3:bigint, val 1) -> 22:bigint) -> 23:bigint) -> 24:double) -> 25:double) -> 26:double, DoubleColMultiplyDoubleColumn(col 34:double, col 35:double)(children: FuncPowerDoubleToDouble(col 33:double)(children: DoubleColDivideLongColumn(col 29:double, col 32:bigint)(children: DoubleColSubtractDoubleColumn(col 4:double, col 28:double)(children: DoubleColDivideLongColumn(col 27:double, col 3:bigint)(children: DoubleColMultiplyDoubleColumn(col 5:double, col 5:double) -> 27:double) -> 28:double) -> 29:double, IfExprNullCondExpr(col 30:boolean, null, col 31:bigint)(children: LongColEqualLongScalar(col 3:bigint, val 1) -> 30:boolean, LongColSubtractLongScalar(col 3:bigint, val 1) -> 31:bigint) -> 32:bigint) -> 33:double) -> 34:double, CastLongToDouble(col 3:bigint) -> 35:double) -> 36:double, DoubleScalarDivideDoubleColumn(val 9763215.5639, col 1:double) -> 37:double, DecimalColDivideDecimalScalar(col 38:decimal(19,0), val -1.389)(children: CastLongToDecimal(col 3:bigint) -> 38:decimal(19,0)) -> 39:decimal(28,6), FuncPowerDoubleToDouble(col 46:double)(children: DoubleColDivideLongColumn(col 42:double, col 45:bigint)(children: DoubleColSubtractDoubleColumn(col 4:double, col 41:double)(children: DoubleColDivideLongColumn(col 40:double, col 3:bigint)(children: DoubleColMultiplyDoubleColumn(col 5:double, col 5:double) -> 40:double) -> 41:double) -> 42:double, IfExprNullCondExpr(col 43:boolean, null, col 44:bigint)(children: LongColEqualLongScalar(col 3:bigint, val 1) -> 43:boolean, LongColSubtractLongScalar(col 3:bigint, val 1) -> 44:bigint) -> 45:bigint) -> 46:double) -> 47:double
- Statistics: Num rows: 2048 Data size: 98958 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2048 Data size: 98905 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
File Sink Vectorization:
className: VectorFileSinkOperator
native: false
- Statistics: Num rows: 2048 Data size: 98958 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2048 Data size: 98905 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_decimal_date.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_decimal_date.q.out
index 2ba16e2..615eca9 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_decimal_date.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_decimal_date.q.out
@@ -38,7 +38,7 @@ STAGE PLANS:
TableScan
alias: date_decimal_test_parquet
filterExpr: (cint is not null and cdouble is not null) (type: boolean)
- Statistics: Num rows: 12288 Data size: 176614 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 176484 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -47,7 +47,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: SelectColumnIsNotNull(col 0:int), SelectColumnIsNotNull(col 1:double))
predicate: (cint is not null and cdouble is not null) (type: boolean)
- Statistics: Num rows: 12288 Data size: 176614 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 176484 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cdate (type: date), cdecimal (type: decimal(20,10))
outputColumnNames: _col0, _col1
@@ -55,7 +55,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [2, 3]
- Statistics: Num rows: 12288 Data size: 176614 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 176484 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 10
Limit Vectorization:
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_div0.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_div0.q.out
index 98637c0..4e43490 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_div0.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_div0.q.out
@@ -25,7 +25,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesparquet
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Select Operator
@@ -36,7 +36,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [13]
selectExpressions: DoubleColDivideDoubleScalar(col 5:double, val 0.0) -> 13:double
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
Limit Vectorization:
@@ -210,7 +210,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((cbigint > 0L) and (cbigint < 100000000L)) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -219,7 +219,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: FilterLongColGreaterLongScalar(col 3:bigint, val 0), FilterLongColLessLongScalar(col 3:bigint, val 100000000))
predicate: ((cbigint > 0L) and (cbigint < 100000000L)) (type: boolean)
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (cbigint - 988888L) (type: bigint), (cdouble / UDFToDouble((cbigint - 988888L))) (type: double), (1.2 / CAST( (cbigint - 988888L) AS decimal(19,0))) (type: decimal(22,21))
outputColumnNames: _col0, _col1, _col2
@@ -228,7 +228,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [13, 16, 19]
selectExpressions: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 13:bigint, DoubleColDivideDoubleColumn(col 5:double, col 15:double)(children: CastLongToDouble(col 14:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 14:bigint) -> 15:double) -> 16:double, DecimalScalarDivideDecimalColumn(val 1.2, col 18:decimal(19,0))(children: CastLongToDecimal(col 17:bigint)(children: LongColSubtractLongScalar(col 3:bigint, val 988888) -> 17:bigint) -> 18:decimal(19,0)) -> 19:decimal(22,21)
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: bigint), _col1 (type: double)
null sort order: zz
@@ -237,7 +237,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col2 (type: decimal(22,21))
Execution mode: vectorized
@@ -266,7 +266,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1, 2]
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
Limit Vectorization:
@@ -432,7 +432,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((cdouble >= -500.0D) and (cdouble < -199.0D)) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -441,7 +441,7 @@ STAGE PLANS:
native: true
predicateExpression: FilterExprAndExpr(children: FilterDoubleColGreaterEqualDoubleScalar(col 5:double, val -500.0), FilterDoubleColLessDoubleScalar(col 5:double, val -199.0))
predicate: ((cdouble >= -500.0D) and (cdouble < -199.0D)) (type: boolean)
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (cdouble + 200.0D) (type: double), (UDFToDouble(cbigint) / (cdouble + 200.0D)) (type: double), ((cdouble + 200.0D) / (cdouble + 200.0D)) (type: double), (3.0D / (cdouble + 200.0D)) (type: double), (1.2D / (cdouble + 200.0D)) (type: double)
outputColumnNames: _col0, _col1, _col2, _col4, _col5
@@ -450,7 +450,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [13, 16, 19, 21, 23]
selectExpressions: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 13:double, DoubleColDivideDoubleColumn(col 14:double, col 15:double)(children: CastLongToDouble(col 3:bigint) -> 14:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 15:double) -> 16:double, DoubleColDivideDoubleColumn(col 17:double, col 18:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 17:double, DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 18:double) -> 19:double, DoubleScalarDivideDoubleColumn(val 3.0, col 20:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 20:double) -> 21:double, DoubleScalarDivideDoubleColumn(val 1.2, col 22:double)(children: DoubleColAddDoubleScalar(col 5:double, val 200.0) -> 22:double) -> 23:double
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double), _col1 (type: double)
null sort order: zz
@@ -459,7 +459,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col2 (type: double), _col4 (type: double), _col5 (type: double)
Execution mode: vectorized
@@ -488,7 +488,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1, 2, 1, 3, 4]
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 100
Limit Vectorization:
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_limit.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_limit.q.out
index 7f03cba..e730d5d 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_limit.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_limit.q.out
@@ -25,14 +25,14 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((cint > 0) and (UDFToDouble(cbigint) < cdouble)) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((cint > 0) and (UDFToDouble(cbigint) < cdouble)) (type: boolean)
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint), cdouble (type: double)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 7
Statistics: Num rows: 7 Data size: 336 Basic stats: COMPLETE Column stats: NONE
@@ -106,7 +106,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ctinyint is not null (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -115,7 +115,7 @@ STAGE PLANS:
native: true
predicateExpression: SelectColumnIsNotNull(col 0:tinyint)
predicate: ctinyint is not null (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), cdouble (type: double), csmallint (type: smallint)
outputColumnNames: _col0, _col1, _col2
@@ -123,7 +123,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 5, 1]
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint), _col1 (type: double)
null sort order: zz
@@ -132,7 +132,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.3
value expressions: _col2 (type: smallint)
Execution mode: vectorized
@@ -161,7 +161,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1, 2]
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 20
Limit Vectorization:
@@ -242,7 +242,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesparquet
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Select Operator
@@ -253,7 +253,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [0, 13]
selectExpressions: DoubleColAddDoubleScalar(col 5:double, val 1.0) -> 13:double
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(_col1), count(_col1)
Group By Vectorization:
@@ -268,7 +268,7 @@ STAGE PLANS:
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
null sort order: z
@@ -278,7 +278,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.3
value expressions: _col1 (type: double), _col2 (type: bigint)
Execution mode: vectorized
@@ -313,7 +313,7 @@ STAGE PLANS:
keys: KEY._col0 (type: tinyint)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: tinyint), (_col1 / _col2) (type: double)
outputColumnNames: _col0, _col1
@@ -322,7 +322,7 @@ STAGE PLANS:
native: true
projectedOutputColumnNums: [0, 3]
selectExpressions: DoubleColDivideLongColumn(col 1:double, col 2:bigint) -> 3:double
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 20
Limit Vectorization:
@@ -403,7 +403,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesparquet
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Select Operator
@@ -413,7 +413,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0]
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
Group By Vectorization:
className: VectorGroupByOperator
@@ -426,7 +426,7 @@ STAGE PLANS:
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
null sort order: a
@@ -436,7 +436,7 @@ STAGE PLANS:
className: VectorReduceSinkLongOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.3
Execution mode: vectorized
Map Vectorization:
@@ -468,7 +468,7 @@ STAGE PLANS:
keys: KEY._col0 (type: tinyint)
mode: mergepartial
outputColumnNames: _col0
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 20
Limit Vectorization:
@@ -549,7 +549,7 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesparquet
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Select Operator
@@ -559,7 +559,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 5]
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
Group By Vectorization:
className: VectorGroupByOperator
@@ -572,7 +572,7 @@ STAGE PLANS:
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint), _col1 (type: double)
null sort order: za
@@ -582,7 +582,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Map Vectorization:
enabled: true
@@ -613,7 +613,7 @@ STAGE PLANS:
keys: KEY._col0 (type: tinyint), KEY._col1 (type: double)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count(_col1)
Group By Vectorization:
@@ -627,7 +627,7 @@ STAGE PLANS:
keys: _col0 (type: tinyint)
mode: complete
outputColumnNames: _col0, _col1
- Statistics: Num rows: 3072 Data size: 148437 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3072 Data size: 148358 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 20
Limit Vectorization:
@@ -742,7 +742,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ctinyint is not null (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -751,7 +751,7 @@ STAGE PLANS:
native: true
predicateExpression: SelectColumnIsNotNull(col 0:tinyint)
predicate: ctinyint is not null (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(ctinyint)
Group By Vectorization:
@@ -766,7 +766,7 @@ STAGE PLANS:
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
null sort order: a
@@ -776,7 +776,7 @@ STAGE PLANS:
className: VectorReduceSinkMultiKeyOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Execution mode: vectorized
Map Vectorization:
@@ -810,7 +810,7 @@ STAGE PLANS:
keys: KEY._col0 (type: double)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col1 (type: bigint), _col0 (type: double)
null sort order: zz
@@ -819,7 +819,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.3
Reducer 3
Execution mode: vectorized
@@ -837,7 +837,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [1, 0]
- Statistics: Num rows: 6144 Data size: 296875 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 296716 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 20
Limit Vectorization:
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_offset_limit.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_offset_limit.q.out
index 804627e..d808811 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_offset_limit.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_offset_limit.q.out
@@ -25,14 +25,14 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ((cint > 0) and (UDFToDouble(cbigint) < cdouble)) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: ((cint > 0) and (UDFToDouble(cbigint) < cdouble)) (type: boolean)
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint), cdouble (type: double)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1365 Data size: 65956 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1365 Data size: 65920 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 2
Offset of rows: 3
@@ -102,7 +102,7 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: ctinyint is not null (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TableScan Vectorization:
native: true
Filter Operator
@@ -111,7 +111,7 @@ STAGE PLANS:
native: true
predicateExpression: SelectColumnIsNotNull(col 0:tinyint)
predicate: ctinyint is not null (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), cdouble (type: double), csmallint (type: smallint)
outputColumnNames: _col0, _col1, _col2
@@ -119,7 +119,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 5, 1]
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint), _col1 (type: double)
null sort order: zz
@@ -128,7 +128,7 @@ STAGE PLANS:
className: VectorReduceSinkObjectHashOperator
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine spark IN [tez, spark] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
value expressions: _col2 (type: smallint)
Execution mode: vectorized
@@ -157,7 +157,7 @@ STAGE PLANS:
className: VectorSelectOperator
native: true
projectedOutputColumnNums: [0, 1, 2]
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 3
Limit Vectorization:
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_part_project.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_part_project.q.out
index 146ce04..35d37b8 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_part_project.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_part_project.q.out
@@ -77,16 +77,16 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypesparquet_part_n0
- Statistics: Num rows: 200 Data size: 9800 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 200 Data size: 9196 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (cdouble + 2.0D) (type: double)
outputColumnNames: _col0
- Statistics: Num rows: 200 Data size: 9800 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 200 Data size: 9196 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: double)
null sort order: z
sort order: +
- Statistics: Num rows: 200 Data size: 9800 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 200 Data size: 9196 Basic stats: COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Map Vectorization:
@@ -110,13 +110,13 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: double)
outputColumnNames: _col0
- Statistics: Num rows: 200 Data size: 9800 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 200 Data size: 9196 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 10
- Statistics: Num rows: 10 Data size: 490 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 450 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 10 Data size: 490 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 10 Data size: 450 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git ql/src/test/results/clientpositive/spark/parquet_vectorization_pushdown.q.out ql/src/test/results/clientpositive/spark/parquet_vectorization_pushdown.q.out
index 333c687..9c4cec7 100644
--- ql/src/test/results/clientpositive/spark/parquet_vectorization_pushdown.q.out
+++ ql/src/test/results/clientpositive/spark/parquet_vectorization_pushdown.q.out
@@ -27,14 +27,14 @@ STAGE PLANS:
TableScan
alias: alltypesparquet
filterExpr: (UDFToDouble(cbigint) < cdouble) (type: boolean)
- Statistics: Num rows: 12288 Data size: 593751 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 593433 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (UDFToDouble(cbigint) < cdouble) (type: boolean)
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cbigint (type: bigint)
outputColumnNames: cbigint
- Statistics: Num rows: 4096 Data size: 197917 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4096 Data size: 197811 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: sum(cbigint), count(cbigint)
minReductionHashAggr: 0.99
diff --git ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out
index 43ac991..dfedca6 100644
--- ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out
+++ ql/src/test/results/clientpositive/spark/spark_dynamic_partition_pruning.q.out
@@ -7274,14 +7274,14 @@ STAGE PLANS:
TableScan
alias: srcpart_parquet
filterExpr: UDFToDouble(hr) is not null (type: boolean)
- Statistics: Num rows: 2000 Data size: 23372 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2000 Data size: 23184 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: UDFToDouble(hr) is not null (type: boolean)
- Statistics: Num rows: 2000 Data size: 23372 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2000 Data size: 23184 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ds (type: string), UDFToDouble(hr) (type: double)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2000 Data size: 23372 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2000 Data size: 23184 Basic stats: COMPLETE Column stats: NONE
Map Join Operator
condition map:
Inner Join 0 to 1
@@ -7290,7 +7290,7 @@ STAGE PLANS:
1 _col0 (type: string), _col1 (type: double)
input vertices:
1 Map 3
- Statistics: Num rows: 2200 Data size: 25709 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2200 Data size: 25502 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
minReductionHashAggr: 0.99
diff --git ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out
index 9677116..4d2e047 100644
--- ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out
+++ ql/src/test/results/clientpositive/spark/vectorization_input_format_excludes.q.out
@@ -78,14 +78,14 @@ STAGE PLANS:
TableScan
alias: alltypes_parquet_n0
filterExpr: (cint = 528534767) (type: boolean)
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (cint = 528534767) (type: boolean)
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: 528534767 (type: int), ctinyint (type: tinyint), csmallint (type: smallint), cfloat (type: float), cdouble (type: double), cstring1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 10
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
@@ -178,24 +178,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypes_parquet_n0
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), cint (type: int), csmallint (type: smallint), cstring1 (type: string), cfloat (type: float), cdouble (type: double), (cdouble * cdouble) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: max(_col1), min(_col2), count(_col3), sum(_col4), count(_col4), sum(_col6), sum(_col5), count(_col5)
keys: _col0 (type: tinyint)
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: double), _col5 (type: bigint), _col6 (type: double), _col7 (type: double), _col8 (type: bigint)
Execution mode: vectorized
Map Vectorization:
@@ -221,14 +221,14 @@ STAGE PLANS:
keys: KEY._col0 (type: tinyint)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), (_col4 / _col5) (type: double), power(((_col6 - ((_col7 * _col7) / _col8)) / _col8), 0.5) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -422,14 +422,14 @@ STAGE PLANS:
TableScan
alias: alltypes_parquet_n0
filterExpr: (cint = 528534767) (type: boolean)
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (cint = 528534767) (type: boolean)
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: 528534767 (type: int), ctinyint (type: tinyint), csmallint (type: smallint), cfloat (type: float), cdouble (type: double), cstring1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 10
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
@@ -516,24 +516,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypes_parquet_n0
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), cint (type: int), csmallint (type: smallint), cstring1 (type: string), cfloat (type: float), cdouble (type: double), (cdouble * cdouble) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: max(_col1), min(_col2), count(_col3), sum(_col4), count(_col4), sum(_col6), sum(_col5), count(_col5)
keys: _col0 (type: tinyint)
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: double), _col5 (type: bigint), _col6 (type: double), _col7 (type: double), _col8 (type: bigint)
Map Vectorization:
enabled: false
@@ -553,14 +553,14 @@ STAGE PLANS:
keys: KEY._col0 (type: tinyint)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), (_col4 / _col5) (type: double), power(((_col6 - ((_col7 * _col7) / _col8)) / _col8), 0.5) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -754,14 +754,14 @@ STAGE PLANS:
TableScan
alias: alltypes_parquet_n0
filterExpr: (cint = 528534767) (type: boolean)
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (cint = 528534767) (type: boolean)
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: 528534767 (type: int), ctinyint (type: tinyint), csmallint (type: smallint), cfloat (type: float), cdouble (type: double), cstring1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
Limit
Number of rows: 10
Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
@@ -854,24 +854,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: alltypes_parquet_n0
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctinyint (type: tinyint), cint (type: int), csmallint (type: smallint), cstring1 (type: string), cfloat (type: float), cdouble (type: double), (cdouble * cdouble) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: max(_col1), min(_col2), count(_col3), sum(_col4), count(_col4), sum(_col6), sum(_col5), count(_col5)
keys: _col0 (type: tinyint)
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: tinyint)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: tinyint)
- Statistics: Num rows: 12288 Data size: 256791 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12288 Data size: 256593 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), _col4 (type: double), _col5 (type: bigint), _col6 (type: double), _col7 (type: double), _col8 (type: bigint)
Execution mode: vectorized
Map Vectorization:
@@ -897,14 +897,14 @@ STAGE PLANS:
keys: KEY._col0 (type: tinyint)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: tinyint), _col1 (type: int), _col2 (type: smallint), _col3 (type: bigint), (_col4 / _col5) (type: double), power(((_col6 - ((_col7 * _col7) / _col8)) / _col8), 0.5) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 6144 Data size: 128395 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 6144 Data size: 128296 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1573,17 +1573,17 @@ STAGE PLANS:
TableScan
alias: parquettbl
filterExpr: (UDFToInteger((t1 + t2)) > 10) (type: boolean)
- Statistics: Num rows: 3 Data size: 126 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 3 Data size: 70 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (UDFToInteger((t1 + t2)) > 10) (type: boolean)
- Statistics: Num rows: 1 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 23 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: t1 (type: tinyint), t2 (type: tinyint), (t1 + t2) (type: tinyint)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 23 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 23 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git ql/src/test/results/clientpositive/spark/vectorization_parquet_projection.q.out ql/src/test/results/clientpositive/spark/vectorization_parquet_projection.q.out
index 7808f85..2dd3af6 100644
--- ql/src/test/results/clientpositive/spark/vectorization_parquet_projection.q.out
+++ ql/src/test/results/clientpositive/spark/vectorization_parquet_projection.q.out
@@ -139,14 +139,14 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: parquet_project_test
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), m1 (type: map)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -221,9 +221,9 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: parquet_project_test
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
minReductionHashAggr: 0.99
@@ -311,24 +311,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: parquet_project_test
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int)
outputColumnNames: cint
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
keys: cint (type: int)
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: int)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: int)
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Execution mode: vectorized
Map Vectorization:
@@ -354,10 +354,10 @@ STAGE PLANS:
keys: KEY._col0 (type: int)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 11 Data size: 354 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 222 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 11 Data size: 354 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 222 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -411,24 +411,24 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: parquet_project_test
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: m1['color'] (type: string)
outputColumnNames: _col0
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
keys: _col0 (type: string)
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Map Vectorization:
enabled: true
@@ -450,10 +450,10 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 11 Data size: 354 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 222 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 11 Data size: 354 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 11 Data size: 222 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -530,9 +530,9 @@ STAGE PLANS:
TableScan
alias: parquet_nullsplit
filterExpr: (len = '1') (type: boolean)
- Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 60 Basic stats: COMPLETE Column stats: NONE
Select Operator
- Statistics: Num rows: 1 Data size: 108 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 1 Data size: 60 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
minReductionHashAggr: 0.99
diff --git ql/src/test/results/clientpositive/vectorization_parquet_projection.q.out ql/src/test/results/clientpositive/vectorization_parquet_projection.q.out
index ca211b3..cb2e1cc 100644
--- ql/src/test/results/clientpositive/vectorization_parquet_projection.q.out
+++ ql/src/test/results/clientpositive/vectorization_parquet_projection.q.out
@@ -213,9 +213,9 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: parquet_project_test
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 22 Data size: 709 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 22 Data size: 445 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
minReductionHashAggr: 0.99
@@ -495,9 +495,9 @@ STAGE PLANS:
TableScan
alias: parquet_nullsplit
filterExpr: (len = '1') (type: boolean)
- Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 68 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- Statistics: Num rows: 1 Data size: 116 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 1 Data size: 68 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count()
minReductionHashAggr: 0.99