diff --git a/ql/src/test/queries/clientpositive/orc_llap_counters.q b/ql/src/test/queries/clientpositive/orc_llap_counters.q index cc0e991..2d8e3b1 100644 --- a/ql/src/test/queries/clientpositive/orc_llap_counters.q +++ b/ql/src/test/queries/clientpositive/orc_llap_counters.q @@ -4,6 +4,8 @@ SET hive.optimize.index.filter=true; SET hive.cbo.enable=false; SET hive.vectorized.execution.enabled=true; SET hive.llap.io.enabled=true; +SET hive.map.aggr=false; +-- disabling map side aggregation as that can lead to different intermediate record counts CREATE TABLE staging(t tinyint, si smallint, @@ -33,17 +35,16 @@ CREATE TABLE orc_ppd_staging(t tinyint, c char(50), v varchar(50), da date, - ts timestamp, dec decimal(4,2), bin binary) STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*"); -insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from staging order by t, s; +insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), dec, bin from staging order by t, s; -- just to introduce a gap in min/max range for bloom filters. The dataset has contiguous values -- which makes it hard to test bloom filters -insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11","1990-03-11 10:11:58.703308",-71.54,"aaa" from staging limit 1; -insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11","2023-03-11 10:11:58.703308",71.54,"zzz" from staging limit 1; +insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11",-71.54,"aaa" from staging limit 1; +insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11",71.54,"zzz" from staging limit 1; CREATE TABLE orc_ppd(t tinyint, si smallint, @@ -56,12 +57,11 @@ CREATE TABLE orc_ppd(t tinyint, c char(50), v varchar(50), da date, - ts timestamp, dec decimal(4,2), bin binary) STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*"); -insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from orc_ppd_staging order by t, s; +insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, dec, bin from orc_ppd_staging order by t, s; describe formatted orc_ppd; diff --git a/ql/src/test/queries/clientpositive/orc_llap_counters1.q b/ql/src/test/queries/clientpositive/orc_llap_counters1.q index 06d6c4f..1e63c4f 100644 --- a/ql/src/test/queries/clientpositive/orc_llap_counters1.q +++ b/ql/src/test/queries/clientpositive/orc_llap_counters1.q @@ -3,6 +3,8 @@ SET hive.optimize.index.filter=true; SET hive.cbo.enable=false; SET hive.vectorized.execution.enabled=true; SET hive.llap.io.enabled=true; +SET hive.map.aggr=false; +-- disabling map side aggregation as that can lead to different intermediate record counts CREATE TABLE staging(t tinyint, si smallint, @@ -32,17 +34,16 @@ CREATE TABLE orc_ppd_staging(t tinyint, c char(50), v varchar(50), da date, - ts timestamp, dec decimal(4,2), bin binary) STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*"); -insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from staging order by t, s; +insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), dec, bin from staging order by t, s; -- just to introduce a gap in min/max range for bloom filters. The dataset has contiguous values -- which makes it hard to test bloom filters -insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11","1990-03-11 10:11:58.703308",-71.54,"aaa" from staging limit 1; -insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11","2023-03-11 10:11:58.703308",71.54,"zzz" from staging limit 1; +insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11",-71.54,"aaa" from staging limit 1; +insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11",71.54,"zzz" from staging limit 1; CREATE TABLE orc_ppd(t tinyint, si smallint, @@ -55,12 +56,11 @@ CREATE TABLE orc_ppd(t tinyint, c char(50), v varchar(50), da date, - ts timestamp, dec decimal(4,2), bin binary) STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*"); -insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from orc_ppd_staging order by t, s; +insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, dec, bin from orc_ppd_staging order by t, s; describe formatted orc_ppd; diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out index e1c1dd8..bc3915a 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out @@ -57,7 +57,6 @@ PREHOOK: query: CREATE TABLE orc_ppd_staging(t tinyint, c char(50), v varchar(50), da date, - ts timestamp, dec decimal(4,2), bin binary) STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*") @@ -75,18 +74,17 @@ POSTHOOK: query: CREATE TABLE orc_ppd_staging(t tinyint, c char(50), v varchar(50), da date, - ts timestamp, dec decimal(4,2), bin binary) STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@orc_ppd_staging -PREHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from staging order by t, s +PREHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), dec, bin from staging order by t, s PREHOOK: type: QUERY PREHOOK: Input: default@staging PREHOOK: Output: default@orc_ppd_staging -POSTHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from staging order by t, s +POSTHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), dec, bin from staging order by t, s POSTHOOK: type: QUERY POSTHOOK: Input: default@staging POSTHOOK: Output: default@orc_ppd_staging @@ -102,13 +100,12 @@ POSTHOOK: Lineage: orc_ppd_staging.i SIMPLE [(staging)staging.FieldSchema(name:i POSTHOOK: Lineage: orc_ppd_staging.s SIMPLE [(staging)staging.FieldSchema(name:s, type:string, comment:null), ] POSTHOOK: Lineage: orc_ppd_staging.si SIMPLE [(staging)staging.FieldSchema(name:si, type:smallint, comment:null), ] POSTHOOK: Lineage: orc_ppd_staging.t SIMPLE [(staging)staging.FieldSchema(name:t, type:tinyint, comment:null), ] -POSTHOOK: Lineage: orc_ppd_staging.ts SIMPLE [(staging)staging.FieldSchema(name:ts, type:timestamp, comment:null), ] POSTHOOK: Lineage: orc_ppd_staging.v EXPRESSION [(staging)staging.FieldSchema(name:s, type:string, comment:null), ] -PREHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11","1990-03-11 10:11:58.703308",-71.54,"aaa" from staging limit 1 +PREHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11",-71.54,"aaa" from staging limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@staging PREHOOK: Output: default@orc_ppd_staging -POSTHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11","1990-03-11 10:11:58.703308",-71.54,"aaa" from staging limit 1 +POSTHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11",-71.54,"aaa" from staging limit 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@staging POSTHOOK: Output: default@orc_ppd_staging @@ -124,13 +121,12 @@ POSTHOOK: Lineage: orc_ppd_staging.i EXPRESSION [] POSTHOOK: Lineage: orc_ppd_staging.s SIMPLE [] POSTHOOK: Lineage: orc_ppd_staging.si EXPRESSION [] POSTHOOK: Lineage: orc_ppd_staging.t EXPRESSION [] -POSTHOOK: Lineage: orc_ppd_staging.ts EXPRESSION [] POSTHOOK: Lineage: orc_ppd_staging.v EXPRESSION [] -PREHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11","2023-03-11 10:11:58.703308",71.54,"zzz" from staging limit 1 +PREHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11",71.54,"zzz" from staging limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@staging PREHOOK: Output: default@orc_ppd_staging -POSTHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11","2023-03-11 10:11:58.703308",71.54,"zzz" from staging limit 1 +POSTHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11",71.54,"zzz" from staging limit 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@staging POSTHOOK: Output: default@orc_ppd_staging @@ -146,7 +142,6 @@ POSTHOOK: Lineage: orc_ppd_staging.i SIMPLE [] POSTHOOK: Lineage: orc_ppd_staging.s SIMPLE [] POSTHOOK: Lineage: orc_ppd_staging.si EXPRESSION [] POSTHOOK: Lineage: orc_ppd_staging.t EXPRESSION [] -POSTHOOK: Lineage: orc_ppd_staging.ts EXPRESSION [] POSTHOOK: Lineage: orc_ppd_staging.v EXPRESSION [] PREHOOK: query: CREATE TABLE orc_ppd(t tinyint, si smallint, @@ -159,7 +154,6 @@ PREHOOK: query: CREATE TABLE orc_ppd(t tinyint, c char(50), v varchar(50), da date, - ts timestamp, dec decimal(4,2), bin binary) STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*") @@ -177,18 +171,17 @@ POSTHOOK: query: CREATE TABLE orc_ppd(t tinyint, c char(50), v varchar(50), da date, - ts timestamp, dec decimal(4,2), bin binary) STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@orc_ppd -PREHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from orc_ppd_staging order by t, s +PREHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, dec, bin from orc_ppd_staging order by t, s PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_staging PREHOOK: Output: default@orc_ppd -POSTHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from orc_ppd_staging order by t, s +POSTHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, dec, bin from orc_ppd_staging order by t, s POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_ppd_staging POSTHOOK: Output: default@orc_ppd @@ -197,14 +190,13 @@ POSTHOOK: Lineage: orc_ppd.bin SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSch POSTHOOK: Lineage: orc_ppd.bo SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:bo, type:boolean, comment:null), ] POSTHOOK: Lineage: orc_ppd.c EXPRESSION [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:s, type:string, comment:null), ] POSTHOOK: Lineage: orc_ppd.d SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:d, type:double, comment:null), ] -POSTHOOK: Lineage: orc_ppd.da EXPRESSION [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:ts, type:timestamp, comment:null), ] +POSTHOOK: Lineage: orc_ppd.da SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:da, type:date, comment:null), ] POSTHOOK: Lineage: orc_ppd.dec SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:dec, type:decimal(4,2), comment:null), ] POSTHOOK: Lineage: orc_ppd.f SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:f, type:float, comment:null), ] POSTHOOK: Lineage: orc_ppd.i SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:i, type:int, comment:null), ] POSTHOOK: Lineage: orc_ppd.s SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:s, type:string, comment:null), ] POSTHOOK: Lineage: orc_ppd.si SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:si, type:smallint, comment:null), ] POSTHOOK: Lineage: orc_ppd.t SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:t, type:tinyint, comment:null), ] -POSTHOOK: Lineage: orc_ppd.ts SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:ts, type:timestamp, comment:null), ] POSTHOOK: Lineage: orc_ppd.v EXPRESSION [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:s, type:string, comment:null), ] PREHOOK: query: describe formatted orc_ppd PREHOOK: type: DESCTABLE @@ -225,7 +217,6 @@ s string c char(50) v varchar(50) da date -ts timestamp dec decimal(4,2) bin binary @@ -241,8 +232,8 @@ Table Parameters: numRows 2100 orc.bloom.filter.columns * orc.row.index.stride 1000 - rawDataSize 1223514 - totalSize 60010 + rawDataSize 1139514 + totalSize 57100 #### A masked pattern was here #### # Storage Information @@ -260,7 +251,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 16711 + HDFS_BYTES_READ: 16677 HDFS_BYTES_WRITTEN: 104 HDFS_READ_OPS: 5 HDFS_LARGE_READ_OPS: 0 @@ -270,7 +261,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 2100 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_MISS: 2 NUM_DECODED_BATCHES: 1 @@ -297,7 +288,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 1501 + HDFS_BYTES_READ: 1467 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 @@ -307,7 +298,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 8 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 269 @@ -334,7 +325,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 22 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -359,7 +350,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 16 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -384,7 +375,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 18 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -434,7 +425,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 32 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -459,7 +450,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 32 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -484,7 +475,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 1697 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -509,7 +500,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 12 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -534,7 +525,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 1713 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -559,7 +550,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -584,7 +575,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 50 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -609,7 +600,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 318 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0 @@ -682,7 +673,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 0 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 0 Stage-1 LLAP IO COUNTERS: METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 @@ -706,7 +697,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 6166 + HDFS_BYTES_READ: 6132 HDFS_BYTES_WRITTEN: 101 HDFS_READ_OPS: 4 HDFS_LARGE_READ_OPS: 0 @@ -716,7 +707,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 786432 ALLOCATED_USED_BYTES: 11299 @@ -744,7 +735,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -769,7 +760,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -808,7 +799,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 2100 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -853,7 +844,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -878,7 +869,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -903,7 +894,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -928,7 +919,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 81 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -953,7 +944,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 74 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -978,7 +969,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 12 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1003,7 +994,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 2000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 13 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1053,7 +1044,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 7 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1138,7 +1129,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1163,7 +1154,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 6 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 3980 CACHE_MISS_BYTES: 0 @@ -1188,7 +1179,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 1000 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4229 CACHE_MISS_BYTES: 0 @@ -1213,7 +1204,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 2 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 4229 CACHE_MISS_BYTES: 0 diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out index de686b0..5c54622 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out @@ -57,7 +57,6 @@ PREHOOK: query: CREATE TABLE orc_ppd_staging(t tinyint, c char(50), v varchar(50), da date, - ts timestamp, dec decimal(4,2), bin binary) STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*") @@ -75,18 +74,17 @@ POSTHOOK: query: CREATE TABLE orc_ppd_staging(t tinyint, c char(50), v varchar(50), da date, - ts timestamp, dec decimal(4,2), bin binary) STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@orc_ppd_staging -PREHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from staging order by t, s +PREHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), dec, bin from staging order by t, s PREHOOK: type: QUERY PREHOOK: Input: default@staging PREHOOK: Output: default@orc_ppd_staging -POSTHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from staging order by t, s +POSTHOOK: query: insert overwrite table orc_ppd_staging select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), dec, bin from staging order by t, s POSTHOOK: type: QUERY POSTHOOK: Input: default@staging POSTHOOK: Output: default@orc_ppd_staging @@ -102,13 +100,12 @@ POSTHOOK: Lineage: orc_ppd_staging.i SIMPLE [(staging)staging.FieldSchema(name:i POSTHOOK: Lineage: orc_ppd_staging.s SIMPLE [(staging)staging.FieldSchema(name:s, type:string, comment:null), ] POSTHOOK: Lineage: orc_ppd_staging.si SIMPLE [(staging)staging.FieldSchema(name:si, type:smallint, comment:null), ] POSTHOOK: Lineage: orc_ppd_staging.t SIMPLE [(staging)staging.FieldSchema(name:t, type:tinyint, comment:null), ] -POSTHOOK: Lineage: orc_ppd_staging.ts SIMPLE [(staging)staging.FieldSchema(name:ts, type:timestamp, comment:null), ] POSTHOOK: Lineage: orc_ppd_staging.v EXPRESSION [(staging)staging.FieldSchema(name:s, type:string, comment:null), ] -PREHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11","1990-03-11 10:11:58.703308",-71.54,"aaa" from staging limit 1 +PREHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11",-71.54,"aaa" from staging limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@staging PREHOOK: Output: default@orc_ppd_staging -POSTHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11","1990-03-11 10:11:58.703308",-71.54,"aaa" from staging limit 1 +POSTHOOK: query: insert into orc_ppd_staging select -10,-321,-65680,-4294967430,-97.94,-13.07,true,"aaa","aaa","aaa","1990-03-11",-71.54,"aaa" from staging limit 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@staging POSTHOOK: Output: default@orc_ppd_staging @@ -124,13 +121,12 @@ POSTHOOK: Lineage: orc_ppd_staging.i EXPRESSION [] POSTHOOK: Lineage: orc_ppd_staging.s SIMPLE [] POSTHOOK: Lineage: orc_ppd_staging.si EXPRESSION [] POSTHOOK: Lineage: orc_ppd_staging.t EXPRESSION [] -POSTHOOK: Lineage: orc_ppd_staging.ts EXPRESSION [] POSTHOOK: Lineage: orc_ppd_staging.v EXPRESSION [] -PREHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11","2023-03-11 10:11:58.703308",71.54,"zzz" from staging limit 1 +PREHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11",71.54,"zzz" from staging limit 1 PREHOOK: type: QUERY PREHOOK: Input: default@staging PREHOOK: Output: default@orc_ppd_staging -POSTHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11","2023-03-11 10:11:58.703308",71.54,"zzz" from staging limit 1 +POSTHOOK: query: insert into orc_ppd_staging select 127,331,65690,4294967440,107.94,23.07,true,"zzz","zzz","zzz","2023-03-11",71.54,"zzz" from staging limit 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@staging POSTHOOK: Output: default@orc_ppd_staging @@ -146,7 +142,6 @@ POSTHOOK: Lineage: orc_ppd_staging.i SIMPLE [] POSTHOOK: Lineage: orc_ppd_staging.s SIMPLE [] POSTHOOK: Lineage: orc_ppd_staging.si EXPRESSION [] POSTHOOK: Lineage: orc_ppd_staging.t EXPRESSION [] -POSTHOOK: Lineage: orc_ppd_staging.ts EXPRESSION [] POSTHOOK: Lineage: orc_ppd_staging.v EXPRESSION [] PREHOOK: query: CREATE TABLE orc_ppd(t tinyint, si smallint, @@ -159,7 +154,6 @@ PREHOOK: query: CREATE TABLE orc_ppd(t tinyint, c char(50), v varchar(50), da date, - ts timestamp, dec decimal(4,2), bin binary) STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*") @@ -177,18 +171,17 @@ POSTHOOK: query: CREATE TABLE orc_ppd(t tinyint, c char(50), v varchar(50), da date, - ts timestamp, dec decimal(4,2), bin binary) STORED AS ORC tblproperties("orc.row.index.stride" = "1000", "orc.bloom.filter.columns"="*") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@orc_ppd -PREHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from orc_ppd_staging order by t, s +PREHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, dec, bin from orc_ppd_staging order by t, s PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd_staging PREHOOK: Output: default@orc_ppd -POSTHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), cast(ts as date), ts, dec, bin from orc_ppd_staging order by t, s +POSTHOOK: query: insert overwrite table orc_ppd select t, si, i, b, f, d, bo, s, cast(s as char(50)), cast(s as varchar(50)), da, dec, bin from orc_ppd_staging order by t, s POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_ppd_staging POSTHOOK: Output: default@orc_ppd @@ -197,14 +190,13 @@ POSTHOOK: Lineage: orc_ppd.bin SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSch POSTHOOK: Lineage: orc_ppd.bo SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:bo, type:boolean, comment:null), ] POSTHOOK: Lineage: orc_ppd.c EXPRESSION [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:s, type:string, comment:null), ] POSTHOOK: Lineage: orc_ppd.d SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:d, type:double, comment:null), ] -POSTHOOK: Lineage: orc_ppd.da EXPRESSION [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:ts, type:timestamp, comment:null), ] +POSTHOOK: Lineage: orc_ppd.da SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:da, type:date, comment:null), ] POSTHOOK: Lineage: orc_ppd.dec SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:dec, type:decimal(4,2), comment:null), ] POSTHOOK: Lineage: orc_ppd.f SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:f, type:float, comment:null), ] POSTHOOK: Lineage: orc_ppd.i SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:i, type:int, comment:null), ] POSTHOOK: Lineage: orc_ppd.s SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:s, type:string, comment:null), ] POSTHOOK: Lineage: orc_ppd.si SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:si, type:smallint, comment:null), ] POSTHOOK: Lineage: orc_ppd.t SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:t, type:tinyint, comment:null), ] -POSTHOOK: Lineage: orc_ppd.ts SIMPLE [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:ts, type:timestamp, comment:null), ] POSTHOOK: Lineage: orc_ppd.v EXPRESSION [(orc_ppd_staging)orc_ppd_staging.FieldSchema(name:s, type:string, comment:null), ] PREHOOK: query: describe formatted orc_ppd PREHOOK: type: DESCTABLE @@ -225,7 +217,6 @@ s string c char(50) v varchar(50) da date -ts timestamp dec decimal(4,2) bin binary @@ -241,8 +232,8 @@ Table Parameters: numRows 2100 orc.bloom.filter.columns * orc.row.index.stride 1000 - rawDataSize 1223514 - totalSize 60010 + rawDataSize 1139514 + totalSize 57100 #### A masked pattern was here #### # Storage Information @@ -260,7 +251,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@orc_ppd #### A masked pattern was here #### Stage-1 FILE SYSTEM COUNTERS: - HDFS_BYTES_READ: 17909 + HDFS_BYTES_READ: 17875 HDFS_BYTES_WRITTEN: 104 HDFS_READ_OPS: 6 HDFS_LARGE_READ_OPS: 0 @@ -270,7 +261,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 2094 Stage-1 LLAP IO COUNTERS: ALLOCATED_BYTES: 524288 ALLOCATED_USED_BYTES: 269 @@ -296,7 +287,7 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 2100 RECORDS_OUT_0: 1 - RECORDS_OUT_INTERMEDIATE_Map_1: 1 + RECORDS_OUT_INTERMEDIATE_Map_1: 2094 Stage-1 LLAP IO COUNTERS: CACHE_HIT_BYTES: 249 CACHE_MISS_BYTES: 0