diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index 0876bf7..a2692ea 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -72,6 +72,7 @@ private final boolean[] included; private final long rowIndexStride; private long rowInStripe = 0; + private long markerPosition = Long.MAX_VALUE; private int currentStripe = -1; private long rowBaseInStripe = 0; private long rowCountInStripe = 0; @@ -2736,7 +2737,37 @@ public VectorizedRowBatch nextBatch(VectorizedRowBatch previous) throws IOExcept readStripe(); } - long batchSize = Math.min(VectorizedRowBatch.DEFAULT_SIZE, (rowCountInStripe - rowInStripe)); + long batchSize = 0; + + // In case of PPD, batch size should be aware of row group boundaries. If only a subset of row + // groups are selected then marker position is set to the end of range (subset of row groups + // within strip). Batch size computed out of marker position makes sure that batch size is + // aware of row group boundary and will not cause overflow when reading rows + // illustration of this case is here https://issues.apache.org/jira/browse/HIVE-6287 + if (rowIndexStride != 0 && includedRowGroups != null && rowInStripe < rowCountInStripe) { + int startRowGroup = (int) (rowInStripe / rowIndexStride); + if (!includedRowGroups[startRowGroup]) { + while (startRowGroup < includedRowGroups.length && !includedRowGroups[startRowGroup]) { + startRowGroup += 1; + } + } + + int endRowGroup = startRowGroup; + while (endRowGroup < includedRowGroups.length && includedRowGroups[endRowGroup]) { + endRowGroup += 1; + } + + markerPosition = (endRowGroup * rowIndexStride) < rowCountInStripe ? (endRowGroup * rowIndexStride) + : rowCountInStripe; + batchSize = Math.min(VectorizedRowBatch.DEFAULT_SIZE, (markerPosition - rowInStripe)); + + if (LOG.isDebugEnabled() && batchSize < VectorizedRowBatch.DEFAULT_SIZE) { + LOG.debug("markerPosition: " + markerPosition + " batchSize: " + batchSize); + } + } else { + batchSize = Math.min(VectorizedRowBatch.DEFAULT_SIZE, (rowCountInStripe - rowInStripe)); + } + rowInStripe += batchSize; if (previous == null) { ColumnVector[] cols = (ColumnVector[]) reader.nextVector(null, (int) batchSize); diff --git ql/src/test/queries/clientpositive/orc_vectorization_ppd.q ql/src/test/queries/clientpositive/orc_vectorization_ppd.q new file mode 100644 index 0000000..b7a8c86 --- /dev/null +++ ql/src/test/queries/clientpositive/orc_vectorization_ppd.q @@ -0,0 +1,69 @@ +-- create table with 1000 rows +create table srcorc(key string, value string) stored as textfile; +insert overwrite table srcorc select * from src; +insert into table srcorc select * from src; + +-- load table with each row group having 1000 rows and stripe 1 & 2 having 5000 & 2000 rows respectively +create table if not exists vectororc +(s1 string, +s2 string, +d double, +s3 string) +stored as ORC tblproperties("orc.row.index.stride"="1000", "orc.stripe.size"="100000", "orc.compress.size"="10000"); + +-- insert creates separate orc files +insert overwrite table vectororc select "apple", "a", rand(1), "zoo" from srcorc; +insert into table vectororc select null, "b", rand(2), "zoo" from srcorc; +insert into table vectororc select null, "c", rand(3), "zoo" from srcorc; +insert into table vectororc select "apple", "d", rand(4), "zoo" from srcorc; +insert into table vectororc select null, "e", rand(5), "z" from srcorc; +insert into table vectororc select "apple", "f", rand(6), "z" from srcorc; +insert into table vectororc select null, "g", rand(7), "zoo" from srcorc; + +-- since vectororc table has multiple orc file we will load them into a single file using another table +create table if not exists testorc +(s1 string, +s2 string, +d double, +s3 string) +stored as ORC tblproperties("orc.row.index.stride"="1000", "orc.stripe.size"="100000", "orc.compress.size"="10000"); +insert overwrite table testorc select * from vectororc order by s2; + +set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; +set hive.optimize.index.filter=true; + +set hive.vectorized.execution.enabled=false; +-- row group (1,4) from stripe 1 and row group (1) from stripe 2 +-- PPD ONLY +select count(*),sum(d) from testorc where s1 is not null; +set hive.vectorized.execution.enabled=true; +-- VECTORIZATION + PPD +select count(*),sum(d) from testorc where s1 is not null; + +set hive.vectorized.execution.enabled=false; +-- row group (2,3,5) from stripe 1 and row group (2) from stripe 2 +-- PPD ONLY +select count(*),sum(d) from testorc where s2 in ("b", "c", "e", "g"); +set hive.vectorized.execution.enabled=true; +-- VECTORIZATION + PPD +select count(*),sum(d) from testorc where s2 in ("b", "c", "e", "g"); + +set hive.vectorized.execution.enabled=false; +-- last row group of stripe 1 and first row group of stripe 2 +-- PPD ONLY +select count(*),sum(d) from testorc where s3="z"; +set hive.vectorized.execution.enabled=true; +-- VECTORIZATION + PPD +select count(*),sum(d) from testorc where s3="z"; + +set hive.vectorized.execution.enabled=false; +-- first row group of stripe 1 and last row group of stripe 2 +-- PPD ONLY +select count(*),sum(d) from testorc where s2="a" or s2="g"; +set hive.vectorized.execution.enabled=true; +-- VECTORIZATION + PPD +select count(*),sum(d) from testorc where s2="a" or s2="g"; + +drop table srcorc; +drop table vectororc; +drop table testorc; \ No newline at end of file diff --git ql/src/test/results/clientpositive/orc_vectorization_ppd.q.out ql/src/test/results/clientpositive/orc_vectorization_ppd.q.out new file mode 100644 index 0000000..b80364e --- /dev/null +++ ql/src/test/results/clientpositive/orc_vectorization_ppd.q.out @@ -0,0 +1,857 @@ +PREHOOK: query: -- create table with 1000 rows +create table srcorc(key string, value string) stored as textfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- create table with 1000 rows +create table srcorc(key string, value string) stored as textfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@srcorc +PREHOOK: query: insert overwrite table srcorc select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@srcorc +POSTHOOK: query: insert overwrite table srcorc select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@srcorc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert into table srcorc select * from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@srcorc +POSTHOOK: query: insert into table srcorc select * from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@srcorc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- load table with each row group having 1000 rows and stripe 1 & 2 having 5000 & 2000 rows respectively +create table if not exists vectororc +(s1 string, +s2 string, +d double, +s3 string) +stored as ORC tblproperties("orc.row.index.stride"="1000", "orc.stripe.size"="100000", "orc.compress.size"="10000") +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- load table with each row group having 1000 rows and stripe 1 & 2 having 5000 & 2000 rows respectively +create table if not exists vectororc +(s1 string, +s2 string, +d double, +s3 string) +stored as ORC tblproperties("orc.row.index.stride"="1000", "orc.stripe.size"="100000", "orc.compress.size"="10000") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@vectororc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- insert creates separate orc files +insert overwrite table vectororc select "apple", "a", rand(1), "zoo" from srcorc +PREHOOK: type: QUERY +PREHOOK: Input: default@srcorc +PREHOOK: Output: default@vectororc +POSTHOOK: query: -- insert creates separate orc files +insert overwrite table vectororc select "apple", "a", rand(1), "zoo" from srcorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcorc +POSTHOOK: Output: default@vectororc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +PREHOOK: query: insert into table vectororc select null, "b", rand(2), "zoo" from srcorc +PREHOOK: type: QUERY +PREHOOK: Input: default@srcorc +PREHOOK: Output: default@vectororc +POSTHOOK: query: insert into table vectororc select null, "b", rand(2), "zoo" from srcorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcorc +POSTHOOK: Output: default@vectororc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +PREHOOK: query: insert into table vectororc select null, "c", rand(3), "zoo" from srcorc +PREHOOK: type: QUERY +PREHOOK: Input: default@srcorc +PREHOOK: Output: default@vectororc +POSTHOOK: query: insert into table vectororc select null, "c", rand(3), "zoo" from srcorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcorc +POSTHOOK: Output: default@vectororc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +PREHOOK: query: insert into table vectororc select "apple", "d", rand(4), "zoo" from srcorc +PREHOOK: type: QUERY +PREHOOK: Input: default@srcorc +PREHOOK: Output: default@vectororc +POSTHOOK: query: insert into table vectororc select "apple", "d", rand(4), "zoo" from srcorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcorc +POSTHOOK: Output: default@vectororc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +PREHOOK: query: insert into table vectororc select null, "e", rand(5), "z" from srcorc +PREHOOK: type: QUERY +PREHOOK: Input: default@srcorc +PREHOOK: Output: default@vectororc +POSTHOOK: query: insert into table vectororc select null, "e", rand(5), "z" from srcorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcorc +POSTHOOK: Output: default@vectororc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +PREHOOK: query: insert into table vectororc select "apple", "f", rand(6), "z" from srcorc +PREHOOK: type: QUERY +PREHOOK: Input: default@srcorc +PREHOOK: Output: default@vectororc +POSTHOOK: query: insert into table vectororc select "apple", "f", rand(6), "z" from srcorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcorc +POSTHOOK: Output: default@vectororc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +PREHOOK: query: insert into table vectororc select null, "g", rand(7), "zoo" from srcorc +PREHOOK: type: QUERY +PREHOOK: Input: default@srcorc +PREHOOK: Output: default@vectororc +POSTHOOK: query: insert into table vectororc select null, "g", rand(7), "zoo" from srcorc +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcorc +POSTHOOK: Output: default@vectororc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +PREHOOK: query: -- since vectororc table has multiple orc file we will load them into a single file using another table +create table if not exists testorc +(s1 string, +s2 string, +d double, +s3 string) +stored as ORC tblproperties("orc.row.index.stride"="1000", "orc.stripe.size"="100000", "orc.compress.size"="10000") +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- since vectororc table has multiple orc file we will load them into a single file using another table +create table if not exists testorc +(s1 string, +s2 string, +d double, +s3 string) +stored as ORC tblproperties("orc.row.index.stride"="1000", "orc.stripe.size"="100000", "orc.compress.size"="10000") +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@testorc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +PREHOOK: query: insert overwrite table testorc select * from vectororc order by s2 +PREHOOK: type: QUERY +PREHOOK: Input: default@vectororc +PREHOOK: Output: default@testorc +POSTHOOK: query: insert overwrite table testorc select * from vectororc order by s2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@vectororc +POSTHOOK: Output: default@testorc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testorc.d SIMPLE [(vectororc)vectororc.FieldSchema(name:d, type:double, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s1 SIMPLE [(vectororc)vectororc.FieldSchema(name:s1, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s2 SIMPLE [(vectororc)vectororc.FieldSchema(name:s2, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s3 SIMPLE [(vectororc)vectororc.FieldSchema(name:s3, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +PREHOOK: query: -- row group (1,4) from stripe 1 and row group (1) from stripe 2 +-- PPD ONLY +select count(*),sum(d) from testorc where s1 is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: query: -- row group (1,4) from stripe 1 and row group (1) from stripe 2 +-- PPD ONLY +select count(*),sum(d) from testorc where s1 is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testorc.d SIMPLE [(vectororc)vectororc.FieldSchema(name:d, type:double, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s1 SIMPLE [(vectororc)vectororc.FieldSchema(name:s1, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s2 SIMPLE [(vectororc)vectororc.FieldSchema(name:s2, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s3 SIMPLE [(vectororc)vectororc.FieldSchema(name:s3, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +3000 1505.654384225757 +PREHOOK: query: -- VECTORIZATION + PPD +select count(*),sum(d) from testorc where s1 is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: query: -- VECTORIZATION + PPD +select count(*),sum(d) from testorc where s1 is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testorc.d SIMPLE [(vectororc)vectororc.FieldSchema(name:d, type:double, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s1 SIMPLE [(vectororc)vectororc.FieldSchema(name:s1, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s2 SIMPLE [(vectororc)vectororc.FieldSchema(name:s2, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s3 SIMPLE [(vectororc)vectororc.FieldSchema(name:s3, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +3000 1505.654384225757 +PREHOOK: query: -- row group (2,3,5) from stripe 1 and row group (2) from stripe 2 +-- PPD ONLY +select count(*),sum(d) from testorc where s2 in ("b", "c", "e", "g") +PREHOOK: type: QUERY +PREHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: query: -- row group (2,3,5) from stripe 1 and row group (2) from stripe 2 +-- PPD ONLY +select count(*),sum(d) from testorc where s2 in ("b", "c", "e", "g") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testorc.d SIMPLE [(vectororc)vectororc.FieldSchema(name:d, type:double, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s1 SIMPLE [(vectororc)vectororc.FieldSchema(name:s1, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s2 SIMPLE [(vectororc)vectororc.FieldSchema(name:s2, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s3 SIMPLE [(vectororc)vectororc.FieldSchema(name:s3, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +4000 2006.824941719367 +PREHOOK: query: -- VECTORIZATION + PPD +select count(*),sum(d) from testorc where s2 in ("b", "c", "e", "g") +PREHOOK: type: QUERY +PREHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: query: -- VECTORIZATION + PPD +select count(*),sum(d) from testorc where s2 in ("b", "c", "e", "g") +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testorc.d SIMPLE [(vectororc)vectororc.FieldSchema(name:d, type:double, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s1 SIMPLE [(vectororc)vectororc.FieldSchema(name:s1, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s2 SIMPLE [(vectororc)vectororc.FieldSchema(name:s2, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s3 SIMPLE [(vectororc)vectororc.FieldSchema(name:s3, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +4000 2006.824941719367 +PREHOOK: query: -- last row group of stripe 1 and first row group of stripe 2 +-- PPD ONLY +select count(*),sum(d) from testorc where s3="z" +PREHOOK: type: QUERY +PREHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: query: -- last row group of stripe 1 and first row group of stripe 2 +-- PPD ONLY +select count(*),sum(d) from testorc where s3="z" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testorc.d SIMPLE [(vectororc)vectororc.FieldSchema(name:d, type:double, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s1 SIMPLE [(vectororc)vectororc.FieldSchema(name:s1, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s2 SIMPLE [(vectororc)vectororc.FieldSchema(name:s2, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s3 SIMPLE [(vectororc)vectororc.FieldSchema(name:s3, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +2000 1011.9234098913687 +PREHOOK: query: -- VECTORIZATION + PPD +select count(*),sum(d) from testorc where s3="z" +PREHOOK: type: QUERY +PREHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: query: -- VECTORIZATION + PPD +select count(*),sum(d) from testorc where s3="z" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testorc.d SIMPLE [(vectororc)vectororc.FieldSchema(name:d, type:double, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s1 SIMPLE [(vectororc)vectororc.FieldSchema(name:s1, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s2 SIMPLE [(vectororc)vectororc.FieldSchema(name:s2, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s3 SIMPLE [(vectororc)vectororc.FieldSchema(name:s3, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +2000 1011.9234098913687 +PREHOOK: query: -- first row group of stripe 1 and last row group of stripe 2 +-- PPD ONLY +select count(*),sum(d) from testorc where s2="a" or s2="g" +PREHOOK: type: QUERY +PREHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: query: -- first row group of stripe 1 and last row group of stripe 2 +-- PPD ONLY +select count(*),sum(d) from testorc where s2="a" or s2="g" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testorc.d SIMPLE [(vectororc)vectororc.FieldSchema(name:d, type:double, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s1 SIMPLE [(vectororc)vectororc.FieldSchema(name:s1, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s2 SIMPLE [(vectororc)vectororc.FieldSchema(name:s2, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s3 SIMPLE [(vectororc)vectororc.FieldSchema(name:s3, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +2000 1006.5827834977173 +PREHOOK: query: -- VECTORIZATION + PPD +select count(*),sum(d) from testorc where s2="a" or s2="g" +PREHOOK: type: QUERY +PREHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: query: -- VECTORIZATION + PPD +select count(*),sum(d) from testorc where s2="a" or s2="g" +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testorc +#### A masked pattern was here #### +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testorc.d SIMPLE [(vectororc)vectororc.FieldSchema(name:d, type:double, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s1 SIMPLE [(vectororc)vectororc.FieldSchema(name:s1, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s2 SIMPLE [(vectororc)vectororc.FieldSchema(name:s2, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s3 SIMPLE [(vectororc)vectororc.FieldSchema(name:s3, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +2000 1006.5827834977173 +PREHOOK: query: drop table srcorc +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@srcorc +PREHOOK: Output: default@srcorc +POSTHOOK: query: drop table srcorc +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@srcorc +POSTHOOK: Output: default@srcorc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testorc.d SIMPLE [(vectororc)vectororc.FieldSchema(name:d, type:double, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s1 SIMPLE [(vectororc)vectororc.FieldSchema(name:s1, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s2 SIMPLE [(vectororc)vectororc.FieldSchema(name:s2, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s3 SIMPLE [(vectororc)vectororc.FieldSchema(name:s3, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +PREHOOK: query: drop table vectororc +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@vectororc +PREHOOK: Output: default@vectororc +POSTHOOK: query: drop table vectororc +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@vectororc +POSTHOOK: Output: default@vectororc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testorc.d SIMPLE [(vectororc)vectororc.FieldSchema(name:d, type:double, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s1 SIMPLE [(vectororc)vectororc.FieldSchema(name:s1, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s2 SIMPLE [(vectororc)vectororc.FieldSchema(name:s2, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s3 SIMPLE [(vectororc)vectororc.FieldSchema(name:s3, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +PREHOOK: query: drop table testorc +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testorc +PREHOOK: Output: default@testorc +POSTHOOK: query: drop table testorc +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testorc +POSTHOOK: Output: default@testorc +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: testorc.d SIMPLE [(vectororc)vectororc.FieldSchema(name:d, type:double, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s1 SIMPLE [(vectororc)vectororc.FieldSchema(name:s1, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s2 SIMPLE [(vectororc)vectororc.FieldSchema(name:s2, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: testorc.s3 SIMPLE [(vectororc)vectororc.FieldSchema(name:s3, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.d EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s1 SIMPLE [] +POSTHOOK: Lineage: vectororc.s1 EXPRESSION [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s2 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE [] +POSTHOOK: Lineage: vectororc.s3 SIMPLE []