diff --git conf/hive-default.xml.template conf/hive-default.xml.template index 181011b..e88aec1 100644 --- conf/hive-default.xml.template +++ conf/hive-default.xml.template @@ -1985,4 +1985,13 @@ + + hive.vectorized.execution.enabled + false + + This flag should be set to true to enable vectorized mode of query execution. + The default value is false. + + + diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 9ffd399..15a2a81 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -52,7 +52,6 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; -import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDeException; diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index 9a844c3..b0c242d 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -102,7 +102,6 @@ this.included = included; this.sarg = sarg; if (sarg != null) { - System.out.println("DEBUG XXXX SARG is not null"); sargLeaves = sarg.getLeaves(); filterColumns = new int[sargLeaves.size()]; for(int i=0; i < filterColumns.length; ++i) { @@ -110,7 +109,6 @@ filterColumns[i] = findColumns(columnNames, colName); } } else { - System.out.println("DEBUG XXXX SARG is null"); sargLeaves = null; filterColumns = null; } diff --git ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java index 3f74673..2096dbe 100644 --- ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java +++ ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java @@ -107,8 +107,7 @@ public static final HashSet srcTables = new HashSet (Arrays.asList(new String [] { "src", "src1", "srcbucket", "srcbucket2", "src_json", "src_thrift", - "src_sequencefile", "srcpart", - AllVectorTypesRecord.TABLE_NAME + "src_sequencefile", "srcpart", "alltypesorc" })); private ParseDriver pd; diff --git ql/src/test/queries/clientpositive/vectorized_rcfile_columnar.q ql/src/test/queries/clientpositive/vectorized_rcfile_columnar.q new file mode 100644 index 0000000..660442f --- /dev/null +++ ql/src/test/queries/clientpositive/vectorized_rcfile_columnar.q @@ -0,0 +1,18 @@ +--This query must pass even when vectorized reader is not available for +--RC files. The query must fall back to the non-vector mode and run successfully. + +CREATE table columnTable (key STRING, value STRING) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' +STORED AS + INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' + OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat'; + +FROM src +INSERT OVERWRITE TABLE columnTable SELECT src.key, src.value LIMIT 10; +describe columnTable; + +SET hive.vectorized.execution.enabled=true; + +SELECT key, value FROM columnTable; + diff --git ql/src/test/results/clientpositive/vectorized_rcfile_columnar.q.out ql/src/test/results/clientpositive/vectorized_rcfile_columnar.q.out new file mode 100644 index 0000000..d293770 --- /dev/null +++ ql/src/test/results/clientpositive/vectorized_rcfile_columnar.q.out @@ -0,0 +1,61 @@ +PREHOOK: query: --This query must pass even when vectorized reader is not available for +--RC files. The query must fall back to the non-vector mode and run successfully. + +CREATE table columnTable (key STRING, value STRING) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' +STORED AS + INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' + OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' +PREHOOK: type: CREATETABLE +POSTHOOK: query: --This query must pass even when vectorized reader is not available for +--RC files. The query must fall back to the non-vector mode and run successfully. + +CREATE table columnTable (key STRING, value STRING) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe' +STORED AS + INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' + OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@columnTable +PREHOOK: query: FROM src +INSERT OVERWRITE TABLE columnTable SELECT src.key, src.value LIMIT 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@columntable +POSTHOOK: query: FROM src +INSERT OVERWRITE TABLE columnTable SELECT src.key, src.value LIMIT 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@columntable +POSTHOOK: Lineage: columntable.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: columntable.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe columnTable +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe columnTable +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: columntable.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: columntable.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +key string None +value string None +PREHOOK: query: SELECT key, value FROM columnTable +PREHOOK: type: QUERY +PREHOOK: Input: default@columntable +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, value FROM columnTable +POSTHOOK: type: QUERY +POSTHOOK: Input: default@columntable +#### A masked pattern was here #### +POSTHOOK: Lineage: columntable.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: columntable.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +238 val_238 +86 val_86 +311 val_311 +27 val_27 +165 val_165 +409 val_409 +255 val_255 +278 val_278 +98 val_98 +484 val_484