diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index 187dc20..944cee4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -335,7 +335,17 @@ else if (statTask != null) { } private void ensureFileFormatsMatch(TableSpec ts, URI fromURI) throws SemanticException { - Class destInputFormat = ts.tableHandle.getInputFormatClass(); + final Class destInputFormat; + try { + if (ts.getPartSpec() == null || ts.getPartSpec().isEmpty()) { + destInputFormat = ts.tableHandle.getInputFormatClass(); + } else { + destInputFormat = ts.partHandle.getInputFormatClass(); + } + } catch (HiveException e) { + throw new SemanticException(e); + } + // Other file formats should do similar check to make sure file formats match // when doing LOAD DATA .. INTO TABLE if (OrcInputFormat.class.equals(destInputFormat)) { diff --git a/ql/src/test/queries/clientnegative/load_orc_negative_part.q b/ql/src/test/queries/clientnegative/load_orc_negative_part.q new file mode 100644 index 0000000..5de4917 --- /dev/null +++ b/ql/src/test/queries/clientnegative/load_orc_negative_part.q @@ -0,0 +1,14 @@ +set hive.default.fileformat=ORC; +create table orc_staging (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp); +create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (ds string); + +load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging; +dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_staging/; + +load data inpath '${hiveconf:hive.metastore.warehouse.dir}/orc_staging/orc_split_elim.orc' into table orc_test partition (ds='10'); +load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10'); +dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_test/ds=10/; + +alter table orc_test add partition(ds='11'); +load data local inpath '../../data/files/kv1.txt' into table orc_test partition(ds='11'); +dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_test/ds=11/; diff --git a/ql/src/test/queries/clientpositive/load_orc_part.q b/ql/src/test/queries/clientpositive/load_orc_part.q new file mode 100644 index 0000000..0927ea4 --- /dev/null +++ b/ql/src/test/queries/clientpositive/load_orc_part.q @@ -0,0 +1,15 @@ +set hive.default.fileformat=ORC; +create table orc_staging (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp); +create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (ds string); + +load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging; +dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_staging/; + +load data inpath '${hiveconf:hive.metastore.warehouse.dir}/orc_staging/orc_split_elim.orc' into table orc_test partition (ds='10'); +load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10'); +dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_test/ds=10/; + +alter table orc_test add partition(ds='11'); +alter table orc_test partition(ds='11') set fileformat textfile; +load data local inpath '../../data/files/kv1.txt' into table orc_test partition(ds='11'); +dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_test/ds=11/; diff --git a/ql/src/test/results/clientnegative/load_orc_negative_part.q.out b/ql/src/test/results/clientnegative/load_orc_negative_part.q.out new file mode 100644 index 0000000..32dd627 --- /dev/null +++ b/ql/src/test/results/clientnegative/load_orc_negative_part.q.out @@ -0,0 +1,52 @@ +PREHOOK: query: create table orc_staging (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orc_staging +POSTHOOK: query: create table orc_staging (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orc_staging +PREHOOK: query: create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orc_test +POSTHOOK: query: create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orc_test +PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@orc_staging +POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@orc_staging +Found 1 items +#### A masked pattern was here #### +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@orc_test +#### A masked pattern was here #### +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@orc_test +POSTHOOK: Output: default@orc_test@ds=10 +PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@orc_test@ds=10 +POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@orc_test@ds=10 +Found 2 items +#### A masked pattern was here #### +PREHOOK: query: alter table orc_test add partition(ds='11') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@orc_test +POSTHOOK: query: alter table orc_test add partition(ds='11') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@orc_test +POSTHOOK: Output: default@orc_test@ds=11 +FAILED: SemanticException [Error 30019]: The file that you are trying to load does not match the file format of the destination table. Destination table is stored as ORC but the file being loaded is not a valid ORC file. diff --git a/ql/src/test/results/clientpositive/load_orc_part.q.out b/ql/src/test/results/clientpositive/load_orc_part.q.out new file mode 100644 index 0000000..34ca493 --- /dev/null +++ b/ql/src/test/results/clientpositive/load_orc_part.q.out @@ -0,0 +1,70 @@ +PREHOOK: query: create table orc_staging (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orc_staging +POSTHOOK: query: create table orc_staging (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orc_staging +PREHOOK: query: create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (ds string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orc_test +POSTHOOK: query: create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (ds string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orc_test +PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@orc_staging +POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@orc_staging +Found 1 items +#### A masked pattern was here #### +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@orc_test +#### A masked pattern was here #### +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@orc_test +POSTHOOK: Output: default@orc_test@ds=10 +PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@orc_test@ds=10 +POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@orc_test@ds=10 +Found 2 items +#### A masked pattern was here #### +PREHOOK: query: alter table orc_test add partition(ds='11') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Output: default@orc_test +POSTHOOK: query: alter table orc_test add partition(ds='11') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Output: default@orc_test +POSTHOOK: Output: default@orc_test@ds=11 +PREHOOK: query: alter table orc_test partition(ds='11') set fileformat textfile +PREHOOK: type: ALTERPARTITION_FILEFORMAT +PREHOOK: Input: default@orc_test +PREHOOK: Output: default@orc_test@ds=11 +POSTHOOK: query: alter table orc_test partition(ds='11') set fileformat textfile +POSTHOOK: type: ALTERPARTITION_FILEFORMAT +POSTHOOK: Input: default@orc_test +POSTHOOK: Input: default@orc_test@ds=11 +POSTHOOK: Output: default@orc_test@ds=11 +PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table orc_test partition(ds='11') +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@orc_test@ds=11 +POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table orc_test partition(ds='11') +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@orc_test@ds=11 +Found 1 items +#### A masked pattern was here ####