diff --git data/files/dynamic_partition_insert.txt data/files/dynamic_partition_insert.txt new file mode 100644 index 0000000..be9396f --- /dev/null +++ data/files/dynamic_partition_insert.txt @@ -0,0 +1,5 @@ +1one +2two +3three +4four +5five diff --git ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java index f49b7cd..111b048 100644 --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java @@ -491,6 +491,9 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx ctx, for (ExprNodeDesc key : keys) { colLists = Utilities.mergeUniqElems(colLists, key.getCols()); } + for (ExprNodeDesc key : conf.getPartitionCols()) { + colLists = Utilities.mergeUniqElems(colLists, key.getCols()); + } assert op.getNumChild() == 1; diff --git ql/src/test/queries/clientpositive/dynamic_partition_insert.q ql/src/test/queries/clientpositive/dynamic_partition_insert.q new file mode 100644 index 0000000..b4ae3b0 --- /dev/null +++ ql/src/test/queries/clientpositive/dynamic_partition_insert.q @@ -0,0 +1,18 @@ +CREATE TABLE t1 (c1 BIGINT, c2 STRING); + +CREATE TABLE t2 (c1 INT, c2 STRING) +PARTITIONED BY (p1 STRING); + +LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1; +LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1; +LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1; +LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1; +LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1; + +SET hive.exec.dynamic.partition.mode=nonstrict; +INSERT OVERWRITE TABLE t2 partition(p1) SELECT *,c1 AS p1 FROM t1 DISTRIBUTE BY p1; + +SELECT * FROM t2; + +DROP TABLE t1; +DROP TABLE t2; diff --git ql/src/test/results/clientpositive/dynamic_partition_insert.q.out ql/src/test/results/clientpositive/dynamic_partition_insert.q.out new file mode 100644 index 0000000..b50ffce --- /dev/null +++ ql/src/test/results/clientpositive/dynamic_partition_insert.q.out @@ -0,0 +1,139 @@ +PREHOOK: query: CREATE TABLE t1 (c1 BIGINT, c2 STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t1 +POSTHOOK: query: CREATE TABLE t1 (c1 BIGINT, c2 STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t1 +PREHOOK: query: CREATE TABLE t2 (c1 INT, c2 STRING) +PARTITIONED BY (p1 STRING) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t2 +POSTHOOK: query: CREATE TABLE t2 (c1 INT, c2 STRING) +PARTITIONED BY (p1 STRING) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t2 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t1 +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1 +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@t1 +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dynamic_partition_insert.txt' INTO TABLE t1 +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@t1 +PREHOOK: query: INSERT OVERWRITE TABLE t2 partition(p1) SELECT *,c1 AS p1 FROM t1 DISTRIBUTE BY p1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t2 +POSTHOOK: query: INSERT OVERWRITE TABLE t2 partition(p1) SELECT *,c1 AS p1 FROM t1 DISTRIBUTE BY p1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t2@p1=1 +POSTHOOK: Output: default@t2@p1=2 +POSTHOOK: Output: default@t2@p1=3 +POSTHOOK: Output: default@t2@p1=4 +POSTHOOK: Output: default@t2@p1=5 +POSTHOOK: Lineage: t2 PARTITION(p1=1).c1 EXPRESSION [(t1)t1.FieldSchema(name:c1, type:bigint, comment:null), ] +POSTHOOK: Lineage: t2 PARTITION(p1=1).c2 SIMPLE [(t1)t1.FieldSchema(name:c2, type:string, comment:null), ] +POSTHOOK: Lineage: t2 PARTITION(p1=2).c1 EXPRESSION [(t1)t1.FieldSchema(name:c1, type:bigint, comment:null), ] +POSTHOOK: Lineage: t2 PARTITION(p1=2).c2 SIMPLE [(t1)t1.FieldSchema(name:c2, type:string, comment:null), ] +POSTHOOK: Lineage: t2 PARTITION(p1=3).c1 EXPRESSION [(t1)t1.FieldSchema(name:c1, type:bigint, comment:null), ] +POSTHOOK: Lineage: t2 PARTITION(p1=3).c2 SIMPLE [(t1)t1.FieldSchema(name:c2, type:string, comment:null), ] +POSTHOOK: Lineage: t2 PARTITION(p1=4).c1 EXPRESSION [(t1)t1.FieldSchema(name:c1, type:bigint, comment:null), ] +POSTHOOK: Lineage: t2 PARTITION(p1=4).c2 SIMPLE [(t1)t1.FieldSchema(name:c2, type:string, comment:null), ] +POSTHOOK: Lineage: t2 PARTITION(p1=5).c1 EXPRESSION [(t1)t1.FieldSchema(name:c1, type:bigint, comment:null), ] +POSTHOOK: Lineage: t2 PARTITION(p1=5).c2 SIMPLE [(t1)t1.FieldSchema(name:c2, type:string, comment:null), ] +PREHOOK: query: SELECT * FROM t2 +PREHOOK: type: QUERY +PREHOOK: Input: default@t2 +PREHOOK: Input: default@t2@p1=1 +PREHOOK: Input: default@t2@p1=2 +PREHOOK: Input: default@t2@p1=3 +PREHOOK: Input: default@t2@p1=4 +PREHOOK: Input: default@t2@p1=5 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM t2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t2 +POSTHOOK: Input: default@t2@p1=1 +POSTHOOK: Input: default@t2@p1=2 +POSTHOOK: Input: default@t2@p1=3 +POSTHOOK: Input: default@t2@p1=4 +POSTHOOK: Input: default@t2@p1=5 +#### A masked pattern was here #### +1 one 1 +1 one 1 +1 one 1 +1 one 1 +1 one 1 +2 two 2 +2 two 2 +2 two 2 +2 two 2 +2 two 2 +3 three 3 +3 three 3 +3 three 3 +3 three 3 +3 three 3 +4 four 4 +4 four 4 +4 four 4 +4 four 4 +4 four 4 +5 five 5 +5 five 5 +5 five 5 +5 five 5 +5 five 5 +PREHOOK: query: DROP TABLE t1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t1 +PREHOOK: Output: default@t1 +POSTHOOK: query: DROP TABLE t1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t1 +POSTHOOK: Output: default@t1 +PREHOOK: query: DROP TABLE t2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@t2 +PREHOOK: Output: default@t2 +POSTHOOK: query: DROP TABLE t2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@t2 +POSTHOOK: Output: default@t2