Index: ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out =================================================================== --- ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out (revision 0) +++ ql/src/test/results/clientpositive/merge_dynamic_partition4.q.out (revision 0) @@ -0,0 +1,235 @@ +PREHOOK: query: -- this test verifies that the block merge task that can follow a query to generate dynamic +-- partitions does not produce incorrect results by dropping partitions + +create table srcpart_merge_dp like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- this test verifies that the block merge task that can follow a query to generate dynamic +-- partitions does not produce incorrect results by dropping partitions + +create table srcpart_merge_dp like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@srcpart_merge_dp +PREHOOK: query: create table srcpart_merge_dp_rc like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table srcpart_merge_dp_rc like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@srcpart_merge_dp_rc +PREHOOK: query: alter table srcpart_merge_dp_rc set fileformat RCFILE +PREHOOK: type: ALTERTABLE_FILEFORMAT +PREHOOK: Input: default@srcpart_merge_dp_rc +PREHOOK: Output: default@srcpart_merge_dp_rc +POSTHOOK: query: alter table srcpart_merge_dp_rc set fileformat RCFILE +POSTHOOK: type: ALTERTABLE_FILEFORMAT +POSTHOOK: Input: default@srcpart_merge_dp_rc +POSTHOOK: Output: default@srcpart_merge_dp_rc +PREHOOK: query: create table merge_dynamic_part like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table merge_dynamic_part like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@merge_dynamic_part +PREHOOK: query: alter table merge_dynamic_part set fileformat RCFILE +PREHOOK: type: ALTERTABLE_FILEFORMAT +PREHOOK: Input: default@merge_dynamic_part +PREHOOK: Output: default@merge_dynamic_part +POSTHOOK: query: alter table merge_dynamic_part set fileformat RCFILE +POSTHOOK: type: ALTERTABLE_FILEFORMAT +POSTHOOK: Input: default@merge_dynamic_part +POSTHOOK: Output: default@merge_dynamic_part +PREHOOK: query: load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +PREHOOK: type: LOAD +PREHOOK: Output: default@srcpart_merge_dp +POSTHOOK: query: load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +POSTHOOK: type: LOAD +POSTHOOK: Output: default@srcpart_merge_dp +POSTHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +PREHOOK: query: load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +PREHOOK: type: LOAD +PREHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +POSTHOOK: query: load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +POSTHOOK: type: LOAD +POSTHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +PREHOOK: query: load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +PREHOOK: type: LOAD +PREHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +POSTHOOK: query: load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +POSTHOOK: type: LOAD +POSTHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +PREHOOK: query: load data local inpath '../data/files/srcbucket23.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +PREHOOK: type: LOAD +PREHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +POSTHOOK: query: load data local inpath '../data/files/srcbucket23.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11) +POSTHOOK: type: LOAD +POSTHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +PREHOOK: query: load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12) +PREHOOK: type: LOAD +PREHOOK: Output: default@srcpart_merge_dp +POSTHOOK: query: load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12) +POSTHOOK: type: LOAD +POSTHOOK: Output: default@srcpart_merge_dp +POSTHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=12 +PREHOOK: query: load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12) +PREHOOK: type: LOAD +PREHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=12 +POSTHOOK: query: load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12) +POSTHOOK: type: LOAD +POSTHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=12 +PREHOOK: query: load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12) +PREHOOK: type: LOAD +PREHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=12 +POSTHOOK: query: load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12) +POSTHOOK: type: LOAD +POSTHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=12 +PREHOOK: query: load data local inpath '../data/files/srcbucket23.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12) +PREHOOK: type: LOAD +PREHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=12 +POSTHOOK: query: load data local inpath '../data/files/srcbucket23.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12) +POSTHOOK: type: LOAD +POSTHOOK: Output: default@srcpart_merge_dp@ds=2008-04-08/hr=12 +PREHOOK: query: insert overwrite table srcpart_merge_dp_rc partition (ds = '2008-04-08', hr) +select key, value, hr from srcpart_merge_dp where ds = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart_merge_dp@ds=2008-04-08/hr=12 +PREHOOK: Output: default@srcpart_merge_dp_rc@ds=2008-04-08 +POSTHOOK: query: insert overwrite table srcpart_merge_dp_rc partition (ds = '2008-04-08', hr) +select key, value, hr from srcpart_merge_dp where ds = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart_merge_dp@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart_merge_dp@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@srcpart_merge_dp_rc@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@srcpart_merge_dp_rc@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: explain +insert overwrite table merge_dynamic_part partition (ds = '2008-04-08', hr) +select key, value, if(key % 2 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc where ds = '2008-04-08' +PREHOOK: type: QUERY +POSTHOOK: query: explain +insert overwrite table merge_dynamic_part partition (ds = '2008-04-08', hr) +select key, value, if(key % 2 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc where ds = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +ABSTRACT SYNTAX TREE: + (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart_merge_dp_rc))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME merge_dynamic_part) (TOK_PARTSPEC (TOK_PARTVAL ds '2008-04-08') (TOK_PARTVAL hr)))) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION if (== (% (TOK_TABLE_OR_COL key) 2) 0) 'a1' 'b1') hr)) (TOK_WHERE (= (TOK_TABLE_OR_COL ds) '2008-04-08')))) + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 + Stage-4 + Stage-0 depends on stages: Stage-4, Stage-3 + Stage-2 depends on stages: Stage-0 + Stage-3 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Alias -> Map Operator Tree: + srcpart_merge_dp_rc + TableScan + alias: srcpart_merge_dp_rc + Select Operator + expressions: + expr: key + type: string + expr: value + type: string + expr: if(((key % 2) = 0), 'a1', 'b1') + type: string + outputColumnNames: _col0, _col1, _col2 + File Output Operator + compressed: false + GlobalTableId: 1 + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.merge_dynamic_part + + Stage: Stage-5 + Conditional Operator + + Stage: Stage-4 + Move Operator + files: + hdfs directory: true +#### A masked pattern was here #### + + Stage: Stage-0 + Move Operator + tables: + partition: + ds 2008-04-08 + hr + replace: true + table: + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe + name: default.merge_dynamic_part + + Stage: Stage-2 + Stats-Aggr Operator + + Stage: Stage-3 + Block level merge + + +PREHOOK: query: insert overwrite table merge_dynamic_part partition (ds = '2008-04-08', hr) +select key, value, if(key % 2 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc where ds = '2008-04-08' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart_merge_dp_rc@ds=2008-04-08/hr=11 +PREHOOK: Input: default@srcpart_merge_dp_rc@ds=2008-04-08/hr=12 +PREHOOK: Output: default@merge_dynamic_part@ds=2008-04-08 +POSTHOOK: query: insert overwrite table merge_dynamic_part partition (ds = '2008-04-08', hr) +select key, value, if(key % 2 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc where ds = '2008-04-08' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart_merge_dp_rc@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@srcpart_merge_dp_rc@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@merge_dynamic_part@ds=2008-04-08/hr=a1 +POSTHOOK: Output: default@merge_dynamic_part@ds=2008-04-08/hr=b1 +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=a1).key SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=a1).value SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=b1).key SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=b1).value SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show partitions merge_dynamic_part +PREHOOK: type: SHOWPARTITIONS +POSTHOOK: query: show partitions merge_dynamic_part +POSTHOOK: type: SHOWPARTITIONS +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=a1).key SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=a1).value SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=b1).key SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=b1).value SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +ds=2008-04-08/hr=a1 +ds=2008-04-08/hr=b1 +PREHOOK: query: select count(*) from merge_dynamic_part +PREHOOK: type: QUERY +PREHOOK: Input: default@merge_dynamic_part@ds=2008-04-08/hr=a1 +PREHOOK: Input: default@merge_dynamic_part@ds=2008-04-08/hr=b1 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from merge_dynamic_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@merge_dynamic_part@ds=2008-04-08/hr=a1 +POSTHOOK: Input: default@merge_dynamic_part@ds=2008-04-08/hr=b1 +#### A masked pattern was here #### +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=a1).key SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=a1).value SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=b1).key SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: merge_dynamic_part PARTITION(ds=2008-04-08,hr=b1).value SIMPLE [(srcpart_merge_dp_rc)srcpart_merge_dp_rc.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_merge_dp_rc PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart_merge_dp)srcpart_merge_dp.FieldSchema(name:value, type:string, comment:default), ] +1000 Index: ql/src/test/queries/clientpositive/merge_dynamic_partition4.q =================================================================== --- ql/src/test/queries/clientpositive/merge_dynamic_partition4.q (revision 0) +++ ql/src/test/queries/clientpositive/merge_dynamic_partition4.q (revision 0) @@ -0,0 +1,42 @@ +-- this test verifies that the block merge task that can follow a query to generate dynamic +-- partitions does not produce incorrect results by dropping partitions + +create table srcpart_merge_dp like srcpart; + +create table srcpart_merge_dp_rc like srcpart; +alter table srcpart_merge_dp_rc set fileformat RCFILE; + +create table merge_dynamic_part like srcpart; +alter table merge_dynamic_part set fileformat RCFILE; + +load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11); +load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11); +load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11); +load data local inpath '../data/files/srcbucket23.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=11); + +load data local inpath '../data/files/srcbucket20.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12); +load data local inpath '../data/files/srcbucket21.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12); +load data local inpath '../data/files/srcbucket22.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12); +load data local inpath '../data/files/srcbucket23.txt' INTO TABLE srcpart_merge_dp partition(ds='2008-04-08', hr=12); + +insert overwrite table srcpart_merge_dp_rc partition (ds = '2008-04-08', hr) +select key, value, hr from srcpart_merge_dp where ds = '2008-04-08'; + +set hive.input.format=org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat; +set hive.merge.mapfiles=true; +set hive.merge.mapredfiles=true; +set hive.merge.smallfiles.avgsize=10000000000000; +set hive.exec.compress.output=false; +set hive.exec.dynamic.partition=true; +set hive.exec.dynamic.partition.mode=nonstrict; + +explain +insert overwrite table merge_dynamic_part partition (ds = '2008-04-08', hr) +select key, value, if(key % 2 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc where ds = '2008-04-08'; + +insert overwrite table merge_dynamic_part partition (ds = '2008-04-08', hr) +select key, value, if(key % 2 == 0, 'a1', 'b1') as hr from srcpart_merge_dp_rc where ds = '2008-04-08'; + +show partitions merge_dynamic_part; + +select count(*) from merge_dynamic_part; Index: ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (revision 1350802) +++ ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (working copy) @@ -519,7 +519,7 @@ } MergeWork work = new MergeWork(inputDirs, finalName, - hasDynamicPartitions); + hasDynamicPartitions, fsInputDesc.getDynPartCtx()); LinkedHashMap> pathToAliases = new LinkedHashMap>(); pathToAliases.put(inputDir, (ArrayList) inputDirs.clone()); Index: ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeWork.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeWork.java (revision 1350802) +++ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/MergeWork.java (working copy) @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; +import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.MapredWork; import org.apache.hadoop.hive.ql.plan.PartitionDesc; @@ -41,20 +42,22 @@ private List inputPaths; private String outputDir; private boolean hasDynamicPartitions; + private DynamicPartitionCtx dynPartCtx; public MergeWork() { } public MergeWork(List inputPaths, String outputDir) { - this(inputPaths, outputDir, false); + this(inputPaths, outputDir, false, null); } public MergeWork(List inputPaths, String outputDir, - boolean hasDynamicPartitions) { + boolean hasDynamicPartitions, DynamicPartitionCtx dynPartCtx) { super(); this.inputPaths = inputPaths; this.outputDir = outputDir; this.hasDynamicPartitions = hasDynamicPartitions; + this.dynPartCtx = dynPartCtx; PartitionDesc partDesc = new PartitionDesc(); partDesc.setInputFileFormatClass(RCFileBlockMergeInputFormat.class); if(this.getPathToPartitionInfo() == null) { @@ -88,14 +91,17 @@ return RCFileMergeMapper.class; } + @Override public Long getMinSplitSize() { return null; } + @Override public String getInputformat() { return CombineHiveInputFormat.class.getName(); } + @Override public boolean isGatheringStats() { return false; } @@ -126,4 +132,12 @@ inputPaths.add(path.toString()); } + public DynamicPartitionCtx getDynPartCtx() { + return dynPartCtx; + } + + public void setDynPartCtx(DynamicPartitionCtx dynPartCtx) { + this.dynPartCtx = dynPartCtx; + } + } Index: ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java (revision 1350802) +++ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/RCFileMergeMapper.java (working copy) @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.RCFile; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; import org.apache.hadoop.hive.shims.CombineHiveKey; import org.apache.hadoop.hive.shims.ShimLoader; @@ -231,11 +232,11 @@ } public static void jobClose(String outputPath, boolean success, JobConf job, - LogHelper console) throws HiveException, IOException { + LogHelper console, DynamicPartitionCtx dynPartCtx) throws HiveException, IOException { Path outpath = new Path(outputPath); FileSystem fs = outpath.getFileSystem(job); Path backupPath = backupOutputPath(fs, outpath, job); - Utilities.mvFileToFinalPath(outputPath, job, success, LOG, null, null); + Utilities.mvFileToFinalPath(outputPath, job, success, LOG, dynPartCtx, null); fs.delete(backupPath, true); } Index: ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java (revision 1350802) +++ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java (working copy) @@ -236,7 +236,7 @@ HadoopJobExecHelper.runningJobKillURIs.remove(rj.getJobID()); jobID = rj.getID().toString(); } - RCFileMergeMapper.jobClose(outputPath, success, job, console); + RCFileMergeMapper.jobClose(outputPath, success, job, console, work.getDynPartCtx()); } catch (Exception e) { } }