Index: ql/src/test/results/clientpositive/partition_wise_fileformat3.q.out =================================================================== --- ql/src/test/results/clientpositive/partition_wise_fileformat3.q.out (revision 0) +++ ql/src/test/results/clientpositive/partition_wise_fileformat3.q.out (revision 0) @@ -0,0 +1,215 @@ +PREHOOK: query: drop table partition_test_partitioned +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table partition_test_partitioned +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table partition_test_partitioned(key string, value string) partitioned by (dt string) +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table partition_test_partitioned(key string, value string) partitioned by (dt string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@partition_test_partitioned +PREHOOK: query: alter table partition_test_partitioned set fileformat rcfile +PREHOOK: type: null +POSTHOOK: query: alter table partition_test_partitioned set fileformat rcfile +POSTHOOK: type: null +POSTHOOK: Input: default@partition_test_partitioned +POSTHOOK: Output: default@partition_test_partitioned +PREHOOK: query: insert overwrite table partition_test_partitioned partition(dt=101) select * from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +PREHOOK: Output: default@partition_test_partitioned@dt=101 +POSTHOOK: query: insert overwrite table partition_test_partitioned partition(dt=101) select * from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@partition_test_partitioned@dt=101 +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show table extended like partition_test_partitioned partition(dt=101) +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like partition_test_partitioned partition(dt=101) +POSTHOOK: type: SHOW_TABLESTATUS +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +tableName:partition_test_partitioned +owner:njain +location:file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/partition_test_partitioned/dt=101 +inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat +outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat +columns:struct columns { string key, string value} +partitioned:true +partitionColumns:struct partition_columns { string dt} +totalNumberFiles:1 +totalFileSize:370 +maxFileSize:370 +minFileSize:370 +lastAccessTime:0 +lastUpdateTime:1274314890000 + +PREHOOK: query: alter table partition_test_partitioned set fileformat Sequencefile +PREHOOK: type: null +POSTHOOK: query: alter table partition_test_partitioned set fileformat Sequencefile +POSTHOOK: type: null +POSTHOOK: Input: default@partition_test_partitioned +POSTHOOK: Output: default@partition_test_partitioned +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table partition_test_partitioned partition(dt=102) select * from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +PREHOOK: Output: default@partition_test_partitioned@dt=102 +POSTHOOK: query: insert overwrite table partition_test_partitioned partition(dt=102) select * from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@partition_test_partitioned@dt=102 +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show table extended like partition_test_partitioned partition(dt=102) +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like partition_test_partitioned partition(dt=102) +POSTHOOK: type: SHOW_TABLESTATUS +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +tableName:partition_test_partitioned +owner:njain +location:file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/partition_test_partitioned/dt=102 +inputformat:org.apache.hadoop.mapred.SequenceFileInputFormat +outputformat:org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +columns:struct columns { string key, string value} +partitioned:true +partitionColumns:struct partition_columns { string dt} +totalNumberFiles:1 +totalFileSize:888 +maxFileSize:888 +minFileSize:888 +lastAccessTime:0 +lastUpdateTime:1274314895000 + +PREHOOK: query: select key from partition_test_partitioned where dt=102 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_test_partitioned@dt=102 +PREHOOK: Output: file:/data/users/njain/hive3/hive3/build/ql/scratchdir/hive_2010-05-19_17-21-36_438_6571635211032352917/10000 +POSTHOOK: query: select key from partition_test_partitioned where dt=102 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_test_partitioned@dt=102 +POSTHOOK: Output: file:/data/users/njain/hive3/hive3/build/ql/scratchdir/hive_2010-05-19_17-21-36_438_6571635211032352917/10000 +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +238 + +311 + + + +255 +278 +98 + + + +401 +150 +273 +224 +369 +66 +128 +213 +146 +406 + + + +PREHOOK: query: insert overwrite table partition_test_partitioned partition(dt=101) select * from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +PREHOOK: Output: default@partition_test_partitioned@dt=101 +POSTHOOK: query: insert overwrite table partition_test_partitioned partition(dt=101) select * from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@partition_test_partitioned@dt=101 +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: show table extended like partition_test_partitioned partition(dt=101) +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like partition_test_partitioned partition(dt=101) +POSTHOOK: type: SHOW_TABLESTATUS +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +tableName:partition_test_partitioned +owner:njain +location:file:/data/users/njain/hive3/hive3/build/ql/test/data/warehouse/partition_test_partitioned/dt=101 +inputformat:org.apache.hadoop.mapred.SequenceFileInputFormat +outputformat:org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat +columns:struct columns { string key, string value} +partitioned:true +partitionColumns:struct partition_columns { string dt} +totalNumberFiles:1 +totalFileSize:888 +maxFileSize:888 +minFileSize:888 +lastAccessTime:0 +lastUpdateTime:1274314904000 + +PREHOOK: query: select key from partition_test_partitioned where dt=101 +PREHOOK: type: QUERY +PREHOOK: Input: default@partition_test_partitioned@dt=101 +PREHOOK: Output: file:/data/users/njain/hive3/hive3/build/ql/scratchdir/hive_2010-05-19_17-21-45_060_1172142624101768381/10000 +POSTHOOK: query: select key from partition_test_partitioned where dt=101 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@partition_test_partitioned@dt=101 +POSTHOOK: Output: file:/data/users/njain/hive3/hive3/build/ql/scratchdir/hive_2010-05-19_17-21-45_060_1172142624101768381/10000 +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +238 + +311 + + + +255 +278 +98 + + + +401 +150 +273 +224 +369 +66 +128 +213 +146 +406 + + + +PREHOOK: query: drop table partition_test_partitioned +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table partition_test_partitioned +POSTHOOK: type: DROPTABLE +POSTHOOK: Output: default@partition_test_partitioned +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=101).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: partition_test_partitioned PARTITION(dt=102).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] Index: ql/src/test/queries/clientpositive/partition_wise_fileformat3.q =================================================================== --- ql/src/test/queries/clientpositive/partition_wise_fileformat3.q (revision 0) +++ ql/src/test/queries/clientpositive/partition_wise_fileformat3.q (revision 0) @@ -0,0 +1,18 @@ +drop table partition_test_partitioned; + +create table partition_test_partitioned(key string, value string) partitioned by (dt string); + +alter table partition_test_partitioned set fileformat rcfile; +insert overwrite table partition_test_partitioned partition(dt=101) select * from src1; +show table extended like partition_test_partitioned partition(dt=101); + +alter table partition_test_partitioned set fileformat Sequencefile; +insert overwrite table partition_test_partitioned partition(dt=102) select * from src1; +show table extended like partition_test_partitioned partition(dt=102); +select key from partition_test_partitioned where dt=102; + +insert overwrite table partition_test_partitioned partition(dt=101) select * from src1; +show table extended like partition_test_partitioned partition(dt=101); +select key from partition_test_partitioned where dt=101; + +drop table partition_test_partitioned; Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (revision 946782) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (working copy) @@ -567,16 +567,15 @@ partPath = part.getPath()[0]; fs = partPath.getFileSystem(getConf()); } + if (replace) { Hive.replaceFiles(loadPath, partPath, fs, tmpDirPath); } else { Hive.copyFiles(loadPath, partPath, fs); } - if (part == null) { - // create the partition if it didn't exist before - part = getPartition(tbl, partSpec, true); - } + // recreate the partition if it existed before + part = getPartition(tbl, partSpec, true); } catch (IOException e) { LOG.error(StringUtils.stringifyException(e)); throw new HiveException(e); @@ -755,10 +754,21 @@ org.apache.hadoop.hive.metastore.api.Partition tpart = null; try { tpart = getMSC().getPartition(tbl.getDbName(), tbl.getTableName(), pvals); - if (tpart == null && forceCreate) { - LOG.debug("creating partition for table " + tbl.getTableName() - + " with partition spec : " + partSpec); - tpart = getMSC().appendPartition(tbl.getDbName(), tbl.getTableName(), pvals); + if (forceCreate) { + if (tpart == null) { + LOG.debug("creating partition for table " + tbl.getTableName() + + " with partition spec : " + partSpec); + tpart = getMSC().appendPartition(tbl.getDbName(), tbl.getTableName(), pvals); + } + else { + LOG.debug("altering partition for table " + tbl.getTableName() + + " with partition spec : " + partSpec); + + tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat()); + tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat()); + tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib()); + alterPartition(tbl.getTableName(), new Partition(tbl, tpart)); + } } if (tpart == null) { return null; Index: ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (revision 946782) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (working copy) @@ -504,13 +504,8 @@ partHandle = null; } else { try { - // In case the partition already exists, we need to get the partition - // data from the metastore - partHandle = db.getPartition(tableHandle, partSpec, false); - if (partHandle == null) { - // this doesn't create partition. partition is created in MoveTask - partHandle = new Partition(tableHandle, partSpec, null); - } + // this doesn't create partition. partition is created in MoveTask + partHandle = new Partition(tableHandle, partSpec, null); } catch (HiveException e) { throw new SemanticException( ErrorMsg.INVALID_PARTITION.getMsg(ast.getChild(childIndex)));