Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1096925) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -428,8 +428,9 @@ // temporary variable for testing. This is added just to turn off this feature in case of a bug in // deployment. It has not been documented in hive-default.xml intentionally, this should be removed // once the feature is stable - HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS("hive.mapper.cannot.span.multiple.partitions", false), - HIVE_REWORK_MAPREDWORK("hive.rework.mapredwork", false), + HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS("hive.mapper.cannot.span.multiple.partitions", false), + HIVE_REWORK_MAPREDWORK("hive.rework.mapredwork", false), + HIVE_CONCATENATE_CHECK_INDEX ("hive.exec.concatenate.check.index", true), ; public final String varname; Index: conf/hive-default.xml =================================================================== --- conf/hive-default.xml (revision 1096925) +++ conf/hive-default.xml (working copy) @@ -1045,4 +1045,14 @@ This is first introduced by SymlinkTextInputFormat to replace symlink files with real paths at compile time. + + hive.exec.concatenate.check.index + true + If this sets to true, hive will throw error when doing + 'alter table tbl_name [partSpec] concatenate' on a table/partition + that has indexes on it. The reason the user want to set this to true + is because it can help user to avoid handling all index drop, recreation, + rebuild work. This is very helpful for tables with thousands of partitions. + + Index: ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision 1096925) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (working copy) @@ -403,6 +403,7 @@ BlockMergeTask taskExec = new BlockMergeTask(); taskExec.initialize(db.getConf(), null, driverCxt); taskExec.setWork(mergeWork); + taskExec.setQueryPlan(this.getQueryPlan()); int ret = taskExec.execute(driverCxt); return ret; Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (revision 1096925) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Task.java (working copy) @@ -486,4 +486,12 @@ public void setRetryCmdWhenFail(boolean retryCmdWhenFail) { this.retryCmdWhenFail = retryCmdWhenFail; } + + public QueryPlan getQueryPlan() { + return queryPlan; + } + + public void setQueryPlan(QueryPlan queryPlan) { + this.queryPlan = queryPlan; + } } Index: ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java (revision 1096925) +++ ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/merge/BlockMergeTask.java (working copy) @@ -111,6 +111,9 @@ if(work.getNumMapTasks() != null) { job.setNumMapTasks(work.getNumMapTasks()); } + + // zero reducers + job.setNumReduceTasks(0); if (work.getMinSplitSize() != null) { HiveConf.setLongVar(job, HiveConf.ConfVars.MAPREDMINSPLITSIZE, work @@ -146,11 +149,18 @@ RunningJob rj = null; boolean noName = StringUtils.isEmpty(HiveConf.getVar(job, HiveConf.ConfVars.HADOOPJOBNAME)); - + + String jobName = null; + if (noName && this.getQueryPlan() != null) { + int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH); + jobName = Utilities.abbreviate(this.getQueryPlan().getQueryStr(), + maxlen - 6); + } + if (noName) { // This is for a special case to ensure unit tests pass - HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME, "JOB" - + Utilities.randGen.nextInt()); + HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME, + jobName != null ? jobName : "JOB" + Utilities.randGen.nextInt()); } try { Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (revision 1096925) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (working copy) @@ -25,6 +25,8 @@ import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_SHOWDATABASES; import java.io.Serializable; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -83,7 +85,9 @@ import org.apache.hadoop.hive.ql.plan.FetchWork; import org.apache.hadoop.hive.ql.plan.GrantDesc; import org.apache.hadoop.hive.ql.plan.GrantRevokeRoleDDL; +import org.apache.hadoop.hive.ql.plan.LoadTableDesc; import org.apache.hadoop.hive.ql.plan.LockTableDesc; +import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.MsckDesc; import org.apache.hadoop.hive.ql.plan.PrincipalDesc; import org.apache.hadoop.hive.ql.plan.PrivilegeDesc; @@ -1116,19 +1120,24 @@ tableName, partSpec); List inputDir = new ArrayList(); - String outputDir = null; + String tblPartLoc = null; + Table tblObj = null; try { - Table tblObj = db.getTable(tableName); + tblObj = db.getTable(tableName); List bucketCols = null; Class inputFormatClass = null; boolean isArchived = false; - List indexes = db.getIndexes(tblObj.getDbName(), tableName, - Short.MAX_VALUE); - if (indexes != null && indexes.size() > 0) { - throw new SemanticException("can not do merge because source table " - + tableName + " is indexed."); + boolean checkIndex = HiveConf.getBoolVar(conf, + HiveConf.ConfVars.HIVE_CONCATENATE_CHECK_INDEX); + if (checkIndex) { + List indexes = db.getIndexes(tblObj.getDbName(), tableName, + Short.MAX_VALUE); + if (indexes != null && indexes.size() > 0) { + throw new SemanticException("can not do merge because source table " + + tableName + " is indexed."); + } } if (tblObj.isPartitioned()) { @@ -1144,12 +1153,12 @@ bucketCols = part.getBucketCols(); inputFormatClass = part.getInputFormatClass(); isArchived = Utilities.isArchived(part); - outputDir = part.getDataLocation().toString(); + tblPartLoc = part.getDataLocation().toString(); } } else { inputFormatClass = tblObj.getInputFormatClass(); bucketCols = tblObj.getBucketCols(); - outputDir = tblObj.getDataLocation().toString(); + tblPartLoc = tblObj.getDataLocation().toString(); } // throw a HiveException for non-rcfile. @@ -1169,29 +1178,34 @@ throw new SemanticException( "Merge can not perform on archived partitions."); } - } catch (HiveException e) { - throw new SemanticException(e); - } - - // input and output are the same - inputDir.add(outputDir); - - mergeDesc.setInputDir(inputDir); - mergeDesc.setOutputDir(outputDir); - addInputsOutputsAlterTable(tableName, partSpec); + // input and output are the same + inputDir.add(tblPartLoc); - DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), mergeDesc); - ddlWork.setNeedLock(true); - Task mergeTask = TaskFactory.get(ddlWork, conf); + mergeDesc.setInputDir(inputDir); - tableSpec tablepart = new tableSpec(this.db, conf, tablePartAST); - StatsWork statDesc = new StatsWork(tablepart); - statDesc.setNoStatsAggregator(true); - Task statTask = TaskFactory.get(statDesc, conf); - mergeTask.addDependentTask(statTask); + addInputsOutputsAlterTable(tableName, partSpec); + DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), mergeDesc); + ddlWork.setNeedLock(true); + Task mergeTask = TaskFactory.get(ddlWork, conf); + TableDesc tblDesc = Utilities.getTableDesc(tblObj); + String queryTmpdir = ctx.getExternalTmpFileURI(new URI(tblPartLoc)); + mergeDesc.setOutputDir(queryTmpdir); + LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, queryTmpdir, tblDesc, + partSpec == null ? new HashMap() : partSpec); + Task moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false), + conf); + mergeTask.addDependentTask(moveTsk); + tableSpec tablepart = new tableSpec(this.db, conf, tablePartAST); + StatsWork statDesc = new StatsWork(tablepart); + statDesc.setNoStatsAggregator(true); + Task statTask = TaskFactory.get(statDesc, conf); + moveTsk.addDependentTask(statTask); - rootTasks.add(mergeTask); + rootTasks.add(mergeTask); + } catch (Exception e) { + throw new SemanticException(e); + } } private void analyzeAlterTableClusterSort(ASTNode ast) Index: ql/src/test/queries/clientnegative/alter_concatenate_indexed_table.q =================================================================== --- ql/src/test/queries/clientnegative/alter_concatenate_indexed_table.q (revision 0) +++ ql/src/test/queries/clientnegative/alter_concatenate_indexed_table.q (revision 0) @@ -0,0 +1,16 @@ +set hive.exec.concatenate.check.index=true; +create table src_rc_concatenate_test(key int, value string) stored as rcfile; + +load data local inpath '../data/files/smbbucket_1.rc' into table src_rc_concatenate_test; +load data local inpath '../data/files/smbbucket_2.rc' into table src_rc_concatenate_test; +load data local inpath '../data/files/smbbucket_3.rc' into table src_rc_concatenate_test; + +show table extended like `src_rc_concatenate_test`; + +select count(1) from src_rc_concatenate_test; +select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test; + +create index src_rc_concatenate_test_index on table src_rc_concatenate_test(key) as 'compact' WITH DEFERRED REBUILD IDXPROPERTIES ("prop1"="val1", "prop2"="val2"); +show indexes on src_rc_concatenate_test; + +alter table src_rc_concatenate_test concatenate; Index: ql/src/test/queries/clientpositive/alter_concatenate_indexed_table.q =================================================================== --- ql/src/test/queries/clientpositive/alter_concatenate_indexed_table.q (revision 0) +++ ql/src/test/queries/clientpositive/alter_concatenate_indexed_table.q (revision 0) @@ -0,0 +1,48 @@ +set hive.exec.concatenate.check.index =false; +create table src_rc_concatenate_test(key int, value string) stored as rcfile; + +load data local inpath '../data/files/smbbucket_1.rc' into table src_rc_concatenate_test; +load data local inpath '../data/files/smbbucket_2.rc' into table src_rc_concatenate_test; +load data local inpath '../data/files/smbbucket_3.rc' into table src_rc_concatenate_test; + +show table extended like `src_rc_concatenate_test`; + +select count(1) from src_rc_concatenate_test; +select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test; + +create index src_rc_concatenate_test_index on table src_rc_concatenate_test(key) as 'compact' WITH DEFERRED REBUILD IDXPROPERTIES ("prop1"="val1", "prop2"="val2"); +show indexes on src_rc_concatenate_test; + +alter table src_rc_concatenate_test concatenate; + +show table extended like `src_rc_concatenate_test`; + +select count(1) from src_rc_concatenate_test; +select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test; + +drop index src_rc_concatenate_test_index on src_rc_concatenate_test; + +create table src_rc_concatenate_test_part(key int, value string) partitioned by (ds string) stored as rcfile; + +alter table src_rc_concatenate_test_part add partition (ds='2011'); + +load data local inpath '../data/files/smbbucket_1.rc' into table src_rc_concatenate_test_part partition (ds='2011'); +load data local inpath '../data/files/smbbucket_2.rc' into table src_rc_concatenate_test_part partition (ds='2011'); +load data local inpath '../data/files/smbbucket_3.rc' into table src_rc_concatenate_test_part partition (ds='2011'); + +show table extended like `src_rc_concatenate_test_part` partition (ds='2011'); + +select count(1) from src_rc_concatenate_test_part; +select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test_part; + +create index src_rc_concatenate_test_part_index on table src_rc_concatenate_test_part(key) as 'compact' WITH DEFERRED REBUILD IDXPROPERTIES ("prop1"="val1", "prop2"="val2"); +show indexes on src_rc_concatenate_test_part; + +alter table src_rc_concatenate_test_part partition (ds='2011') concatenate; + +show table extended like `src_rc_concatenate_test_part` partition (ds='2011'); + +select count(1) from src_rc_concatenate_test_part; +select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test_part; + +drop index src_rc_concatenate_test_part_index on src_rc_concatenate_test_part; Index: ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out =================================================================== --- ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out (revision 0) +++ ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out (revision 0) @@ -0,0 +1,70 @@ +PREHOOK: query: create table src_rc_concatenate_test(key int, value string) stored as rcfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table src_rc_concatenate_test(key int, value string) stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@src_rc_concatenate_test +PREHOOK: query: load data local inpath '../data/files/smbbucket_1.rc' into table src_rc_concatenate_test +PREHOOK: type: LOAD +PREHOOK: Output: default@src_rc_concatenate_test +POSTHOOK: query: load data local inpath '../data/files/smbbucket_1.rc' into table src_rc_concatenate_test +POSTHOOK: type: LOAD +POSTHOOK: Output: default@src_rc_concatenate_test +PREHOOK: query: load data local inpath '../data/files/smbbucket_2.rc' into table src_rc_concatenate_test +PREHOOK: type: LOAD +PREHOOK: Output: default@src_rc_concatenate_test +POSTHOOK: query: load data local inpath '../data/files/smbbucket_2.rc' into table src_rc_concatenate_test +POSTHOOK: type: LOAD +POSTHOOK: Output: default@src_rc_concatenate_test +PREHOOK: query: load data local inpath '../data/files/smbbucket_3.rc' into table src_rc_concatenate_test +PREHOOK: type: LOAD +PREHOOK: Output: default@src_rc_concatenate_test +POSTHOOK: query: load data local inpath '../data/files/smbbucket_3.rc' into table src_rc_concatenate_test +POSTHOOK: type: LOAD +POSTHOOK: Output: default@src_rc_concatenate_test +PREHOOK: query: show table extended like `src_rc_concatenate_test` +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_rc_concatenate_test` +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_rc_concatenate_test +owner:heyongqiang +location:pfile:/Users/heyongqiang/Documents/workspace/Hive-3/build/ql/test/data/warehouse/src_rc_concatenate_test +inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat +outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat +columns:struct columns { i32 key, string value} +partitioned:false +partitionColumns: +totalNumberFiles:3 +totalFileSize:636 +maxFileSize:222 +minFileSize:206 +lastAccessTime:0 +lastUpdateTime:1303856313000 + +PREHOOK: query: select count(1) from src_rc_concatenate_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_rc_concatenate_test +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-18-34_169_7540361530581615138/-mr-10000 +POSTHOOK: query: select count(1) from src_rc_concatenate_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_rc_concatenate_test +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-18-34_169_7540361530581615138/-mr-10000 +15 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_rc_concatenate_test +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-18-42_067_46716291029153270/-mr-10000 +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_rc_concatenate_test +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-18-42_067_46716291029153270/-mr-10000 +214 -7678496319 +PREHOOK: query: create index src_rc_concatenate_test_index on table src_rc_concatenate_test(key) as 'compact' WITH DEFERRED REBUILD IDXPROPERTIES ("prop1"="val1", "prop2"="val2") +PREHOOK: type: CREATEINDEX +POSTHOOK: query: create index src_rc_concatenate_test_index on table src_rc_concatenate_test(key) as 'compact' WITH DEFERRED REBUILD IDXPROPERTIES ("prop1"="val1", "prop2"="val2") +POSTHOOK: type: CREATEINDEX +PREHOOK: query: show indexes on src_rc_concatenate_test +PREHOOK: type: SHOWINDEXES +POSTHOOK: query: show indexes on src_rc_concatenate_test +POSTHOOK: type: SHOWINDEXES +src_rc_concatenate_test_index src_rc_concatenate_test key default__src_rc_concatenate_test_src_rc_concatenate_test_index__ compact +FAILED: Error in semantic analysis: org.apache.hadoop.hive.ql.parse.SemanticException: can not do merge because source table src_rc_concatenate_test is indexed. Index: ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out =================================================================== --- ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out (revision 0) +++ ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out (revision 0) @@ -0,0 +1,243 @@ +PREHOOK: query: create table src_rc_concatenate_test(key int, value string) stored as rcfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table src_rc_concatenate_test(key int, value string) stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@src_rc_concatenate_test +PREHOOK: query: load data local inpath '../data/files/smbbucket_1.rc' into table src_rc_concatenate_test +PREHOOK: type: LOAD +PREHOOK: Output: default@src_rc_concatenate_test +POSTHOOK: query: load data local inpath '../data/files/smbbucket_1.rc' into table src_rc_concatenate_test +POSTHOOK: type: LOAD +POSTHOOK: Output: default@src_rc_concatenate_test +PREHOOK: query: load data local inpath '../data/files/smbbucket_2.rc' into table src_rc_concatenate_test +PREHOOK: type: LOAD +PREHOOK: Output: default@src_rc_concatenate_test +POSTHOOK: query: load data local inpath '../data/files/smbbucket_2.rc' into table src_rc_concatenate_test +POSTHOOK: type: LOAD +POSTHOOK: Output: default@src_rc_concatenate_test +PREHOOK: query: load data local inpath '../data/files/smbbucket_3.rc' into table src_rc_concatenate_test +PREHOOK: type: LOAD +PREHOOK: Output: default@src_rc_concatenate_test +POSTHOOK: query: load data local inpath '../data/files/smbbucket_3.rc' into table src_rc_concatenate_test +POSTHOOK: type: LOAD +POSTHOOK: Output: default@src_rc_concatenate_test +PREHOOK: query: show table extended like `src_rc_concatenate_test` +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_rc_concatenate_test` +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_rc_concatenate_test +owner:heyongqiang +location:pfile:/Users/heyongqiang/Documents/workspace/Hive-3/build/ql/test/data/warehouse/src_rc_concatenate_test +inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat +outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat +columns:struct columns { i32 key, string value} +partitioned:false +partitionColumns: +totalNumberFiles:3 +totalFileSize:636 +maxFileSize:222 +minFileSize:206 +lastAccessTime:0 +lastUpdateTime:1303856093000 + +PREHOOK: query: select count(1) from src_rc_concatenate_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_rc_concatenate_test +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-14-54_116_5056538740056177435/-mr-10000 +POSTHOOK: query: select count(1) from src_rc_concatenate_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_rc_concatenate_test +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-14-54_116_5056538740056177435/-mr-10000 +15 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_rc_concatenate_test +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-15-01_971_2647514620376325946/-mr-10000 +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_rc_concatenate_test +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-15-01_971_2647514620376325946/-mr-10000 +214 -7678496319 +PREHOOK: query: create index src_rc_concatenate_test_index on table src_rc_concatenate_test(key) as 'compact' WITH DEFERRED REBUILD IDXPROPERTIES ("prop1"="val1", "prop2"="val2") +PREHOOK: type: CREATEINDEX +POSTHOOK: query: create index src_rc_concatenate_test_index on table src_rc_concatenate_test(key) as 'compact' WITH DEFERRED REBUILD IDXPROPERTIES ("prop1"="val1", "prop2"="val2") +POSTHOOK: type: CREATEINDEX +PREHOOK: query: show indexes on src_rc_concatenate_test +PREHOOK: type: SHOWINDEXES +POSTHOOK: query: show indexes on src_rc_concatenate_test +POSTHOOK: type: SHOWINDEXES +src_rc_concatenate_test_index src_rc_concatenate_test key default__src_rc_concatenate_test_src_rc_concatenate_test_index__ compact +PREHOOK: query: alter table src_rc_concatenate_test concatenate +PREHOOK: type: ALTER_TABLE_MERGE +PREHOOK: Input: default@src_rc_concatenate_test +PREHOOK: Output: default@src_rc_concatenate_test +POSTHOOK: query: alter table src_rc_concatenate_test concatenate +POSTHOOK: type: ALTER_TABLE_MERGE +POSTHOOK: Input: default@src_rc_concatenate_test +POSTHOOK: Output: default@src_rc_concatenate_test +PREHOOK: query: show table extended like `src_rc_concatenate_test` +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_rc_concatenate_test` +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_rc_concatenate_test +owner:heyongqiang +location:pfile:/Users/heyongqiang/Documents/workspace/Hive-3/build/ql/test/data/warehouse/src_rc_concatenate_test +inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat +outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat +columns:struct columns { i32 key, string value} +partitioned:false +partitionColumns: +totalNumberFiles:1 +totalFileSize:334 +maxFileSize:334 +minFileSize:334 +lastAccessTime:0 +lastUpdateTime:1303856111000 + +PREHOOK: query: select count(1) from src_rc_concatenate_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_rc_concatenate_test +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-15-12_232_1765943327509437357/-mr-10000 +POSTHOOK: query: select count(1) from src_rc_concatenate_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_rc_concatenate_test +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-15-12_232_1765943327509437357/-mr-10000 +15 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test +PREHOOK: type: QUERY +PREHOOK: Input: default@src_rc_concatenate_test +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-15-18_738_6289595467011155110/-mr-10000 +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_rc_concatenate_test +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-15-18_738_6289595467011155110/-mr-10000 +214 -7678496319 +PREHOOK: query: drop index src_rc_concatenate_test_index on src_rc_concatenate_test +PREHOOK: type: DROPINDEX +POSTHOOK: query: drop index src_rc_concatenate_test_index on src_rc_concatenate_test +POSTHOOK: type: DROPINDEX +PREHOOK: query: create table src_rc_concatenate_test_part(key int, value string) partitioned by (ds string) stored as rcfile +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table src_rc_concatenate_test_part(key int, value string) partitioned by (ds string) stored as rcfile +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@src_rc_concatenate_test_part +PREHOOK: query: alter table src_rc_concatenate_test_part add partition (ds='2011') +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@src_rc_concatenate_test_part +POSTHOOK: query: alter table src_rc_concatenate_test_part add partition (ds='2011') +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@src_rc_concatenate_test_part +POSTHOOK: Output: default@src_rc_concatenate_test_part@ds=2011 +PREHOOK: query: load data local inpath '../data/files/smbbucket_1.rc' into table src_rc_concatenate_test_part partition (ds='2011') +PREHOOK: type: LOAD +PREHOOK: Output: default@src_rc_concatenate_test_part@ds=2011 +POSTHOOK: query: load data local inpath '../data/files/smbbucket_1.rc' into table src_rc_concatenate_test_part partition (ds='2011') +POSTHOOK: type: LOAD +POSTHOOK: Output: default@src_rc_concatenate_test_part@ds=2011 +PREHOOK: query: load data local inpath '../data/files/smbbucket_2.rc' into table src_rc_concatenate_test_part partition (ds='2011') +PREHOOK: type: LOAD +PREHOOK: Output: default@src_rc_concatenate_test_part@ds=2011 +POSTHOOK: query: load data local inpath '../data/files/smbbucket_2.rc' into table src_rc_concatenate_test_part partition (ds='2011') +POSTHOOK: type: LOAD +POSTHOOK: Output: default@src_rc_concatenate_test_part@ds=2011 +PREHOOK: query: load data local inpath '../data/files/smbbucket_3.rc' into table src_rc_concatenate_test_part partition (ds='2011') +PREHOOK: type: LOAD +PREHOOK: Output: default@src_rc_concatenate_test_part@ds=2011 +POSTHOOK: query: load data local inpath '../data/files/smbbucket_3.rc' into table src_rc_concatenate_test_part partition (ds='2011') +POSTHOOK: type: LOAD +POSTHOOK: Output: default@src_rc_concatenate_test_part@ds=2011 +PREHOOK: query: show table extended like `src_rc_concatenate_test_part` partition (ds='2011') +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_rc_concatenate_test_part` partition (ds='2011') +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_rc_concatenate_test_part +owner:heyongqiang +location:pfile:/Users/heyongqiang/Documents/workspace/Hive-3/build/ql/test/data/warehouse/src_rc_concatenate_test_part/ds=2011 +inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat +outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat +columns:struct columns { i32 key, string value} +partitioned:true +partitionColumns:struct partition_columns { string ds} +totalNumberFiles:3 +totalFileSize:636 +maxFileSize:222 +minFileSize:206 +lastAccessTime:0 +lastUpdateTime:1303856128000 + +PREHOOK: query: select count(1) from src_rc_concatenate_test_part +PREHOOK: type: QUERY +PREHOOK: Input: default@src_rc_concatenate_test_part@ds=2011 +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-15-28_709_6508379632697292274/-mr-10000 +POSTHOOK: query: select count(1) from src_rc_concatenate_test_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_rc_concatenate_test_part@ds=2011 +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-15-28_709_6508379632697292274/-mr-10000 +15 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test_part +PREHOOK: type: QUERY +PREHOOK: Input: default@src_rc_concatenate_test_part@ds=2011 +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-15-35_496_4356530739014843487/-mr-10000 +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_rc_concatenate_test_part@ds=2011 +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-15-35_496_4356530739014843487/-mr-10000 +214 -7678496319 +PREHOOK: query: create index src_rc_concatenate_test_part_index on table src_rc_concatenate_test_part(key) as 'compact' WITH DEFERRED REBUILD IDXPROPERTIES ("prop1"="val1", "prop2"="val2") +PREHOOK: type: CREATEINDEX +POSTHOOK: query: create index src_rc_concatenate_test_part_index on table src_rc_concatenate_test_part(key) as 'compact' WITH DEFERRED REBUILD IDXPROPERTIES ("prop1"="val1", "prop2"="val2") +POSTHOOK: type: CREATEINDEX +PREHOOK: query: show indexes on src_rc_concatenate_test_part +PREHOOK: type: SHOWINDEXES +POSTHOOK: query: show indexes on src_rc_concatenate_test_part +POSTHOOK: type: SHOWINDEXES +src_rc_concatenate_test_part_index src_rc_concatenate_test_part key default__src_rc_concatenate_test_part_src_rc_concatenate_test_part_index__ compact +PREHOOK: query: alter table src_rc_concatenate_test_part partition (ds='2011') concatenate +PREHOOK: type: ALTER_PARTITION_MERGE +PREHOOK: Input: default@src_rc_concatenate_test_part +PREHOOK: Output: default@src_rc_concatenate_test_part@ds=2011 +POSTHOOK: query: alter table src_rc_concatenate_test_part partition (ds='2011') concatenate +POSTHOOK: type: ALTER_PARTITION_MERGE +POSTHOOK: Input: default@src_rc_concatenate_test_part +POSTHOOK: Output: default@src_rc_concatenate_test_part@ds=2011 +PREHOOK: query: show table extended like `src_rc_concatenate_test_part` partition (ds='2011') +PREHOOK: type: SHOW_TABLESTATUS +POSTHOOK: query: show table extended like `src_rc_concatenate_test_part` partition (ds='2011') +POSTHOOK: type: SHOW_TABLESTATUS +tableName:src_rc_concatenate_test_part +owner:heyongqiang +location:pfile:/Users/heyongqiang/Documents/workspace/Hive-3/build/ql/test/data/warehouse/src_rc_concatenate_test_part/ds=2011 +inputformat:org.apache.hadoop.hive.ql.io.RCFileInputFormat +outputformat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat +columns:struct columns { i32 key, string value} +partitioned:true +partitionColumns:struct partition_columns { string ds} +totalNumberFiles:1 +totalFileSize:334 +maxFileSize:334 +minFileSize:334 +lastAccessTime:0 +lastUpdateTime:1303856144000 + +PREHOOK: query: select count(1) from src_rc_concatenate_test_part +PREHOOK: type: QUERY +PREHOOK: Input: default@src_rc_concatenate_test_part@ds=2011 +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-15-45_308_2380163877365308340/-mr-10000 +POSTHOOK: query: select count(1) from src_rc_concatenate_test_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_rc_concatenate_test_part@ds=2011 +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-15-45_308_2380163877365308340/-mr-10000 +15 +PREHOOK: query: select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test_part +PREHOOK: type: QUERY +PREHOOK: Input: default@src_rc_concatenate_test_part@ds=2011 +PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-15-51_822_7148108227690658912/-mr-10000 +POSTHOOK: query: select sum(hash(key)), sum(hash(value)) from src_rc_concatenate_test_part +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src_rc_concatenate_test_part@ds=2011 +POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2011-04-26_15-15-51_822_7148108227690658912/-mr-10000 +214 -7678496319 +PREHOOK: query: drop index src_rc_concatenate_test_part_index on src_rc_concatenate_test_part +PREHOOK: type: DROPINDEX +POSTHOOK: query: drop index src_rc_concatenate_test_part_index on src_rc_concatenate_test_part +POSTHOOK: type: DROPINDEX