Index: ql/src/test/results/clientnegative/archive1.q.out =================================================================== --- ql/src/test/results/clientnegative/archive1.q.out (revision 1148470) +++ ql/src/test/results/clientnegative/archive1.q.out (working copy) @@ -35,5 +35,5 @@ PREHOOK: type: ALTERTABLE_ARCHIVE PREHOOK: Input: default@srcpart_archived PREHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12 -FAILED: Error in metadata: Specified partition is already archived +FAILED: Error in metadata: Partition(s) not archived FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_multi1.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_multi1.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_multi1.q.out (revision 0) @@ -0,0 +1,52 @@ +PREHOOK: query: -- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +FAILED: Error in metadata: Partition(s) already archived +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_multi4.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_multi4.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_multi4.q.out (revision 0) @@ -0,0 +1,53 @@ +PREHOOK: query: -- Tests trying to archive inner partition contained in archived partition group. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to archive inner partition contained in archived partition group. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +FAILED: Error in metadata: Partition ds=2008-04-08/hr=12 seems to be already archived in different place +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_multi7.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_multi7.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_multi7.q.out (revision 0) @@ -0,0 +1,39 @@ +PREHOOK: query: -- Tests trying to archive a partition group with custom locations. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to archive a partition group with custom locations. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ADD PARTITION (ds='2008-04-08', hr='12') +LOCATION "/tmp/tstsrc" +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@tstsrcpart +POSTHOOK: query: ALTER TABLE tstsrcpart ADD PARTITION (ds='2008-04-08', hr='12') +LOCATION "/tmp/tstsrc" +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +FAILED: Error in metadata: ARCHIVE cannot run for partition groups with custom locations like pfile:/tmp/tstsrc +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive2.q.out =================================================================== --- ql/src/test/results/clientnegative/archive2.q.out (revision 1148470) +++ ql/src/test/results/clientnegative/archive2.q.out (working copy) @@ -29,5 +29,5 @@ PREHOOK: type: ALTERTABLE_UNARCHIVE PREHOOK: Input: default@tstsrcpart PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 -FAILED: Error in metadata: Specified partition is not archived +FAILED: Error in metadata: Partition(s) not archived FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_multi2.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_multi2.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_multi2.q.out (revision 0) @@ -0,0 +1,47 @@ +PREHOOK: query: -- Tests trying to unarchive a non-archived partition group +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +drop table tstsrcpart +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- Tests trying to unarchive a non-archived partition group +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +drop table tstsrcpart +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table tstsrcpart like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table tstsrcpart like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr='12') +PREHOOK: type: ALTERTABLE_UNARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +FAILED: Error in metadata: Partition(s) not archived +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_multi5.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_multi5.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_multi5.q.out (revision 0) @@ -0,0 +1,54 @@ +PREHOOK: query: -- Tests trying to unarchive outer partition group containing other partition inside. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to unarchive outer partition group containing other partition inside. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_UNARCHIVE +PREHOOK: Input: default@tstsrcpart +FAILED: Error in metadata: Partition ds=2008-04-08/hr=12 seems to be archived in wrong place +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_multi3.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_multi3.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_multi3.q.out (revision 0) @@ -0,0 +1,54 @@ +PREHOOK: query: -- Tests trying to archive outer partition group containing other partition inside. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to archive outer partition group containing other partition inside. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +FAILED: Error in metadata: Partition ds=2008-04-08/hr=12 seems to be already archived in different place +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_multi6.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_multi6.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_multi6.q.out (revision 0) @@ -0,0 +1,53 @@ +PREHOOK: query: -- Tests trying to unarchive inner partition contained in archived partition group. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to unarchive inner partition contained in archived partition group. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr='12') +PREHOOK: type: ALTERTABLE_UNARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +FAILED: Error in metadata: Partition ds=2008-04-08/hr=12 seems to be archived in wrong place +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientpositive/archive_multi.q.out =================================================================== --- ql/src/test/results/clientpositive/archive_multi.q.out (revision 0) +++ ql/src/test/results/clientpositive/archive_multi.q.out (revision 0) @@ -0,0 +1,253 @@ +PREHOOK: query: drop table tstsrc +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tstsrc +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table tstsrcpart +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tstsrcpart +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table tstsrc like src +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table tstsrc like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrc +PREHOOK: query: insert overwrite table tstsrc select key, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@tstsrc +POSTHOOK: query: insert overwrite table tstsrc select key, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@tstsrc +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create table tstsrcpart like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table tstsrcpart like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11') +select key, value from srcpart where ds='2008-04-09' and hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=11 +POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11') +select key, value from srcpart where ds='2008-04-09' and hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=11 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12') +select key, value from srcpart where ds='2008-04-09' and hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=12 +POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12') +select key, value from srcpart where ds='2008-04-09' and hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: file:/tmp/marcin/hive_2011-07-18_19-29-59_773_8713480336063891237/-mr-10000 +POSTHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-18_19-29-59_773_8713480336063891237/-mr-10000 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +48479881068 +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: file:/tmp/marcin/hive_2011-07-18_19-30-06_839_4619250197768227142/-mr-10000 +POSTHOOK: query: SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-18_19-30-06_839_4619250197768227142/-mr-10000 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +48479881068 +PREHOOK: query: SELECT key, count(1) FROM tstsrcpart WHERE ds='2008-04-08' AND hr='12' AND key='0' GROUP BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: file:/tmp/marcin/hive_2011-07-18_19-30-11_608_5088148917761799212/-mr-10000 +POSTHOOK: query: SELECT key, count(1) FROM tstsrcpart WHERE ds='2008-04-08' AND hr='12' AND key='0' GROUP BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-18_19-30-11_608_5088148917761799212/-mr-10000 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +0 3 +PREHOOK: query: SELECT * FROM tstsrcpart a JOIN tstsrc b ON a.key=b.key +WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0' +PREHOOK: type: QUERY +PREHOOK: Input: default@tstsrc +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: file:/tmp/marcin/hive_2011-07-18_19-30-15_403_6180737976141088906/-mr-10000 +POSTHOOK: query: SELECT * FROM tstsrcpart a JOIN tstsrc b ON a.key=b.key +WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstsrc +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-18_19-30-15_403_6180737976141088906/-mr-10000 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_UNARCHIVE +PREHOOK: Input: default@tstsrcpart +POSTHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08') +POSTHOOK: type: ALTERTABLE_UNARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: file:/tmp/marcin/hive_2011-07-18_19-30-20_884_5145964972487019399/-mr-10000 +POSTHOOK: query: SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-18_19-30-20_884_5145964972487019399/-mr-10000 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +48479881068 Index: ql/src/test/queries/clientnegative/archive_multi5.q =================================================================== --- ql/src/test/queries/clientnegative/archive_multi5.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_multi5.q (revision 0) @@ -0,0 +1,13 @@ +set hive.archive.enabled = true; +-- Tests trying to unarchive outer partition group containing other partition inside. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12'); +ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08'); Index: ql/src/test/queries/clientnegative/archive_multi6.q =================================================================== --- ql/src/test/queries/clientnegative/archive_multi6.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_multi6.q (revision 0) @@ -0,0 +1,13 @@ +set hive.archive.enabled = true; +-- Tests trying to unarchive inner partition contained in archived partition group. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); +ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr='12'); Index: ql/src/test/queries/clientnegative/archive_multi7.q =================================================================== --- ql/src/test/queries/clientnegative/archive_multi7.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_multi7.q (revision 0) @@ -0,0 +1,12 @@ +set hive.archive.enabled = true; +-- Tests trying to archive a partition group with custom locations. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; +ALTER TABLE tstsrcpart ADD PARTITION (ds='2008-04-08', hr='12') +LOCATION "/tmp/tstsrc"; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); Index: ql/src/test/queries/clientnegative/archive_multi1.q =================================================================== --- ql/src/test/queries/clientnegative/archive_multi1.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_multi1.q (revision 0) @@ -0,0 +1,13 @@ +set hive.archive.enabled = true; +-- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); Index: ql/src/test/queries/clientnegative/archive_multi2.q =================================================================== --- ql/src/test/queries/clientnegative/archive_multi2.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_multi2.q (revision 0) @@ -0,0 +1,12 @@ +set hive.archive.enabled = true; +-- Tests trying to unarchive a non-archived partition group +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +drop table tstsrcpart; +create table tstsrcpart like srcpart; +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; +insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12'; + +ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr='12'); Index: ql/src/test/queries/clientnegative/archive_multi3.q =================================================================== --- ql/src/test/queries/clientnegative/archive_multi3.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_multi3.q (revision 0) @@ -0,0 +1,13 @@ +set hive.archive.enabled = true; +-- Tests trying to archive outer partition group containing other partition inside. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12'); +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); Index: ql/src/test/queries/clientnegative/archive_multi4.q =================================================================== --- ql/src/test/queries/clientnegative/archive_multi4.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_multi4.q (revision 0) @@ -0,0 +1,13 @@ +set hive.archive.enabled = true; +-- Tests trying to archive inner partition contained in archived partition group. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12'); Index: ql/src/test/queries/clientpositive/archive_multi.q =================================================================== --- ql/src/test/queries/clientpositive/archive_multi.q (revision 0) +++ ql/src/test/queries/clientpositive/archive_multi.q (revision 0) @@ -0,0 +1,42 @@ +set hive.archive.enabled = true; +set hive.enforce.bucketing = true; + +drop table tstsrc; +drop table tstsrcpart; + +create table tstsrc like src; +insert overwrite table tstsrc select key, value from src; + +create table tstsrcpart like srcpart; + +insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11'; + +insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12'; + +insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11') +select key, value from srcpart where ds='2008-04-09' and hr='11'; + +insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12') +select key, value from srcpart where ds='2008-04-09' and hr='12'; + +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); + +SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2; + +SELECT key, count(1) FROM tstsrcpart WHERE ds='2008-04-08' AND hr='12' AND key='0' GROUP BY key; + +SELECT * FROM tstsrcpart a JOIN tstsrc b ON a.key=b.key +WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0'; + +ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08'); + +SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2; Index: ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision 1148470) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (working copy) @@ -985,55 +985,19 @@ } } - // Returns only the path component of the URI - private String getArchiveDirOnly(Path parentDir, String archiveName) { - URI parentUri = parentDir.toUri(); - Path harDir = new Path(parentUri.getPath(), archiveName); - return harDir.toString(); - } - /** * Sets the appropriate attributes in the supplied Partition object to mark * it as archived. Note that the metastore is not touched - a separate * call to alter_partition is needed. * * @param p - the partition object to modify - * @param parentDir - the parent directory of the archive, which is the - * original directory that the partition's files resided in - * @param dirInArchive - the directory within the archive file that contains - * the partitions files - * @param archiveName - the name of the archive - * @throws URISyntaxException + * @param harUri - new location of partition (har schema URI) */ - private void setArchived(Partition p, Path parentDir, String dirInArchive, String archiveName) - throws URISyntaxException { + private void setArchived(Partition p, URI harUri) { assert(Utilities.isArchived(p) == false); - Map params = p.getParameters(); - URI parentUri = parentDir.toUri(); - String parentHost = parentUri.getHost(); - String harHost = null; - if (parentHost == null) { - harHost = ""; - } else { - harHost = parentUri.getScheme() + "-" + parentHost; - } - - // harUri is used to access the partition's files, which are in the archive - // The format of the RI is something like: - // har://underlyingfsscheme-host:port/archivepath - URI harUri = null; - if (dirInArchive.length() == 0) { - harUri = new URI("har", parentUri.getUserInfo(), harHost, parentUri.getPort(), - getArchiveDirOnly(parentDir, archiveName), - parentUri.getQuery(), parentUri.getFragment()); - } else { - harUri = new URI("har", parentUri.getUserInfo(), harHost, parentUri.getPort(), - new Path(getArchiveDirOnly(parentDir, archiveName), dirInArchive).toUri().getPath(), - parentUri.getQuery(), parentUri.getFragment()); - } setIsArchived(p, true); - setOriginalLocation(p, parentDir.toString()); + setOriginalLocation(p, p.getLocation()); // TODO: check if it's right way p.setLocation(harUri.toString()); } @@ -1053,6 +1017,108 @@ p.setLocation(parentDir); } + private String addSlash(String s) { + return s.endsWith("/") ? s : s + "/"; + } + + /** + * Makes sure, that URI points to directory by adding slash to it. + * Useful in relativizing URIs. + */ + private URI addSlash(URI u) throws HiveException { + if(u.getPath().endsWith("/")) { + return u; + } else { + try { + return new URI(u.getScheme(), u.getAuthority(), u.getPath() + "/", u.getQuery(), u.getFragment()); + } catch (URISyntaxException e) { + // It cannot happen (can it?) + throw new HiveException("Couldn't append slash to a URI", e); + } + } + } + + /** + * HarPathHelper helps to create har:/ URIs for locations inside of archive. + */ + private class HarPathHelper { + boolean parentSettable = conf.getBoolVar(HiveConf.ConfVars.HIVEHARPARENTDIRSETTABLE); + private final URI base; + + /** + * Creates helper for archive. + * @param archive absolute location of archive in underlying filesystem + */ + public HarPathHelper(URI archive) throws HiveException { + String parentHost = archive.getHost(); + String harHost = null; + if (parentHost == null) { + harHost = archive.getScheme(); //TODO check if we are allowed to do this! + } else { + harHost = archive.getScheme() + "-" + parentHost; + } + + // have to make sure there's slash after .har, otherwise resolve doesn't work + String path = archive.getPath(); + if(!path.endsWith("/")) { + path = path + "/"; + } + if(!path.endsWith(".har/")) { + throw new HiveException("HAR archive path must end with .har"); + } + // harUri is used to access the partition's files, which are in the archive + // The format of the RI is something like: + // har://underlyingfsscheme-host:port/archivepath + try { + base = new URI("har", archive.getUserInfo(), harHost, archive.getPort(), + path, archive.getQuery(), archive.getFragment()); + } catch (URISyntaxException e) { + throw new HiveException("Couldn't create har URI from archive URI", e); + } + } + + /** + * Creates har URI for file/directory that was put there when creating HAR. + * + * With older versions of Hadoop, archiving a directory would produce + * the same directory structure, reflecting absoulute paths. + * If you created myArchive.har of /tmp/myDir the files in /tmp/myDir + * will be located under myArchive.har/tmp/myDir/* + * + * With newer versions, the parent directory can be specified. Assuming + * the parent directory was set to /tmp/myDir when creating the archive, + * the files can be found under myArchive.har/* + * + * This is why originalBase is argument - with new versions we can + * relativize URI, in older we keep absolute one. + * + * @param original file/directory path + * @param originalBase directory for which Hadoop archive was created + * @return absolute HAR uri + */ + URI getHarURI(URI original, URI originalBase) throws HiveException { + URI relative = null; + if (!parentSettable) { + String dirInArchive = original.getPath(); + if(dirInArchive.length() > 1 && dirInArchive.charAt(0)=='/') { + dirInArchive = dirInArchive.substring(1); + } + try { + relative = new URI(null, null, dirInArchive, null); + } catch (URISyntaxException e) { + throw new HiveException("Couldn't create har URI for location"); + } // relative URI with path only + } + else { + relative = originalBase.relativize(original); + if(relative.isAbsolute()) { + throw new HiveException("Unable to relativize URI"); + } + } + return base.resolve(relative); + } + } + private boolean pathExists(Path p) throws HiveException { try { FileSystem fs = p.getFileSystem(conf); @@ -1081,6 +1147,111 @@ } } + /** + * PartSpecInfo keeps fields and values extracted from partial partition info + * which is prefix of the full info. + */ + private static class PartSpecInfo { + public boolean full; + public List fields; + public List values; + public PartSpecInfo(boolean full, List fields, + List values) { + this.full = full; + this.fields = fields; + this.values = values; + } + + /** + * Extract partial prefix specification from table and key-value map + * + * @param tbl table in which partition is + * @param partSpec specification of partition + * @return extracted specification + */ + static public PartSpecInfo create(Table tbl, Map partSpec) + throws HiveException { + // we have to check if we receive prefix of partition keys so in table + // scheme like table/ds=2011-01-02/hr=13/ + // ARCHIVE PARTITION (ds='2011-01-02') will work and + // ARCHIVE PARTITION(hr='13') won't + List prefixFields = new ArrayList(); + List prefixValues = new ArrayList(); + String unboundKey = null; + for(FieldSchema fs: tbl.getPartCols()) { + if(partSpec.containsKey(fs.getName())) { + if(unboundKey != null) { + String message = String.format("Command can run only when prefix " + + "of partition keys list is bound - can't bind %s when %s unbound", + fs.getName(), unboundKey); + throw new HiveException(message); + } + else { + prefixFields.add(fs); + prefixValues.add(partSpec.get(fs.getName())); + } + } + else if(unboundKey == null) { + unboundKey = fs.getName(); + } + } + return new PartSpecInfo(unboundKey == null, prefixFields, prefixValues); + } + + /** + * Creates path where partitions matching prefix should lie in filesystem + * @param tbl table in which partition is + * @return expected location of partitions matchin prefix in filesystem + */ + public Path createPath(Table tbl) throws HiveException { + String prefixSubdir; + try { + prefixSubdir = Warehouse.makePartName(fields, values); + } catch (MetaException e) { + throw new HiveException("Unable to get partitions directories prefix", e); + } + URI tableDir = tbl.getDataLocation(); + if(tableDir == null) { + throw new HiveException("Table has no location set"); + } + return new Path(tableDir.toString(), prefixSubdir); + } + public String getName() throws HiveException { + try { + return Warehouse.makePartName(fields, values); + } catch (MetaException e) { + throw new HiveException("Unable to create partial name", e); + } + } + } + + /** + * Checks in partition is in custom (not-standard) location. + * @param tbl - table in which partition is + * @param p - partition + * @return true if partition location is custom, false if it is standard + */ + boolean partitionInCustomLocation(Table tbl, Partition p) + throws HiveException { + String subdir = null; + try { + subdir = Warehouse.makePartName(tbl.getPartCols(), p.getValues()); + } catch (MetaException e) { + throw new HiveException("Unable to get partition's directory", e); + } + URI tableDir = tbl.getDataLocation(); + if(tableDir == null) { + throw new HiveException("Table has no location set"); + } + + String standardLocation = (new Path(tableDir.toString(), subdir)).toString(); + if(Utilities.isArchived(p)) { + return !getOriginalLocation(p).equals(standardLocation); + } else { + return !p.getLocation().equals(standardLocation); + } + } + private int archive(Hive db, AlterTableSimpleDesc simpleDesc, DriverContext driverContext) throws HiveException { String dbName = simpleDesc.getDbName(); @@ -1089,35 +1260,42 @@ Table tbl = db.getTable(dbName, tblName); validateAlterTableType(tbl, AlterTableDesc.AlterTableTypes.ARCHIVE); - Map partSpec = simpleDesc.getPartSpec(); - Partition p = db.getPartition(tbl, partSpec, false); - if (tbl.getTableType() != TableType.MANAGED_TABLE) { throw new HiveException("ARCHIVE can only be performed on managed tables"); } - if (p == null) { - throw new HiveException("Specified partition does not exist"); - } + Map partSpec = simpleDesc.getPartSpec(); + PartSpecInfo partSpecInfo = PartSpecInfo.create(tbl, partSpec); + List partitions = db.getPartitions(tbl, partSpec); - if (Utilities.isArchived(p)) { - // If there were a failure right after the metadata was updated in an - // archiving operation, it's possible that the original, unarchived files - // weren't deleted. - Path originalDir = new Path(getOriginalLocation(p)); - Path leftOverIntermediateOriginal = new Path(originalDir.getParent(), - originalDir.getName() + INTERMEDIATE_ORIGINAL_DIR_SUFFIX); + Path originalDir = null; - if (pathExists(leftOverIntermediateOriginal)) { - console.printInfo("Deleting " + leftOverIntermediateOriginal + - " left over from a previous archiving operation"); - deleteDir(leftOverIntermediateOriginal); + // when we have partial partitions specification we must assume partitions + // lie in standard place - if they were in custom locations putting + // them into one archive would involve mass amount of copying + // in full partition specification case we allow custom locations + // to keep backward compatibility + if (partitions.isEmpty()) { + throw new HiveException("No partition matches the specification"); + } else if(!partSpecInfo.full) { + // for partial specifications we need partitions to follow the scheme + for(Partition p: partitions){ + if(partitionInCustomLocation(tbl, p)) { + String message = String.format("ARCHIVE cannot run for partition " + + "groups with custom locations like %s", p.getLocation()); + throw new HiveException(message); + } } - - throw new HiveException("Specified partition is already archived"); + originalDir = partSpecInfo.createPath(tbl); + } else { + Partition p = partitions.get(0); + if(Utilities.isArchived(p)) { + originalDir = new Path(getOriginalLocation(p)); + } else { + originalDir = p.getPartitionPath(); + } } - Path originalDir = p.getPartitionPath(); Path intermediateArchivedDir = new Path(originalDir.getParent(), originalDir.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX); Path intermediateOriginalDir = new Path(originalDir.getParent(), @@ -1130,6 +1308,41 @@ throw new HiveException(e); } + URI originalUri = addSlash(originalDir.toUri()); + URI archiveURI = (new Path(originalDir, archiveName)).toUri(); + HarPathHelper harHelper = new HarPathHelper(archiveURI); + + // we checked if partitions matching specification are marked as archived + // in the metadata; if they are and their HAR locations are the same + // as we would set it later it means previous run failed and we have + // to do the recovery; if they are different it means they were archived + // in different depth, so we throw an error + boolean anyArchived = false, anyUnarchived = false; + for(Partition p: partitions) { + if(Utilities.isArchived(p)) { + // use Path because string can contain unescaped characters + URI partitionOriginalUri = addSlash((new Path(getOriginalLocation(p))).toUri()); + URI partitionHarUri = harHelper.getHarURI(partitionOriginalUri, originalUri); + if(!addSlash(p.getLocation()).equals(partitionHarUri.toString())) { + throw new HiveException(String.format("Partition %s seems to be already archived " + + "in different place", p.getName())); + } + anyArchived = true; + } else { + anyUnarchived = true; + } + } + + boolean recovery = false; + if(pathExists(intermediateArchivedDir) || pathExists(intermediateOriginalDir)) { + recovery = true; + console.printInfo("Starting recovery after failed ARCHIVE"); + } else if(!anyUnarchived) { + throw new HiveException("Partition(s) already archived"); + } else if(anyArchived) { + throw new HiveException("Metadata is only partially updated and temporary " + + "directories were deleted, cannot recover."); + } // The following steps seem roundabout, but they are meant to aid in // recovery if a failure occurs and to keep a consistent state in the FS @@ -1157,10 +1370,12 @@ // First create the archive in a tmp dir so that if the job fails, the // bad files don't pollute the filesystem - Path tmpDir = new Path(driverContext.getCtx().getExternalTmpFileURI(originalDir.toUri()), "partlevel"); + Path tmpPath = new Path(driverContext.getCtx() + .getExternalTmpFileURI(originalDir.toUri()), "partlevel"); - console.printInfo("Creating " + archiveName + " for " + originalDir.toString()); - console.printInfo("in " + tmpDir); + console.printInfo("Creating " + archiveName + + " for " + originalDir.toString()); + console.printInfo("in " + tmpPath); console.printInfo("Please wait... (this may take a while)"); // Create the Hadoop archive @@ -1169,10 +1384,10 @@ try { int maxJobNameLen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH); String jobname = String.format("Archiving %s@%s", - tbl.getTableName(), p.getName()); + tbl.getTableName(), partSpecInfo.getName()); jobname = Utilities.abbreviate(jobname, maxJobNameLen - 6); conf.setVar(HiveConf.ConfVars.HADOOPJOBNAME, jobname); - ret = shim.createHadoopArchive(conf, originalDir, tmpDir, archiveName); + ret = shim.createHadoopArchive(conf, originalDir, tmpPath, archiveName); } catch (Exception e) { throw new HiveException(e); } @@ -1182,11 +1397,11 @@ // Move from the tmp dir to an intermediate directory, in the same level as // the partition directory. e.g. .../hr=12-intermediate-archived try { - console.printInfo("Moving " + tmpDir + " to " + intermediateArchivedDir); + console.printInfo("Moving " + tmpPath + " to " + intermediateArchivedDir); if (pathExists(intermediateArchivedDir)) { throw new HiveException("The intermediate archive directory already exists."); } - fs.rename(tmpDir, intermediateArchivedDir); + fs.rename(tmpPath, intermediateArchivedDir); } catch (IOException e) { throw new HiveException("Error while moving tmp directory"); } @@ -1218,7 +1433,7 @@ // partition directory.) But re-running the archive command will allow // recovery - // Move the intermediate archived directory to the original parent directory + // Move the intermediate archi'eved directory to the original parent directory if (!pathExists(originalDir)) { console.printInfo("Moving " + intermediateArchivedDir + " to " + originalDir); @@ -1230,30 +1445,15 @@ // Record this change in the metastore try { - boolean parentSettable = - conf.getBoolVar(HiveConf.ConfVars.HIVEHARPARENTDIRSETTABLE); - - // dirInArchive is the directory within the archive that has all the files - // for this partition. With older versions of Hadoop, archiving a - // a directory would produce the same directory structure - // in the archive. So if you created myArchive.har of /tmp/myDir, the - // files in /tmp/myDir would be located under myArchive.har/tmp/myDir/* - // In this case, dirInArchive should be tmp/myDir - - // With newer versions of Hadoop, the parent directory could be specified. - // Assuming the parent directory was set to /tmp/myDir when creating the - // archive, the files can be found under myArchive.har/* - // In this case, dirInArchive should be empty - - String dirInArchive = ""; - if (!parentSettable) { - dirInArchive = originalDir.toUri().getPath(); - if(dirInArchive.length() > 1 && dirInArchive.charAt(0)=='/') { - dirInArchive = dirInArchive.substring(1); - } + for(Partition p: partitions) { + Path originalPartitionDir = p.getPartitionPath(); + URI harPartitionDir = harHelper.getHarURI(originalPartitionDir.toUri(), + originalUri); + LOG.debug("Archive URI: " + archiveURI.toString()); + LOG.debug("HAR URI: " + harPartitionDir.toString()); + setArchived(p, harPartitionDir); + db.alterPartition(tblName, p); } - setArchived(p, originalDir, dirInArchive, archiveName); - db.alterPartition(tblName, p); } catch (Exception e) { throw new HiveException("Unable to change the partition info for HAR", e); } @@ -1262,6 +1462,9 @@ // will not be deleted. The user will run ARCHIVE again to clear this up deleteDir(intermediateOriginalDir); + if(recovery) { + console.printInfo("Recovery after ARCHIVE succeeded"); + } return 0; } @@ -1276,95 +1479,160 @@ // Means user specified a table, not a partition if (simpleDesc.getPartSpec() == null) { - throw new HiveException("ARCHIVE is for partitions only"); + throw new HiveException("UNARCHIVE is for partitions only"); } - Map partSpec = simpleDesc.getPartSpec(); - Partition p = db.getPartition(tbl, partSpec, false); - if (tbl.getTableType() != TableType.MANAGED_TABLE) { throw new HiveException("UNARCHIVE can only be performed on managed tables"); } - if (p == null) { - throw new HiveException("Specified partition does not exist"); + Map partSpec = simpleDesc.getPartSpec(); + PartSpecInfo partSpecInfo = PartSpecInfo.create(tbl, partSpec); + List partitions = db.getPartitions(tbl, partSpec); + + Path originalDir = null; + + // when we have partial partitions specification we must assume partitions + // lie in standard place - if they were in custom locations putting + // them into one archive would involve mass amount of copying + // in full partition specification case we allow custom locations + // to keep backward compatibility + if (partitions.isEmpty()) { + throw new HiveException("No partition matches the specification"); + } else if(!partSpecInfo.full) { + // for partial specifications we need partitions to follow the scheme + for(Partition p: partitions){ + if(partitionInCustomLocation(tbl, p)) { + String message = String.format("ARCHIVE cannot run for partition " + + "groups with custom locations like %s", p.getLocation()); + throw new HiveException(message); + } + } + originalDir = partSpecInfo.createPath(tbl); + } else { + Partition p = partitions.get(0); + if(Utilities.isArchived(p)) { + originalDir = new Path(getOriginalLocation(p)); + } else { + originalDir = p.getPartitionPath(); + } } - if (!Utilities.isArchived(p)) { - Path location = new Path(p.getLocation()); - Path leftOverArchiveDir = new Path(location.getParent(), - location.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX); + URI originalUri = addSlash(originalDir.toUri()); + Path intermediateArchivedDir = new Path(originalDir.getParent(), + originalDir.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX); + Path intermediateExtractedDir = new Path(originalDir.getParent(), + originalDir.getName() + INTERMEDIATE_EXTRACTED_DIR_SUFFIX); + String archiveName = "data.har"; + FileSystem fs = null; + try { + fs = originalDir.getFileSystem(conf); + } catch (IOException e) { + throw new HiveException(e); + } - if (pathExists(leftOverArchiveDir)) { - console.printInfo("Deleting " + leftOverArchiveDir + " left over " + - "from a previous unarchiving operation"); - deleteDir(leftOverArchiveDir); + // if nothing is archived we throw an error saying partition not archived + // if only some partitions are archived we check if these are archived + // in right place and if so, it means we are during the recovery + boolean anyUnarchived = false, anyArchived = false; + for(Partition p: partitions) { + if(!Utilities.isArchived(p)) { + anyUnarchived = true; + } else { + anyArchived = true; } + } - throw new HiveException("Specified partition is not archived"); + // assume the archive is in the original dir, check if it exists + Path archivePath = new Path(originalDir, archiveName); + URI archiveURI = archivePath.toUri(); + HarPathHelper harHelper = new HarPathHelper(archiveURI); + URI sourceUri = harHelper.getHarURI(originalUri, originalUri); + Path sourceDir = new Path(sourceUri.getScheme(), sourceUri.getAuthority(), sourceUri.getPath()); + + // check if all partitions' metadata tells they lie in the archive + for(Partition p: partitions) { + if(Utilities.isArchived(p)){ + // use Path because string can contain unescaped characters + URI partitionOriginalUri = addSlash((new Path(getOriginalLocation(p))).toUri()); + URI partitionHarUri = harHelper.getHarURI(partitionOriginalUri, originalUri); + LOG.debug(p.getLocation()); + LOG.debug(partitionHarUri.toString()); + if(!addSlash(p.getLocation()).equals(partitionHarUri.toString())) { + throw new HiveException(String.format("Partition %s seems to be archived " + + "in wrong place", p.getName())); + } + } } - Path originalLocation = new Path(getOriginalLocation(p)); - Path sourceDir = new Path(p.getLocation()); - Path intermediateArchiveDir = new Path(originalLocation.getParent(), - originalLocation.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX); - Path intermediateExtractedDir = new Path(originalLocation.getParent(), - originalLocation.getName() + INTERMEDIATE_EXTRACTED_DIR_SUFFIX); + boolean recovery = false; + if(pathExists(intermediateArchivedDir) || pathExists(intermediateExtractedDir)) { + recovery = true; + console.printInfo("Starting recovery after failed UNARCHIVE"); + } else if(!anyArchived) { + throw new HiveException("Partition(s) not archived"); + } else if(anyUnarchived) { + throw new HiveException("Metadata is only partially updated and temporary " + + "directories were deleted, cannot recover."); + } - Path tmpDir = new Path(driverContext + if(!pathExists(intermediateArchivedDir) && !pathExists(archivePath)) { + throw new HiveException("Haven't found any archive where it should be"); + } + + Path tmpPath = new Path(driverContext .getCtx() - .getExternalTmpFileURI(originalLocation.toUri())); + .getExternalTmpFileURI(originalDir.toUri())); - FileSystem fs = null; try { - fs = tmpDir.getFileSystem(conf); + fs = tmpPath.getFileSystem(conf); + /* // Verify that there are no files in the tmp dir, because if there are, it // would be copied to the partition - FileStatus [] filesInTmpDir = fs.listStatus(tmpDir); + FileStatus [] filesInTmpDir = fs.listStatus(tmpPath); if (filesInTmpDir != null && filesInTmpDir.length != 0) { for (FileStatus file : filesInTmpDir) { console.printInfo(file.getPath().toString()); } - throw new HiveException("Temporary directory " + tmpDir + " is not empty"); - } + throw new HiveException("Temporary directory " + tmpPath + " is not empty"); + }*/ + //TODO check if we can remove this -- after all it's not dir now } catch (IOException e) { throw new HiveException(e); } // Some sanity checks - if (originalLocation == null) { + // TODO maybe do this earlier? + if (originalDir == null) { throw new HiveException("Missing archive data in the partition"); } - if (!"har".equals(sourceDir.toUri().getScheme())) { - throw new HiveException("Location should refer to a HAR"); - } // Clarification of terms: - // - The originalLocation directory represents the original directory of the - // partition's files. They now contain an archived version of those files + // - The originalDir directory represents the original directory of the + // partitions' files. They now contain an archived version of those files // eg. hdfs:/warehouse/myTable/ds=1/ // - The source directory is the directory containing all the files that - // should be in the partition. e.g. har:/warehouse/myTable/ds=1/myTable.har/ + // should be in the partitions. e.g. har:/warehouse/myTable/ds=1/myTable.har/ // Note the har:/ scheme // Steps: // 1. Extract the archive in a temporary folder // 2. Move the archive dir to an intermediate dir that is in at the same // dir as originalLocation. Call the new dir intermediate-extracted. - // 3. Rename the original partition dir to an intermediate dir. Call the + // 3. Rename the original partitions dir to an intermediate dir. Call the // renamed dir intermediate-archive - // 4. Rename intermediate-extracted to the original partition dir + // 4. Rename intermediate-extracted to the original partitions dir // 5. Change the metadata - // 6. Delete the archived partition files in intermediate-archive + // 6. Delete the archived partitions files in intermediate-archive if (!pathExists(intermediateExtractedDir) && - !pathExists(intermediateArchiveDir)) { + !pathExists(intermediateArchivedDir)) { try { // Copy the files out of the archive into the temporary directory String copySource = sourceDir.toString(); - String copyDest = tmpDir.toString(); + String copyDest = tmpPath.toString(); List args = new ArrayList(); args.add("-cp"); args.add(copySource); @@ -1382,12 +1650,12 @@ throw new HiveException("Error while copying files from archive"); } - console.printInfo("Moving " + tmpDir + " to " + intermediateExtractedDir); + console.printInfo("Moving " + tmpPath + " to " + intermediateExtractedDir); if (fs.exists(intermediateExtractedDir)) { throw new HiveException("Invalid state: the intermediate extracted " + "directory already exists."); } - fs.rename(tmpDir, intermediateExtractedDir); + fs.rename(tmpPath, intermediateExtractedDir); } catch (Exception e) { throw new HiveException(e); } @@ -1396,15 +1664,15 @@ // At this point, we know that the extracted files are in the intermediate // extracted dir, or in the the original directory. - if (!pathExists(intermediateArchiveDir)) { + if (!pathExists(intermediateArchivedDir)) { try { - console.printInfo("Moving " + originalLocation + " to " + intermediateArchiveDir); - fs.rename(originalLocation, intermediateArchiveDir); + console.printInfo("Moving " + originalDir + " to " + intermediateArchivedDir); + fs.rename(originalDir, intermediateArchivedDir); } catch (IOException e) { throw new HiveException(e); } } else { - console.printInfo(intermediateArchiveDir + " already exists. " + + console.printInfo(intermediateArchivedDir + " already exists. " + "Assuming it contains the archived version of the partition"); } @@ -1414,28 +1682,35 @@ // If the original location exists here, then it must be the extracted files // because in the previous step, we moved the previous original location // (containing the archived version of the files) to intermediateArchiveDir - if (!pathExists(originalLocation)) { + if (!pathExists(originalDir)) { try { - console.printInfo("Moving " + intermediateExtractedDir + " to " + originalLocation); - fs.rename(intermediateExtractedDir, originalLocation); + console.printInfo("Moving " + intermediateExtractedDir + " to " + originalDir); + fs.rename(intermediateExtractedDir, originalDir); } catch (IOException e) { throw new HiveException(e); } } else { - console.printInfo(originalLocation + " already exists. " + + console.printInfo(originalDir + " already exists. " + "Assuming it contains the extracted files in the partition"); } - setUnArchived(p); - try { - db.alterPartition(tblName, p); - } catch (InvalidOperationException e) { - throw new HiveException(e); + for(Partition p: partitions) { + setUnArchived(p); + try { + db.alterPartition(tblName, p); + } catch (InvalidOperationException e) { + throw new HiveException(e); + } } + // If a failure happens here, the intermediate archive files won't be // deleted. The user will need to call unarchive again to clear those up. - deleteDir(intermediateArchiveDir); + deleteDir(intermediateArchivedDir); + if(recovery) { + console.printInfo("Recovery after UNARCHIVE succeeded"); + } + return 0; }