Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java =================================================================== --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (revision 1153271) +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (working copy) @@ -27,8 +27,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Map.Entry; -import java.util.Properties; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -52,9 +52,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge; import org.apache.hadoop.util.StringUtils; Index: metastore/src/gen/thrift/gen-py/hive_metastore/constants.py =================================================================== --- metastore/src/gen/thrift/gen-py/hive_metastore/constants.py (revision 1153271) +++ metastore/src/gen/thrift/gen-py/hive_metastore/constants.py (working copy) @@ -13,6 +13,8 @@ HIVE_FILTER_FIELD_LAST_ACCESS = "hive_filter_field_last_access__" IS_ARCHIVED = "is_archived" ORIGINAL_LOCATION = "original_location" +ARCHIVING_LEVEL = "archiving_level" +CONTAINS_ARCHIVED_PARTITIONS = "contains_archived_partitions" META_TABLE_COLUMNS = "columns" META_TABLE_COLUMN_TYPES = "columns.types" BUCKET_FIELD_NAME = "bucket_field_name" Index: metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h =================================================================== --- metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h (revision 1153271) +++ metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h (working copy) @@ -20,6 +20,8 @@ std::string HIVE_FILTER_FIELD_LAST_ACCESS; std::string IS_ARCHIVED; std::string ORIGINAL_LOCATION; + std::string ARCHIVING_LEVEL; + std::string CONTAINS_ARCHIVED_PARTITIONS; std::string META_TABLE_COLUMNS; std::string META_TABLE_COLUMN_TYPES; std::string BUCKET_FIELD_NAME; Index: metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp =================================================================== --- metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp (revision 1153271) +++ metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp (working copy) @@ -22,6 +22,10 @@ ORIGINAL_LOCATION = "original_location"; + ARCHIVING_LEVEL = "archiving_level"; + + CONTAINS_ARCHIVED_PARTITIONS = "contains_archived_partitions"; + META_TABLE_COLUMNS = "columns"; META_TABLE_COLUMN_TYPES = "columns.types"; Index: metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb =================================================================== --- metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb (revision 1153271) +++ metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb (working copy) @@ -18,6 +18,10 @@ ORIGINAL_LOCATION = %q"original_location" +ARCHIVING_LEVEL = %q"archiving_level" + +CONTAINS_ARCHIVED_PARTITIONS = %q"contains_archived_partitions" + META_TABLE_COLUMNS = %q"columns" META_TABLE_COLUMN_TYPES = %q"columns.types" Index: metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Constants.java =================================================================== --- metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Constants.java (revision 1153271) +++ metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Constants.java (working copy) @@ -34,6 +34,10 @@ public static final String ORIGINAL_LOCATION = "original_location"; + public static final String ARCHIVING_LEVEL = "archiving_level"; + + public static final String CONTAINS_ARCHIVED_PARTITIONS = "contains_archived_partitions"; + public static final String META_TABLE_COLUMNS = "columns"; public static final String META_TABLE_COLUMN_TYPES = "columns.types"; Index: metastore/src/gen/thrift/gen-php/hive_metastore/hive_metastore_constants.php =================================================================== --- metastore/src/gen/thrift/gen-php/hive_metastore/hive_metastore_constants.php (revision 1153271) +++ metastore/src/gen/thrift/gen-php/hive_metastore/hive_metastore_constants.php (working copy) @@ -20,6 +20,10 @@ $GLOBALS['hive_metastore_CONSTANTS']['ORIGINAL_LOCATION'] = "original_location"; +$GLOBALS['hive_metastore_CONSTANTS']['ARCHIVING_LEVEL'] = "archiving_level"; + +$GLOBALS['hive_metastore_CONSTANTS']['CONTAINS_ARCHIVED_PARTITIONS'] = "contains_archived_partitions"; + $GLOBALS['hive_metastore_CONSTANTS']['META_TABLE_COLUMNS'] = "columns"; $GLOBALS['hive_metastore_CONSTANTS']['META_TABLE_COLUMN_TYPES'] = "columns.types"; Index: metastore/if/hive_metastore.thrift =================================================================== --- metastore/if/hive_metastore.thrift (revision 1153271) +++ metastore/if/hive_metastore.thrift (working copy) @@ -454,6 +454,11 @@ // this directory will contain the archive. When the partition // is dropped, this directory will be deleted const string ORIGINAL_LOCATION = "original_location", +// Level of archiving, when table is partitioned by ds, hr then if +// partition was archived at ds level it is 1 at hr level it is 2 +const string ARCHIVING_LEVEL = "archiving_level"; +// Counts archived partitions in table +const string CONTAINS_ARCHIVED_PARTITIONS = "contains_archived_partitions"; // these should be needed only for backward compatibility with filestore const string META_TABLE_COLUMNS = "columns", @@ -469,6 +474,3 @@ const string FILE_INPUT_FORMAT = "file.inputformat", const string FILE_OUTPUT_FORMAT = "file.outputformat", const string META_TABLE_STORAGE = "storage_handler", - - - Index: data/files/archive_corrupt.rc =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: data/files/archive_corrupt.rc ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java =================================================================== --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1153271) +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy) @@ -425,6 +425,7 @@ // For har files HIVEARCHIVEENABLED("hive.archive.enabled", false), HIVEHARPARENTDIRSETTABLE("hive.archive.har.parentdir.settable", false), + HIVEARCHIVECONTAINSARCHIVEDUPDATE("hive.archive.contains.archived.update", true), HIVEOUTERJOINSUPPORTSFILTERS("hive.outerjoin.supports.filters", true), // Serde for FetchTask Index: ql/src/test/results/clientnegative/archive1.q.out =================================================================== --- ql/src/test/results/clientnegative/archive1.q.out (revision 1153271) +++ ql/src/test/results/clientnegative/archive1.q.out (working copy) @@ -35,5 +35,5 @@ PREHOOK: type: ALTERTABLE_ARCHIVE PREHOOK: Input: default@srcpart_archived PREHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12 -FAILED: Error in metadata: Specified partition is already archived +FAILED: Error in metadata: Partition(s) already archived FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_multi1.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_multi1.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_multi1.q.out (revision 0) @@ -0,0 +1,58 @@ +PREHOOK: query: -- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +FAILED: Error in metadata: Partition(s) already archived +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_insert3.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_insert3.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_insert3.q.out (revision 0) @@ -0,0 +1,34 @@ +PREHOOK: query: -- Tests trying to create partition inside of archived directory. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to create partition inside of archived directory. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +FAILED: Error in semantic analysis: Insert conflict with existing archive: ds=2008-04-08 Index: ql/src/test/results/clientnegative/archive_multi4.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_multi4.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_multi4.q.out (revision 0) @@ -0,0 +1,57 @@ +PREHOOK: query: -- Tests trying to archive inner partition contained in archived partition group. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to archive inner partition contained in archived partition group. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +FAILED: Error in metadata: Conflict with existing archive ds=2008-04-08 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_partspec2.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_partspec2.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_partspec2.q.out (revision 0) @@ -0,0 +1,24 @@ +PREHOOK: query: -- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE srcpart_archived LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE srcpart_archived LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@srcpart_archived +PREHOOK: query: INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: srcpart_archived PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_archived PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +FAILED: Error in semantic analysis: partition spec is not prefix: hr is bound while ds is not Index: ql/src/test/results/clientnegative/archive_multi7.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_multi7.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_multi7.q.out (revision 0) @@ -0,0 +1,41 @@ +PREHOOK: query: -- Tests trying to archive a partition group with custom locations. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to archive a partition group with custom locations. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ADD PARTITION (ds='2008-04-08', hr='12') +LOCATION "/tmp/tstsrc" +PREHOOK: type: ALTERTABLE_ADDPARTS +PREHOOK: Input: default@tstsrcpart +POSTHOOK: query: ALTER TABLE tstsrcpart ADD PARTITION (ds='2008-04-08', hr='12') +LOCATION "/tmp/tstsrc" +POSTHOOK: type: ALTERTABLE_ADDPARTS +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +FAILED: Error in metadata: ARCHIVE cannot run for partition groups with custom locations like pfile:/tmp/tstsrc +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_insert1.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_insert1.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_insert1.q.out (revision 0) @@ -0,0 +1,34 @@ +PREHOOK: query: -- Tests trying to insert into archived partition. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to insert into archived partition. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +FAILED: Error in semantic analysis: Insert conflict with existing archive: ds=2008-04-08/hr=12 Index: ql/src/test/results/clientnegative/archive2.q.out =================================================================== --- ql/src/test/results/clientnegative/archive2.q.out (revision 1153271) +++ ql/src/test/results/clientnegative/archive2.q.out (working copy) @@ -29,5 +29,5 @@ PREHOOK: type: ALTERTABLE_UNARCHIVE PREHOOK: Input: default@tstsrcpart PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 -FAILED: Error in metadata: Specified partition is not archived +FAILED: Error in metadata: Partition(s) not archived FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_multi2.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_multi2.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_multi2.q.out (revision 0) @@ -0,0 +1,47 @@ +PREHOOK: query: -- Tests trying to unarchive a non-archived partition group +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +drop table tstsrcpart +PREHOOK: type: DROPTABLE +POSTHOOK: query: -- Tests trying to unarchive a non-archived partition group +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +drop table tstsrcpart +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table tstsrcpart like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table tstsrcpart like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr='12') +PREHOOK: type: ALTERTABLE_UNARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +FAILED: Error in metadata: Partition(s) not archived +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_insert4.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_insert4.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_insert4.q.out (revision 0) @@ -0,0 +1,34 @@ +PREHOOK: query: -- Tests trying to (possible) dynamic insert into archived partition. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to (possible) dynamic insert into archived partition. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +FAILED: Error in semantic analysis: Insert conflict with existing archive: ds=2008-04-08/hr=12 Index: ql/src/test/results/clientnegative/archive_multi5.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_multi5.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_multi5.q.out (revision 0) @@ -0,0 +1,56 @@ +PREHOOK: query: -- Tests trying to unarchive outer partition group containing other partition inside. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to unarchive outer partition group containing other partition inside. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_UNARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +FAILED: Error in metadata: Archiving was at different level, use ds=2008-04-08/hr=12 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_partspec3.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_partspec3.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_partspec3.q.out (revision 0) @@ -0,0 +1,25 @@ +PREHOOK: query: -- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE srcpart_archived LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE srcpart_archived LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@srcpart_archived +PREHOOK: query: INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: srcpart_archived PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_archived PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +FAILED: Parse Error: line 3:48 mismatched input ')' expecting Identifier near '(' in archive statement + Index: ql/src/test/results/clientnegative/archive_insert2.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_insert2.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_insert2.q.out (revision 0) @@ -0,0 +1,34 @@ +PREHOOK: query: -- Tests trying to insert into archived partition. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to insert into archived partition. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +FAILED: Error in semantic analysis: Insert conflict with existing archive: ds=2008-04-08 Index: ql/src/test/results/clientnegative/archive_multi3.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_multi3.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_multi3.q.out (revision 0) @@ -0,0 +1,56 @@ +PREHOOK: query: -- Tests trying to archive outer partition group containing other partition inside. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to archive outer partition group containing other partition inside. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +FAILED: Error in metadata: Conflict with existing archive ds=2008-04-08/hr=12 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientnegative/archive_partspec1.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_partspec1.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_partspec1.q.out (revision 0) @@ -0,0 +1,24 @@ +PREHOOK: query: -- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE srcpart_archived LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE srcpart_archived LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@srcpart_archived +PREHOOK: query: INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: srcpart_archived PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: srcpart_archived PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +FAILED: Error in semantic analysis: partition spec is not valid: {ds=2008-04-08, nonexistingpart=12} Index: ql/src/test/results/clientnegative/archive_multi6.q.out =================================================================== --- ql/src/test/results/clientnegative/archive_multi6.q.out (revision 0) +++ ql/src/test/results/clientnegative/archive_multi6.q.out (revision 0) @@ -0,0 +1,57 @@ +PREHOOK: query: -- Tests trying to unarchive inner partition contained in archived partition group. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: -- Tests trying to unarchive inner partition contained in archived partition group. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr='12') +PREHOOK: type: ALTERTABLE_UNARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +FAILED: Error in metadata: Archiving was at different level, use ds=2008-04-08 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask Index: ql/src/test/results/clientpositive/archive_corrupt.q.out =================================================================== --- ql/src/test/results/clientpositive/archive_corrupt.q.out (revision 0) +++ ql/src/test/results/clientpositive/archive_corrupt.q.out (revision 0) @@ -0,0 +1,122 @@ +PREHOOK: query: drop table tstsrcpart +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tstsrcpart +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table tstsrcpart like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table tstsrcpart like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +PREHOOK: query: load data local inpath '../data/files/archive_corrupt.rc' overwrite into table tstsrcpart partition (ds='2008-04-08', hr='11') +PREHOOK: type: LOAD +PREHOOK: Output: default@tstsrcpart +POSTHOOK: query: load data local inpath '../data/files/archive_corrupt.rc' overwrite into table tstsrcpart partition (ds='2008-04-08', hr='11') +POSTHOOK: type: LOAD +POSTHOOK: Output: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11') +select key, value from srcpart where ds='2008-04-09' and hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=11 +POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11') +select key, value from srcpart where ds='2008-04-09' and hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12') +select key, value from srcpart where ds='2008-04-09' and hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=12 +POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12') +select key, value from srcpart where ds='2008-04-09' and hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +describe extended tstsrcpart partition (ds='2008-04-08', hr='11') +PREHOOK: type: DESCTABLE +POSTHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +describe extended tstsrcpart partition (ds='2008-04-08', hr='11') +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +key string default +value string default +ds string +hr string + +Detailed Partition Information Partition(values:[2008-04-08, 11], dbName:default, tableName:tstsrcpart, createTime:1312317091, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/marcin/hive/build/ql/test/data/warehouse/tstsrcpart/ds=2008-04-08/hr=11, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{transient_lastDdlTime=1312317091}) +PREHOOK: query: alter table tstsrcpart archive partition (ds='2008-04-08', hr='11') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: alter table tstsrcpart archive partition (ds='2008-04-08', hr='11') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: describe extended tstsrcpart partition (ds='2008-04-08', hr='11') +PREHOOK: type: DESCTABLE +POSTHOOK: query: describe extended tstsrcpart partition (ds='2008-04-08', hr='11') +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +key string default +value string default +ds string +hr string + +Detailed Partition Information Partition(values:[2008-04-08, 11], dbName:default, tableName:tstsrcpart, createTime:1312317091, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:har://pfile/data/users/marcin/hive/build/ql/test/data/warehouse/tstsrcpart/ds=2008-04-08/hr=11/data.har/data/users/marcin/hive/build/ql/test/data/warehouse/tstsrcpart/ds=2008-04-08/hr=11/, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{archiving_offset=2, transient_lastDdlTime=1312317109, is_archived=true, original_location=pfile:/data/users/marcin/hive/build/ql/test/data/warehouse/tstsrcpart/ds=2008-04-08/hr=11}) +PREHOOK: query: alter table tstsrcpart unarchive partition (ds='2008-04-08', hr='11') +PREHOOK: type: ALTERTABLE_UNARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: alter table tstsrcpart unarchive partition (ds='2008-04-08', hr='11') +POSTHOOK: type: ALTERTABLE_UNARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] Index: ql/src/test/results/clientpositive/archive_multi.q.out =================================================================== --- ql/src/test/results/clientpositive/archive_multi.q.out (revision 0) +++ ql/src/test/results/clientpositive/archive_multi.q.out (revision 0) @@ -0,0 +1,261 @@ +PREHOOK: query: drop table tstsrc +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tstsrc +POSTHOOK: type: DROPTABLE +PREHOOK: query: drop table tstsrcpart +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table tstsrcpart +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table tstsrc like src +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table tstsrc like src +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrc +PREHOOK: query: insert overwrite table tstsrc select key, value from src +PREHOOK: type: QUERY +PREHOOK: Input: default@src +PREHOOK: Output: default@tstsrc +POSTHOOK: query: insert overwrite table tstsrc select key, value from src +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +POSTHOOK: Output: default@tstsrc +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: create table tstsrcpart like srcpart +PREHOOK: type: CREATETABLE +POSTHOOK: query: create table tstsrcpart like srcpart +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: default@tstsrcpart +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11') +select key, value from srcpart where ds='2008-04-09' and hr='11' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=11 +POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11') +select key, value from srcpart where ds='2008-04-09' and hr='11' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=11 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12') +select key, value from srcpart where ds='2008-04-09' and hr='12' +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=12 +POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12') +select key, value from srcpart where ds='2008-04-09' and hr='12' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=12 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-21_228_2560964695734311201/-mr-10000 +POSTHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-21_228_2560964695734311201/-mr-10000 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +48479881068 +PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-28_034_424539810414173134/-mr-10000 +POSTHOOK: query: SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-28_034_424539810414173134/-mr-10000 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +48479881068 +PREHOOK: query: SELECT key, count(1) FROM tstsrcpart WHERE ds='2008-04-08' AND hr='12' AND key='0' GROUP BY key +PREHOOK: type: QUERY +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-31_642_7853891038069299073/-mr-10000 +POSTHOOK: query: SELECT key, count(1) FROM tstsrcpart WHERE ds='2008-04-08' AND hr='12' AND key='0' GROUP BY key +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-31_642_7853891038069299073/-mr-10000 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +0 3 +PREHOOK: query: SELECT * FROM tstsrcpart a JOIN tstsrc b ON a.key=b.key +WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0' +PREHOOK: type: QUERY +PREHOOK: Input: default@tstsrc +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-35_365_2814905405536965778/-mr-10000 +POSTHOOK: query: SELECT * FROM tstsrcpart a JOIN tstsrc b ON a.key=b.key +WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstsrc +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-35_365_2814905405536965778/-mr-10000 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +0 val_0 2008-04-08 12 0 val_0 +PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_UNARCHIVE +PREHOOK: Input: default@tstsrcpart +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08') +POSTHOOK: type: ALTERTABLE_UNARCHIVE +POSTHOOK: Input: default@tstsrcpart +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2 +PREHOOK: type: QUERY +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +PREHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-40_944_4568299913242010267/-mr-10000 +POSTHOOK: query: SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-40_944_4568299913242010267/-mr-10000 +POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +48479881068 Index: ql/src/test/queries/clientnegative/archive_multi5.q =================================================================== --- ql/src/test/queries/clientnegative/archive_multi5.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_multi5.q (revision 0) @@ -0,0 +1,13 @@ +set hive.archive.enabled = true; +-- Tests trying to unarchive outer partition group containing other partition inside. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12'); +ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08'); Index: ql/src/test/queries/clientnegative/archive_multi6.q =================================================================== --- ql/src/test/queries/clientnegative/archive_multi6.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_multi6.q (revision 0) @@ -0,0 +1,13 @@ +set hive.archive.enabled = true; +-- Tests trying to unarchive inner partition contained in archived partition group. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); +ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr='12'); Index: ql/src/test/queries/clientnegative/archive_multi7.q =================================================================== --- ql/src/test/queries/clientnegative/archive_multi7.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_multi7.q (revision 0) @@ -0,0 +1,12 @@ +set hive.archive.enabled = true; +-- Tests trying to archive a partition group with custom locations. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; +ALTER TABLE tstsrcpart ADD PARTITION (ds='2008-04-08', hr='12') +LOCATION "/tmp/tstsrc"; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); Index: ql/src/test/queries/clientnegative/archive_insert1.q =================================================================== --- ql/src/test/queries/clientnegative/archive_insert1.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_insert1.q (revision 0) @@ -0,0 +1,13 @@ +set hive.archive.enabled = true; +-- Tests trying to insert into archived partition. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12'); + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; Index: ql/src/test/queries/clientnegative/archive_insert2.q =================================================================== --- ql/src/test/queries/clientnegative/archive_insert2.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_insert2.q (revision 0) @@ -0,0 +1,13 @@ +set hive.archive.enabled = true; +-- Tests trying to insert into archived partition. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; Index: ql/src/test/queries/clientnegative/archive_multi1.q =================================================================== --- ql/src/test/queries/clientnegative/archive_multi1.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_multi1.q (revision 0) @@ -0,0 +1,13 @@ +set hive.archive.enabled = true; +-- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); Index: ql/src/test/queries/clientnegative/archive_partspec1.q =================================================================== --- ql/src/test/queries/clientnegative/archive_partspec1.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_partspec1.q (revision 0) @@ -0,0 +1,10 @@ +set hive.archive.enabled = true; +-- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE srcpart_archived LIKE srcpart; + +INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE srcpart_archived ARCHIVE PARTITION (ds='2008-04-08', nonexistingpart='12'); Index: ql/src/test/queries/clientnegative/archive_insert3.q =================================================================== --- ql/src/test/queries/clientnegative/archive_insert3.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_insert3.q (revision 0) @@ -0,0 +1,13 @@ +set hive.archive.enabled = true; +-- Tests trying to create partition inside of archived directory. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; Index: ql/src/test/queries/clientnegative/archive_multi2.q =================================================================== --- ql/src/test/queries/clientnegative/archive_multi2.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_multi2.q (revision 0) @@ -0,0 +1,12 @@ +set hive.archive.enabled = true; +-- Tests trying to unarchive a non-archived partition group +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +drop table tstsrcpart; +create table tstsrcpart like srcpart; +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; +insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12'; + +ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr='12'); Index: ql/src/test/queries/clientnegative/archive_partspec2.q =================================================================== --- ql/src/test/queries/clientnegative/archive_partspec2.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_partspec2.q (revision 0) @@ -0,0 +1,10 @@ +set hive.archive.enabled = true; +-- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE srcpart_archived LIKE srcpart; + +INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE srcpart_archived ARCHIVE PARTITION (hr='12'); Index: ql/src/test/queries/clientnegative/archive_insert4.q =================================================================== --- ql/src/test/queries/clientnegative/archive_insert4.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_insert4.q (revision 0) @@ -0,0 +1,15 @@ +set hive.archive.enabled = true; +-- Tests trying to (possible) dynamic insert into archived partition. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12'); + +SET hive.exec.dynamic.partition=true; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr) +SELECT key, value, hr FROM srcpart WHERE ds='2008-04-08' AND hr='12'; Index: ql/src/test/queries/clientnegative/archive_multi3.q =================================================================== --- ql/src/test/queries/clientnegative/archive_multi3.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_multi3.q (revision 0) @@ -0,0 +1,13 @@ +set hive.archive.enabled = true; +-- Tests trying to archive outer partition group containing other partition inside. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12'); +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); Index: ql/src/test/queries/clientnegative/archive_partspec3.q =================================================================== --- ql/src/test/queries/clientnegative/archive_partspec3.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_partspec3.q (revision 0) @@ -0,0 +1,10 @@ +set hive.archive.enabled = true; +-- Tests trying to archive a partition twice. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE srcpart_archived LIKE srcpart; + +INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE srcpart_archived ARCHIVE PARTITION (); Index: ql/src/test/queries/clientnegative/archive_multi4.q =================================================================== --- ql/src/test/queries/clientnegative/archive_multi4.q (revision 0) +++ ql/src/test/queries/clientnegative/archive_multi4.q (revision 0) @@ -0,0 +1,13 @@ +set hive.archive.enabled = true; +-- Tests trying to archive inner partition contained in archived partition group. +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +CREATE TABLE tstsrcpart LIKE srcpart; + +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'; +INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12') +SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12'); Index: ql/src/test/queries/clientpositive/archive_multi.q =================================================================== --- ql/src/test/queries/clientpositive/archive_multi.q (revision 0) +++ ql/src/test/queries/clientpositive/archive_multi.q (revision 0) @@ -0,0 +1,42 @@ +set hive.archive.enabled = true; +set hive.enforce.bucketing = true; + +drop table tstsrc; +drop table tstsrcpart; + +create table tstsrc like src; +insert overwrite table tstsrc select key, value from src; + +create table tstsrcpart like srcpart; + +insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='11') +select key, value from srcpart where ds='2008-04-08' and hr='11'; + +insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12'; + +insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11') +select key, value from srcpart where ds='2008-04-09' and hr='11'; + +insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12') +select key, value from srcpart where ds='2008-04-09' and hr='12'; + +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2; + +ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); + +SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2; + +SELECT key, count(1) FROM tstsrcpart WHERE ds='2008-04-08' AND hr='12' AND key='0' GROUP BY key; + +SELECT * FROM tstsrcpart a JOIN tstsrc b ON a.key=b.key +WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0'; + +ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08'); + +SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col +FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2; Index: ql/src/test/queries/clientpositive/archive_corrupt.q =================================================================== --- ql/src/test/queries/clientpositive/archive_corrupt.q (revision 0) +++ ql/src/test/queries/clientpositive/archive_corrupt.q (revision 0) @@ -0,0 +1,27 @@ +set hive.archive.enabled = true; +set hive.enforce.bucketing = true; + +drop table tstsrcpart; + +create table tstsrcpart like srcpart; + +load data local inpath '../data/files/archive_corrupt.rc' overwrite into table tstsrcpart partition (ds='2008-04-08', hr='11'); + +insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12') +select key, value from srcpart where ds='2008-04-08' and hr='12'; + +insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11') +select key, value from srcpart where ds='2008-04-09' and hr='11'; + +insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12') +select key, value from srcpart where ds='2008-04-09' and hr='12'; + +-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19) + +describe extended tstsrcpart partition (ds='2008-04-08', hr='11'); + +alter table tstsrcpart archive partition (ds='2008-04-08', hr='11'); + +describe extended tstsrcpart partition (ds='2008-04-08', hr='11'); + +alter table tstsrcpart unarchive partition (ds='2008-04-08', hr='11'); Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java (revision 1153271) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java (working copy) @@ -18,8 +18,14 @@ package org.apache.hadoop.hive.ql.metadata; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.metastore.api.FieldSchema; /** * A Hive Table Partition: is a fundamental storage unit within a Table. Currently, Hive does not support @@ -35,14 +41,26 @@ .getLog("hive.ql.metadata.DummyPartition"); private String name; + private LinkedHashMap partSpec; + private Table table; + public DummyPartition() { } - public DummyPartition(Table tbl, String name) throws HiveException { + public DummyPartition(Table tbl, String name, Map partSpec) + throws HiveException { setTable(tbl); this.name = name; + this.partSpec = new LinkedHashMap(partSpec); + this.table = tbl; } + @Override + public Table getTable() { + return table; + } + + @Override public String getName() { return name; } @@ -51,7 +69,24 @@ this.name = name; } + @Override public String getCompleteName() { return getName(); } + + @Override + public LinkedHashMap getSpec() { + return partSpec; + } + + @Override + public List getValues() { + List values = new ArrayList(); + for(FieldSchema fs: table.getPartCols()) { + values.add(partSpec.get(fs.getName())); + } + return values; + } + + } Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (revision 1153271) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (working copy) @@ -355,6 +355,55 @@ return true; } + /** + * Checks if given specification is proper specification for prefix of + * partition cols, for table partitioned by ds, hr, min valid ones are + * (ds='2008-04-08'), (ds='2008-04-08', hr='12'), (ds='2008-04-08', hr='12', min='30') + * invalid one is for example (ds='2008-04-08', min='30') + * @param spec specification key-value map + * @return true if the specification is prefix; never returns false, but throws + * @throws HiveException + */ + final public boolean isValidPrefixSpec(LinkedHashMap spec) + throws HiveException { + + // TODO - types need to be checked. + List partCols = tTable.getPartitionKeys(); + if (partCols == null || (partCols.size() == 0)) { + if (spec != null) { + throw new HiveException( + "table is not partitioned but partition spec exists: " + spec); + } else { + return true; + } + } + + if(spec == null) { + throw new HiveException("partition spec is not specified"); + } + + String unboundKey = null; + int boundKeys = 0; + for (FieldSchema field : partCols) { + if (spec.get(field.getName()) == null) { + unboundKey = field.getName(); + } else if(unboundKey != null) { + throw new HiveException("partition spec is not prefix: " + + field.getName() + " is bound while " + unboundKey + " is not"); + } else { + boundKeys += 1; + } + } + + if (spec.size() != boundKeys) { + throw new HiveException( + "partition spec is not valid: " + + spec); + } + + return true; + } + public void setProperty(String name, String value) { tTable.getParameters().put(name, value); } @@ -709,7 +758,7 @@ public boolean isView() { return TableType.VIRTUAL_VIEW.equals(getTableType()); } - + /** * @return whether this table is actually an index table */ Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (revision 1153271) +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (working copy) @@ -1506,14 +1506,16 @@ * * @param tbl * object for which partition is needed. Must be partitioned. + * @param limit number of partitions to return * @return list of partition objects * @throws HiveException */ - public List getPartitions(Table tbl, Map partialPartSpec) + public List getPartitions(Table tbl, Map partialPartSpec, + short limit) throws HiveException { if (!tbl.isPartitioned()) { throw new HiveException("Partition spec should only be supplied for a " + - "partitioned table"); + "partitioned table"); } List partialPvals = getPvals(tbl.getPartCols(), partialPartSpec); @@ -1521,7 +1523,7 @@ List partitions = null; try { partitions = getMSC().listPartitionsWithAuthInfo(tbl.getDbName(), tbl.getTableName(), - partialPvals, (short) -1, getUserName(), getGroupNames()); + partialPvals, limit, getUserName(), getGroupNames()); } catch (Exception e) { throw new HiveException(e); } @@ -1541,6 +1543,21 @@ * * @param tbl * object for which partition is needed. Must be partitioned. + * @return list of partition objects + * @throws HiveException + */ + public List getPartitions(Table tbl, Map partialPartSpec) + throws HiveException { + return getPartitions(tbl, partialPartSpec, (short)-1); + } + + /** + * get all the partitions of the table that matches the given partial + * specification. partition columns whose value is can be anything should be + * an empty string. + * + * @param tbl + * object for which partition is needed. Must be partitioned. * @param partialPartSpec * partial partition specification (some subpartitions can be empty). * @return list of partition objects @@ -1816,7 +1833,7 @@ // Note: there are race conditions here, but I don't believe // they're worse than what was already present. int counter = 1; - + // Strip off the file type, if any so we don't make: // 000000_0.gz -> 000000_0.gz_copy_1 String name = itemStaging.getName(); @@ -1828,10 +1845,10 @@ } else { filetype = ""; } - + Path itemDest = new Path(destf, itemStaging.getName()); Path itemStagingBase = new Path(itemStaging.getParent(), name); - + while (fs.exists(itemDest)) { Path proposedStaging = itemStagingBase .suffix("_copy_" + counter++).suffix(filetype); Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java (revision 0) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java (revision 0) @@ -0,0 +1,355 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; + +/** + * MetaUtils. + * + */ +@SuppressWarnings("nls") +public final class ArchiveUtils { + private static final Log LOG = LogFactory.getLog(ArchiveUtils.class.getName()); + + /** + * PartSpecInfo keeps fields and values extracted from partial partition info + * which is prefix of the full info. + */ + public static class PartSpecInfo { + public List fields; + public List values; + private PartSpecInfo(List fields, List values) { + this.fields = fields; + this.values = values; + } + + /** + * Extract partial prefix specification from table and key-value map + * + * @param tbl table in which partition is + * @param partSpec specification of partition; must be prefix spec, for + * table partitioned by ds, hr it must not be (hr='11') + * @return extracted specification + */ + static public PartSpecInfo create(Table tbl, LinkedHashMap partSpec) + throws HiveException { + List prefixFields = new ArrayList(); + List prefixValues = new ArrayList(); + for(FieldSchema fs: tbl.getPartCols()) { + if(partSpec.containsKey(fs.getName())) { + prefixFields.add(fs); + prefixValues.add(partSpec.get(fs.getName())); + } else { + break; + } + } + return new PartSpecInfo(prefixFields, prefixValues); + } + + /** + * Creates path where partitions matching prefix should lie in filesystem + * @param tbl table in which partition is + * @return expected location of partitions matching prefix in filesystem + */ + public Path createPath(Table tbl) throws HiveException { + String prefixSubdir; + try { + prefixSubdir = Warehouse.makePartName(fields, values); + } catch (MetaException e) { + throw new HiveException("Unable to get partitions directories prefix", e); + } + URI tableDir = tbl.getDataLocation(); + if(tableDir == null) { + throw new HiveException("Table has no location set"); + } + return new Path(tableDir.toString(), prefixSubdir); + } + /** + * Generates name for prefix partial partition specification. + */ + public String getName() throws HiveException { + try { + return Warehouse.makePartName(fields, values); + } catch (MetaException e) { + throw new HiveException("Unable to create partial name", e); + } + } + } + + /** + * HarPathHelper helps to create har:/ URIs for locations inside of archive. + */ + public static class HarPathHelper { + boolean parentSettable; + private final URI base, originalBase; + + /** + * Creates helper for archive. + * @param archive absolute location of archive in underlying filesystem + * @param originalBase directory for which Hadoop archive was created + */ + public HarPathHelper(HiveConf hconf, URI archive, URI originalBase) throws HiveException { + parentSettable = hconf.getBoolVar(HiveConf.ConfVars.HIVEHARPARENTDIRSETTABLE); + this.originalBase = addSlash(originalBase); + String parentHost = archive.getHost(); + String harHost = null; + if (parentHost == null) { + harHost = archive.getScheme(); + } else { + harHost = archive.getScheme() + "-" + parentHost; + } + + // have to make sure there's slash after .har, otherwise resolve doesn't work + String path = addSlash(archive.getPath()); + if(!path.endsWith(".har/")) { + throw new HiveException("HAR archive path must end with .har"); + } + // harUri is used to access the partition's files, which are in the archive + // The format of the RI is something like: + // har://underlyingfsscheme-host:port/archivepath + try { + base = new URI("har", archive.getUserInfo(), harHost, archive.getPort(), + path, archive.getQuery(), archive.getFragment()); + } catch (URISyntaxException e) { + throw new HiveException("Couldn't create har URI from archive URI", e); + } + } + + /** + * Creates har URI for file/directory that was put there when creating HAR. + * + * With older versions of Hadoop, archiving a directory would produce + * the same directory structure, reflecting absoulute paths. + * If you created myArchive.har of /tmp/myDir the files in /tmp/myDir + * will be located under myArchive.har/tmp/myDir/* + * + * With newer versions, the parent directory can be specified. Assuming + * the parent directory was set to /tmp/myDir when creating the archive, + * the files can be found under myArchive.har/* + * + * This is why originalBase is argument - with new versions we can + * relativize URI, in older we keep absolute one. + * + * @param original file/directory path + * @return absolute HAR uri + */ + public URI getHarUri(URI original) throws HiveException { + URI relative = null; + if (!parentSettable) { + String dirInArchive = original.getPath(); + if(dirInArchive.length() > 1 && dirInArchive.charAt(0)=='/') { + dirInArchive = dirInArchive.substring(1); + } + try { + relative = new URI(null, null, dirInArchive, null); + } catch (URISyntaxException e) { + throw new HiveException("Couldn't create har URI for location"); + } // relative URI with path only + } else { + relative = originalBase.relativize(original); + if(relative.isAbsolute()) { + throw new HiveException("Unable to relativize URI"); + } + } + return base.resolve(relative); + } + } + + public static String addSlash(String s) { + return s.endsWith("/") ? s : s + "/"; + } + + /** + * Makes sure, that URI points to directory by adding slash to it. + * Useful in relativizing URIs. + */ + public static URI addSlash(URI u) throws HiveException { + if(u.getPath().endsWith("/")) { + return u; + } else { + try { + return new URI(u.getScheme(), u.getAuthority(), u.getPath() + "/", u.getQuery(), u.getFragment()); + } catch (URISyntaxException e) { + // It cannot happen (can it?) + throw new HiveException("Couldn't append slash to a URI", e); + } + } + } + + /** + * Determines whether a partition has been archived + * + * @param p + * @return + */ + public static boolean isArchived(Partition p) { + Map params = p.getParameters(); + if ("true".equalsIgnoreCase(params.get( + org.apache.hadoop.hive.metastore.api.Constants.IS_ARCHIVED))) { + return true; + } else { + return false; + } + } + + /** + * Returns archiving level, which is how many fields were set in partial + * specification ARCHIVE was run for + */ + public static int getArchivingLevel(Partition p) throws HiveException { + if(!isArchived(p)) { + throw new HiveException("Getting level of unarchived partition"); + } + + Map params = p.getParameters(); + String lv = params.get( + org.apache.hadoop.hive.metastore.api.Constants.ARCHIVING_LEVEL); + if(lv != null) { + return Integer.parseInt(lv); + } else { // partitions archived before introducing multiple archiving + return p.getValues().size(); + } + } + + /** + * Checks if table is flagged as containing archived partition. + * @param tbl Table object + * @return Counter value + */ + public static boolean containsArchivedPartitions(Table tbl) { + String val = tbl.getProperty(org.apache.hadoop.hive.metastore + .api.Constants.CONTAINS_ARCHIVED_PARTITIONS); + return "true".equals(val); + } + + public static String getPartialName(Partition p, int level) throws HiveException { + List ffields = p.getTable().getPartCols(); + List fvalues = p.getValues(); + List fields = new ArrayList(level); + List values = new ArrayList(level); + int lev = level; + for(String s: fvalues) { + values.add(s); + if(--lev == 0) { + break; + } + } + lev = level; + for(FieldSchema fs: ffields) { + fields.add(fs); + if(--lev == 0) { + break; + } + } + try { + return Warehouse.makePartName(fields, values); + } catch (MetaException e) { + throw new HiveException("Wasn't able to generate name" + + " for partial specification"); + } + } + + /** + * Determines if one can insert into partition(s), or there's a conflict with + * archive. It can be because partition is itself archived or it is to be + * created inside existing archive. The second case is when partition doesn't + * exist yet, but it existed it would be inside of an archive. This one is + * quite tricky to check, we need to find at least one partition inside of + * the parent directory. If it is archived and archiving level tells that + * the archival was done of directory partition is in it means we cannot + * insert; otherwise we can. + * This method works both for full specifications and partial ones - in second + * case it checks if any partition that could possibly match such + * specification is inside archive. + * + * @param db - Hive object + * @param tbl - table where partition is + * @param partSpec - partition specification with possible nulls in case of + * dynamic partiton inserts + * @return null if partition can be inserted, string with colliding archive + * name when it can't + * @throws HiveException + */ + public static String conflictingArchiveNameOrNull(Hive db, Table tbl, + LinkedHashMap partSpec) + throws HiveException { + if(!containsArchivedPartitions(tbl)) { + return null; + } + List partitions = db.getPartitions(tbl, partSpec); + if(!partitions.isEmpty()) { + LOG.debug("Found some partitions matching the provided specification"); + for(Partition p: partitions) { + if(isArchived(p)) { + return getPartialName(p, getArchivingLevel(p)); + } + } + } else { + LOG.debug("Haven't found any partitions matching the provided specification"); + Map spec = new HashMap(partSpec); + List reversedKeys = new LinkedList(); + for(FieldSchema fs: tbl.getPartCols()) { + if(spec.containsKey(fs.getName())) { + reversedKeys.add(0, fs.getName()); + } + } + for(String rk: reversedKeys) { + spec.remove(rk); + List parts = db.getPartitions(tbl, spec, (short)1); + if(parts.size() != 0) { + Partition p = parts.get(0); + if(!isArchived(p)) { + // if archiving was done at this or at upper level, every matched + // partition would be archived, so it not being archived means + // no archiving was done neither at this nor at upper level + return null; + } else if(getArchivingLevel(p) > spec.size()) { + // if archiving was done at this or at upper level its level + // would be lesser or equal to specification size + // it is not, which means no archiving at this or upper level + return null; + } else { + return getPartialName(p, getArchivingLevel(p)); + } + } + } + } + return null; + } +} Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (revision 1153271) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (working copy) @@ -1929,22 +1929,6 @@ } /** - * Determines whether a partition has been archived - * - * @param p - * @return - */ - public static boolean isArchived(Partition p) { - Map params = p.getParameters(); - if ("true".equalsIgnoreCase(params.get( - org.apache.hadoop.hive.metastore.api.Constants.IS_ARCHIVED))) { - return true; - } else { - return false; - } - } - - /** * Check if a function can be pushed down to JDO. * Now only {=, AND, OR} are supported. * @param func a generic function. Index: ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision 1153271) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (working copy) @@ -35,6 +35,7 @@ import java.util.Comparator; import java.util.Date; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -73,6 +74,7 @@ import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.exec.ArchiveUtils.PartSpecInfo; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.rcfile.merge.BlockMergeTask; @@ -969,22 +971,37 @@ return 0; } - private void setIsArchived(Partition p, boolean state) { + /** + * Sets archiving flag locally; it has to be pushed into metastore + * @param p partition to set flag + * @param state desired state of IS_ARCHIVED flag + * @param level desired level for state == true, anything for false + */ + private void setIsArchived(Partition p, boolean state, int level) { Map params = p.getParameters(); if (state) { params.put(org.apache.hadoop.hive.metastore.api.Constants.IS_ARCHIVED, "true"); + params.put(org.apache.hadoop.hive.metastore.api.Constants.ARCHIVING_LEVEL, + Integer.toString(level)); } else { params.remove(org.apache.hadoop.hive.metastore.api.Constants.IS_ARCHIVED); + params.remove(org.apache.hadoop.hive.metastore.api.Constants.ARCHIVING_LEVEL); } } + /** + * Returns original partition of archived partition, null for unarchived one + */ private String getOriginalLocation(Partition p) { Map params = p.getParameters(); return params.get( org.apache.hadoop.hive.metastore.api.Constants.ORIGINAL_LOCATION); } + /** + * Sets original location of partition which is to be archived + */ private void setOriginalLocation(Partition p, String loc) { Map params = p.getParameters(); if (loc == null) { @@ -994,56 +1011,65 @@ } } - // Returns only the path component of the URI - private String getArchiveDirOnly(Path parentDir, String archiveName) { - URI parentUri = parentDir.toUri(); - Path harDir = new Path(parentUri.getPath(), archiveName); - return harDir.toString(); - } - /** * Sets the appropriate attributes in the supplied Partition object to mark * it as archived. Note that the metastore is not touched - a separate * call to alter_partition is needed. * * @param p - the partition object to modify - * @param parentDir - the parent directory of the archive, which is the - * original directory that the partition's files resided in - * @param dirInArchive - the directory within the archive file that contains - * the partitions files - * @param archiveName - the name of the archive - * @throws URISyntaxException + * @param harPath - new location of partition (har schema URI) */ - private void setArchived(Partition p, Path parentDir, String dirInArchive, String archiveName) - throws URISyntaxException { - assert(Utilities.isArchived(p) == false); - Map params = p.getParameters(); + private void setArchived(Partition p, Path harPath, int level) { + assert(ArchiveUtils.isArchived(p) == false); - URI parentUri = parentDir.toUri(); - String parentHost = parentUri.getHost(); - String harHost = null; - if (parentHost == null) { - harHost = ""; - } else { - harHost = parentUri.getScheme() + "-" + parentHost; + setIsArchived(p, true, level); + setOriginalLocation(p, p.getLocation()); + p.setLocation(harPath.toString()); + } + + /** + * Sets CONTAINS_ARCHIVED_PARTITION flag to true. + * @param db Hive object for database + * @param t table to change the flag + */ + private void setContainsArchived(Hive db, Table t) throws HiveException { + Table tbl = db.getTable(t.getTableName()); + String key = org.apache.hadoop.hive.metastore.api.Constants.CONTAINS_ARCHIVED_PARTITIONS; + tbl.getParameters().put(key, "true"); + try { + db.alterTable(t.getTableName(), tbl); + } catch (Exception e) { + throw new HiveException(e); } + } - // harUri is used to access the partition's files, which are in the archive - // The format of the RI is something like: - // har://underlyingfsscheme-host:port/archivepath - URI harUri = null; - if (dirInArchive.length() == 0) { - harUri = new URI("har", parentUri.getUserInfo(), harHost, parentUri.getPort(), - getArchiveDirOnly(parentDir, archiveName), - parentUri.getQuery(), parentUri.getFragment()); - } else { - harUri = new URI("har", parentUri.getUserInfo(), harHost, parentUri.getPort(), - new Path(getArchiveDirOnly(parentDir, archiveName), dirInArchive).toUri().getPath(), - parentUri.getQuery(), parentUri.getFragment()); + /** + * Checks if there are any archived partitions in table and sets + * CONTAINS_ARCHIVED_PARTITION flag properly. + * Omited if hive.archive.contains.archived.update is false. + * @param db Hive object for database + * @param t table to change the flag + */ + private void updateContainsArchived(Hive db, Table t) throws HiveException { + if(!conf.getBoolVar(HiveConf.ConfVars.HIVEARCHIVECONTAINSARCHIVEDUPDATE)) { + return; } - setIsArchived(p, true); - setOriginalLocation(p, parentDir.toString()); - p.setLocation(harUri.toString()); + Table tbl = db.getTable(t.getTableName()); + List partitions = db.getPartitions(tbl); + String val = "false"; + for(Partition p: partitions) { + if(ArchiveUtils.isArchived(p)) { + val = "true"; + break; + } + } + String key = org.apache.hadoop.hive.metastore.api.Constants.CONTAINS_ARCHIVED_PARTITIONS; + tbl.getParameters().put(key, val); + try { + db.alterTable(t.getTableName(), tbl); + } catch (Exception e) { + throw new HiveException(e); + } } /** @@ -1054,9 +1080,9 @@ * @param p - the partition to modify */ private void setUnArchived(Partition p) { - assert(Utilities.isArchived(p) == true); + assert(ArchiveUtils.isArchived(p) == true); String parentDir = getOriginalLocation(p); - setIsArchived(p, false); + setIsArchived(p, false, 0); setOriginalLocation(p, null); assert(parentDir != null); p.setLocation(parentDir); @@ -1090,6 +1116,33 @@ } } + /** + * Checks in partition is in custom (not-standard) location. + * @param tbl - table in which partition is + * @param p - partition + * @return true if partition location is custom, false if it is standard + */ + boolean partitionInCustomLocation(Table tbl, Partition p) + throws HiveException { + String subdir = null; + try { + subdir = Warehouse.makePartName(tbl.getPartCols(), p.getValues()); + } catch (MetaException e) { + throw new HiveException("Unable to get partition's directory", e); + } + URI tableDir = tbl.getDataLocation(); + if(tableDir == null) { + throw new HiveException("Table has no location set"); + } + + String standardLocation = (new Path(tableDir.toString(), subdir)).toString(); + if(ArchiveUtils.isArchived(p)) { + return !getOriginalLocation(p).equals(standardLocation); + } else { + return !p.getLocation().equals(standardLocation); + } + } + private int archive(Hive db, AlterTableSimpleDesc simpleDesc, DriverContext driverContext) throws HiveException { String dbName = simpleDesc.getDbName(); @@ -1098,35 +1151,43 @@ Table tbl = db.getTable(dbName, tblName); validateAlterTableType(tbl, AlterTableDesc.AlterTableTypes.ARCHIVE); - Map partSpec = simpleDesc.getPartSpec(); - Partition p = db.getPartition(tbl, partSpec, false); - if (tbl.getTableType() != TableType.MANAGED_TABLE) { throw new HiveException("ARCHIVE can only be performed on managed tables"); } - if (p == null) { - throw new HiveException("Specified partition does not exist"); - } + LinkedHashMap partSpec = simpleDesc.getPartSpec(); + PartSpecInfo partSpecInfo = PartSpecInfo.create(tbl, partSpec); + List partitions = db.getPartitions(tbl, partSpec); - if (Utilities.isArchived(p)) { - // If there were a failure right after the metadata was updated in an - // archiving operation, it's possible that the original, unarchived files - // weren't deleted. - Path originalDir = new Path(getOriginalLocation(p)); - Path leftOverIntermediateOriginal = new Path(originalDir.getParent(), - originalDir.getName() + INTERMEDIATE_ORIGINAL_DIR_SUFFIX); + Path originalDir = null; - if (pathExists(leftOverIntermediateOriginal)) { - console.printInfo("Deleting " + leftOverIntermediateOriginal + - " left over from a previous archiving operation"); - deleteDir(leftOverIntermediateOriginal); + // when we have partial partitions specification we must assume partitions + // lie in standard place - if they were in custom locations putting + // them into one archive would involve mass amount of copying + // in full partition specification case we allow custom locations + // to keep backward compatibility + if (partitions.isEmpty()) { + throw new HiveException("No partition matches the specification"); + } else if(partSpecInfo.values.size() != tbl.getPartCols().size()) { + // for partial specifications we need partitions to follow the scheme + for(Partition p: partitions){ + if(partitionInCustomLocation(tbl, p)) { + String message = String.format("ARCHIVE cannot run for partition " + + "groups with custom locations like %s", p.getLocation()); + throw new HiveException(message); + } } - - throw new HiveException("Specified partition is already archived"); + originalDir = partSpecInfo.createPath(tbl); + } else { + Partition p = partitions.get(0); + // partition can be archived if during recovery + if(ArchiveUtils.isArchived(p)) { + originalDir = new Path(getOriginalLocation(p)); + } else { + originalDir = p.getPartitionPath(); + } } - Path originalDir = p.getPartitionPath(); Path intermediateArchivedDir = new Path(originalDir.getParent(), originalDir.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX); Path intermediateOriginalDir = new Path(originalDir.getParent(), @@ -1139,6 +1200,36 @@ throw new HiveException(e); } + URI originalUri = ArchiveUtils.addSlash(originalDir.toUri()); + URI archiveUri = (new Path(originalDir, archiveName)).toUri(); + ArchiveUtils.HarPathHelper harHelper = new ArchiveUtils.HarPathHelper( + conf, archiveUri, originalUri); + + // we checked if partitions matching specification are marked as archived + // in the metadata; if they are and their levels are the same as we would + // set it later it means previous run failed and we have to do the recovery; + // if they are different, we throw an error + boolean anyUnarchived = false; + for(Partition p: partitions) { + if(ArchiveUtils.isArchived(p)) { + if(ArchiveUtils.getArchivingLevel(p) != partSpecInfo.values.size()) { + String name = ArchiveUtils.getPartialName(p, ArchiveUtils.getArchivingLevel(p)); + String m = String.format("Conflict with existing archive %s", name); + throw new HiveException(m); + } + } else { + anyUnarchived = true; + } + } + + boolean recovery = false; + if(pathExists(intermediateArchivedDir) || pathExists(intermediateOriginalDir)) { + recovery = true; + console.printInfo("Starting recovery after failed ARCHIVE"); + } else if(!anyUnarchived) { + throw new HiveException("Partition(s) already archived"); + } + // The following steps seem roundabout, but they are meant to aid in // recovery if a failure occurs and to keep a consistent state in the FS @@ -1166,10 +1257,12 @@ // First create the archive in a tmp dir so that if the job fails, the // bad files don't pollute the filesystem - Path tmpDir = new Path(driverContext.getCtx().getExternalTmpFileURI(originalDir.toUri()), "partlevel"); + Path tmpPath = new Path(driverContext.getCtx() + .getExternalTmpFileURI(originalDir.toUri()), "partlevel"); - console.printInfo("Creating " + archiveName + " for " + originalDir.toString()); - console.printInfo("in " + tmpDir); + console.printInfo("Creating " + archiveName + + " for " + originalDir.toString()); + console.printInfo("in " + tmpPath); console.printInfo("Please wait... (this may take a while)"); // Create the Hadoop archive @@ -1178,10 +1271,10 @@ try { int maxJobNameLen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH); String jobname = String.format("Archiving %s@%s", - tbl.getTableName(), p.getName()); + tbl.getTableName(), partSpecInfo.getName()); jobname = Utilities.abbreviate(jobname, maxJobNameLen - 6); conf.setVar(HiveConf.ConfVars.HADOOPJOBNAME, jobname); - ret = shim.createHadoopArchive(conf, originalDir, tmpDir, archiveName); + ret = shim.createHadoopArchive(conf, originalDir, tmpPath, archiveName); } catch (Exception e) { throw new HiveException(e); } @@ -1191,11 +1284,11 @@ // Move from the tmp dir to an intermediate directory, in the same level as // the partition directory. e.g. .../hr=12-intermediate-archived try { - console.printInfo("Moving " + tmpDir + " to " + intermediateArchivedDir); + console.printInfo("Moving " + tmpPath + " to " + intermediateArchivedDir); if (pathExists(intermediateArchivedDir)) { throw new HiveException("The intermediate archive directory already exists."); } - fs.rename(tmpDir, intermediateArchivedDir); + fs.rename(tmpPath, intermediateArchivedDir); } catch (IOException e) { throw new HiveException("Error while moving tmp directory"); } @@ -1239,39 +1332,31 @@ // Record this change in the metastore try { - boolean parentSettable = - conf.getBoolVar(HiveConf.ConfVars.HIVEHARPARENTDIRSETTABLE); - - // dirInArchive is the directory within the archive that has all the files - // for this partition. With older versions of Hadoop, archiving a - // a directory would produce the same directory structure - // in the archive. So if you created myArchive.har of /tmp/myDir, the - // files in /tmp/myDir would be located under myArchive.har/tmp/myDir/* - // In this case, dirInArchive should be tmp/myDir - - // With newer versions of Hadoop, the parent directory could be specified. - // Assuming the parent directory was set to /tmp/myDir when creating the - // archive, the files can be found under myArchive.har/* - // In this case, dirInArchive should be empty - - String dirInArchive = ""; - if (!parentSettable) { - dirInArchive = originalDir.toUri().getPath(); - if(dirInArchive.length() > 1 && dirInArchive.charAt(0)=='/') { - dirInArchive = dirInArchive.substring(1); - } + for(Partition p: partitions) { + URI originalPartitionUri = ArchiveUtils.addSlash(p.getPartitionPath().toUri()); + URI harPartitionDir = harHelper.getHarUri(originalPartitionUri); + Path harPath = new Path(harPartitionDir.getScheme(), + harPartitionDir.getAuthority(), + harPartitionDir.getPath()); // make in Path to ensure no slash at the end + setArchived(p, harPath, partSpecInfo.values.size()); + db.alterPartition(tblName, p); } - setArchived(p, originalDir, dirInArchive, archiveName); - db.alterPartition(tblName, p); } catch (Exception e) { throw new HiveException("Unable to change the partition info for HAR", e); } // If a failure occurs here, the directory containing the original files // will not be deleted. The user will run ARCHIVE again to clear this up - deleteDir(intermediateOriginalDir); + if(pathExists(intermediateOriginalDir)) { + deleteDir(intermediateOriginalDir); + } + setContainsArchived(db, tbl); + if(recovery) { + console.printInfo("Recovery after ARCHIVE succeeded"); + } + return 0; } @@ -1285,95 +1370,140 @@ // Means user specified a table, not a partition if (simpleDesc.getPartSpec() == null) { - throw new HiveException("ARCHIVE is for partitions only"); + throw new HiveException("UNARCHIVE is for partitions only"); } - Map partSpec = simpleDesc.getPartSpec(); - Partition p = db.getPartition(tbl, partSpec, false); - if (tbl.getTableType() != TableType.MANAGED_TABLE) { throw new HiveException("UNARCHIVE can only be performed on managed tables"); } - if (p == null) { - throw new HiveException("Specified partition does not exist"); + LinkedHashMap partSpec = simpleDesc.getPartSpec(); + PartSpecInfo partSpecInfo = PartSpecInfo.create(tbl, partSpec); + List partitions = db.getPartitions(tbl, partSpec); + + Path originalDir = null; + + // when we have partial partitions specification we must assume partitions + // lie in standard place - if they were in custom locations putting + // them into one archive would involve mass amount of copying + // in full partition specification case we allow custom locations + // to keep backward compatibility + if (partitions.isEmpty()) { + throw new HiveException("No partition matches the specification"); + } else if(partSpecInfo.values.size() != tbl.getPartCols().size()) { + // for partial specifications we need partitions to follow the scheme + for(Partition p: partitions){ + if(partitionInCustomLocation(tbl, p)) { + String message = String.format("UNARCHIVE cannot run for partition " + + "groups with custom locations like %s", p.getLocation()); + throw new HiveException(message); + } + } + originalDir = partSpecInfo.createPath(tbl); + } else { + Partition p = partitions.get(0); + if(ArchiveUtils.isArchived(p)) { + originalDir = new Path(getOriginalLocation(p)); + } else { + originalDir = p.getPartitionPath(); + } } - if (!Utilities.isArchived(p)) { - Path location = new Path(p.getLocation()); - Path leftOverArchiveDir = new Path(location.getParent(), - location.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX); + URI originalUri = ArchiveUtils.addSlash(originalDir.toUri()); + Path intermediateArchivedDir = new Path(originalDir.getParent(), + originalDir.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX); + Path intermediateExtractedDir = new Path(originalDir.getParent(), + originalDir.getName() + INTERMEDIATE_EXTRACTED_DIR_SUFFIX); + String archiveName = "data.har"; + FileSystem fs = null; + try { + fs = originalDir.getFileSystem(conf); + } catch (IOException e) { + throw new HiveException(e); + } - if (pathExists(leftOverArchiveDir)) { - console.printInfo("Deleting " + leftOverArchiveDir + " left over " + - "from a previous unarchiving operation"); - deleteDir(leftOverArchiveDir); + // if nothing is archived we throw an error saying partition not archived + // if only some partitions are archived we check if these are archived + // in right place and if so, it means we are during the recovery + boolean anyArchived = false; + for(Partition p: partitions) { + if(ArchiveUtils.isArchived(p)) { + anyArchived = true; } + } - throw new HiveException("Specified partition is not archived"); + // assume the archive is in the original dir, check if it exists + Path archivePath = new Path(originalDir, archiveName); + URI archiveUri = archivePath.toUri(); + ArchiveUtils.HarPathHelper harHelper = new ArchiveUtils.HarPathHelper(conf, + archiveUri, originalUri); + URI sourceUri = harHelper.getHarUri(originalUri); + Path sourceDir = new Path(sourceUri.getScheme(), sourceUri.getAuthority(), sourceUri.getPath()); + + // check if all partitions' metadata tells they lie in the archive + for(Partition p: partitions) { + if(ArchiveUtils.isArchived(p)){ + if(ArchiveUtils.getArchivingLevel(p) != partSpecInfo.values.size()) { + String name = ArchiveUtils.getPartialName(p, ArchiveUtils.getArchivingLevel(p)); + String m = String.format("Archiving was at different level, use %s", + name); + throw new HiveException(m); + } + } } - Path originalLocation = new Path(getOriginalLocation(p)); - Path sourceDir = new Path(p.getLocation()); - Path intermediateArchiveDir = new Path(originalLocation.getParent(), - originalLocation.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX); - Path intermediateExtractedDir = new Path(originalLocation.getParent(), - originalLocation.getName() + INTERMEDIATE_EXTRACTED_DIR_SUFFIX); + boolean recovery = false; + if(pathExists(intermediateArchivedDir) || pathExists(intermediateExtractedDir)) { + recovery = true; + console.printInfo("Starting recovery after failed UNARCHIVE"); + } else if(!anyArchived) { + throw new HiveException("Partition(s) not archived"); + } - Path tmpDir = new Path(driverContext + if(!pathExists(intermediateArchivedDir) && !pathExists(archivePath)) { + throw new HiveException("Haven't found any archive where it should be"); + } + + Path tmpPath = new Path(driverContext .getCtx() - .getExternalTmpFileURI(originalLocation.toUri())); + .getExternalTmpFileURI(originalDir.toUri())); - FileSystem fs = null; try { - fs = tmpDir.getFileSystem(conf); - // Verify that there are no files in the tmp dir, because if there are, it - // would be copied to the partition - FileStatus [] filesInTmpDir = fs.listStatus(tmpDir); - if (filesInTmpDir != null && filesInTmpDir.length != 0) { - for (FileStatus file : filesInTmpDir) { - console.printInfo(file.getPath().toString()); - } - throw new HiveException("Temporary directory " + tmpDir + " is not empty"); - } - + fs = tmpPath.getFileSystem(conf); } catch (IOException e) { throw new HiveException(e); } // Some sanity checks - if (originalLocation == null) { + if (originalDir == null) { throw new HiveException("Missing archive data in the partition"); } - if (!"har".equals(sourceDir.toUri().getScheme())) { - throw new HiveException("Location should refer to a HAR"); - } // Clarification of terms: - // - The originalLocation directory represents the original directory of the - // partition's files. They now contain an archived version of those files + // - The originalDir directory represents the original directory of the + // partitions' files. They now contain an archived version of those files // eg. hdfs:/warehouse/myTable/ds=1/ // - The source directory is the directory containing all the files that - // should be in the partition. e.g. har:/warehouse/myTable/ds=1/myTable.har/ + // should be in the partitions. e.g. har:/warehouse/myTable/ds=1/myTable.har/ // Note the har:/ scheme // Steps: // 1. Extract the archive in a temporary folder // 2. Move the archive dir to an intermediate dir that is in at the same // dir as originalLocation. Call the new dir intermediate-extracted. - // 3. Rename the original partition dir to an intermediate dir. Call the + // 3. Rename the original partitions dir to an intermediate dir. Call the // renamed dir intermediate-archive - // 4. Rename intermediate-extracted to the original partition dir + // 4. Rename intermediate-extracted to the original partitions dir // 5. Change the metadata - // 6. Delete the archived partition files in intermediate-archive + // 6. Delete the archived partitions files in intermediate-archive if (!pathExists(intermediateExtractedDir) && - !pathExists(intermediateArchiveDir)) { + !pathExists(intermediateArchivedDir)) { try { // Copy the files out of the archive into the temporary directory String copySource = sourceDir.toString(); - String copyDest = tmpDir.toString(); + String copyDest = tmpPath.toString(); List args = new ArrayList(); args.add("-cp"); args.add(copySource); @@ -1391,12 +1521,12 @@ throw new HiveException("Error while copying files from archive"); } - console.printInfo("Moving " + tmpDir + " to " + intermediateExtractedDir); + console.printInfo("Moving " + tmpPath + " to " + intermediateExtractedDir); if (fs.exists(intermediateExtractedDir)) { throw new HiveException("Invalid state: the intermediate extracted " + "directory already exists."); } - fs.rename(tmpDir, intermediateExtractedDir); + fs.rename(tmpPath, intermediateExtractedDir); } catch (Exception e) { throw new HiveException(e); } @@ -1405,15 +1535,15 @@ // At this point, we know that the extracted files are in the intermediate // extracted dir, or in the the original directory. - if (!pathExists(intermediateArchiveDir)) { + if (!pathExists(intermediateArchivedDir)) { try { - console.printInfo("Moving " + originalLocation + " to " + intermediateArchiveDir); - fs.rename(originalLocation, intermediateArchiveDir); + console.printInfo("Moving " + originalDir + " to " + intermediateArchivedDir); + fs.rename(originalDir, intermediateArchivedDir); } catch (IOException e) { throw new HiveException(e); } } else { - console.printInfo(intermediateArchiveDir + " already exists. " + + console.printInfo(intermediateArchivedDir + " already exists. " + "Assuming it contains the archived version of the partition"); } @@ -1423,28 +1553,39 @@ // If the original location exists here, then it must be the extracted files // because in the previous step, we moved the previous original location // (containing the archived version of the files) to intermediateArchiveDir - if (!pathExists(originalLocation)) { + if (!pathExists(originalDir)) { try { - console.printInfo("Moving " + intermediateExtractedDir + " to " + originalLocation); - fs.rename(intermediateExtractedDir, originalLocation); + console.printInfo("Moving " + intermediateExtractedDir + " to " + originalDir); + fs.rename(intermediateExtractedDir, originalDir); } catch (IOException e) { throw new HiveException(e); } } else { - console.printInfo(originalLocation + " already exists. " + + console.printInfo(originalDir + " already exists. " + "Assuming it contains the extracted files in the partition"); } - setUnArchived(p); - try { - db.alterPartition(tblName, p); - } catch (InvalidOperationException e) { - throw new HiveException(e); + for(Partition p: partitions) { + setUnArchived(p); + try { + db.alterPartition(tblName, p); + } catch (InvalidOperationException e) { + throw new HiveException(e); + } } + // If a failure happens here, the intermediate archive files won't be // deleted. The user will need to call unarchive again to clear those up. - deleteDir(intermediateArchiveDir); + if(pathExists(intermediateArchivedDir)) { + deleteDir(intermediateArchivedDir); + } + updateContainsArchived(db, tbl); + + if(recovery) { + console.printInfo("Recovery after UNARCHIVE succeeded"); + } + return 0; } @@ -2840,9 +2981,9 @@ } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ALTERLOCATION) { String newLocation = alterTbl.getNewLocation(); try { - URI locURI = new URI(newLocation); - if (!locURI.isAbsolute() || locURI.getScheme() == null - || locURI.getScheme().trim().equals("")) { + URI locUri = new URI(newLocation); + if (!locUri.isAbsolute() || locUri.getScheme() == null + || locUri.getScheme().trim().equals("")) { throw new HiveException( newLocation + " is not absolute or has no scheme information. " @@ -2851,7 +2992,7 @@ if (part != null) { part.setLocation(newLocation); } else { - tbl.setDataLocation(locURI); + tbl.setDataLocation(locUri); } } catch (URISyntaxException e) { throw new HiveException(e); @@ -2983,14 +3124,48 @@ List partsToDelete = new ArrayList(); for (Map partSpec : dropTbl.getPartSpecs()) { List partitions = db.getPartitions(tbl, partSpec); + // calculated archivingLevel is to be minimal of partitions' archiving + // levels if there are any and -1 otherwise + int archivingLevel = -1; for (Partition p : partitions) { + partsToDelete.add(p); if (!p.canDrop()) { throw new HiveException("Table " + tbl.getTableName() + " Partition " + p.getName() + " is protected from being dropped"); + } else if(ArchiveUtils.isArchived(p)) { + if(archivingLevel == -1 || + ArchiveUtils.getArchivingLevel(p) < archivingLevel) { + archivingLevel = ArchiveUtils.getArchivingLevel(p); + } } - partsToDelete.add(p); } + if(archivingLevel != -1) { + // dropped archived partition + int tail = 0; // how many unbound partition keys are at the end of spec + for(FieldSchema fs: tbl.getPartCols()) { + if(partSpec.get(fs.getName()) == null) { + tail += 1; + } else { + tail = 0; + } + } + + // let's say table is partitioned by ds, hr, min and user + // requested drop WHERE ds='1' AND hr='2' + // then tail = 1 and tbl.getPartCols().size() - tail = 2 + // this difference is always number of last set column + // if it's bigger that minimal archiving level it means we are + // choosing some partitions of archive with smaller level + // which is kind of wrong and impossible with current implementation + if(tbl.getPartCols().size() - tail > archivingLevel) { + throw new HiveException("Cannot drop a subset of partitions in an archive"); + } else if(tbl.getPartCols().size() > archivingLevel) { + throw new HiveException("Dropping archives containing many partitions" + + "not yet implemented"); + } + updateContainsArchived(db, tbl); + } } // drop all existing partitions from the list Index: ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java (revision 1153271) +++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java (working copy) @@ -603,7 +603,7 @@ } if (partn == null) { - return new HiveLockObject(new DummyPartition(tab, path), data); + return new HiveLockObject(new DummyPartition(tab, path, partSpec), data); } return new HiveLockObject(partn, data); Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1153271) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy) @@ -27,9 +27,9 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; +import java.util.Map.Entry; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -47,6 +47,7 @@ import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; +import org.apache.hadoop.hive.ql.exec.ArchiveUtils; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.ConditionalTask; import org.apache.hadoop.hive.ql.exec.ExecDriver; @@ -92,7 +93,6 @@ import org.apache.hadoop.hive.ql.optimizer.GenMRFileSink1; import org.apache.hadoop.hive.ql.optimizer.GenMROperator; import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext; -import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink1; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink2; import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink3; @@ -102,10 +102,15 @@ import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; import org.apache.hadoop.hive.ql.optimizer.MapJoinFactory; import org.apache.hadoop.hive.ql.optimizer.Optimizer; +import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext; import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalOptimizer; import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner; import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeCreateCommonVars; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.RowFormatParams; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.StorageFormat; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec.SpecType; import org.apache.hadoop.hive.ql.plan.AggregationDesc; import org.apache.hadoop.hive.ql.plan.CreateTableDesc; @@ -122,7 +127,6 @@ import org.apache.hadoop.hive.ql.plan.FetchWork; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; -import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; import org.apache.hadoop.hive.ql.plan.ForwardDesc; import org.apache.hadoop.hive.ql.plan.GroupByDesc; import org.apache.hadoop.hive.ql.plan.HiveOperation; @@ -145,12 +149,13 @@ import org.apache.hadoop.hive.ql.plan.TableScanDesc; import org.apache.hadoop.hive.ql.plan.UDTFDesc; import org.apache.hadoop.hive.ql.plan.UnionDesc; +import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.ResourceType; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash; import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; import org.apache.hadoop.hive.serde.Constants; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe; @@ -158,9 +163,9 @@ import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.StructField; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -3773,7 +3778,9 @@ ppath = ppath.substring(0, ppath.length()-1); DummyPartition p = new DummyPartition(dest_tab, - dest_tab.getDbName() + "@" + dest_tab.getTableName() + "@" + ppath); + dest_tab.getDbName() + "@" + + dest_tab.getTableName() + "@" + ppath, + partSpec); outputs.add(new WriteEntity(p, false)); } catch (HiveException e) { throw new SemanticException(e.getMessage()); @@ -3795,10 +3802,6 @@ dest_path = new Path(tabPath.toUri().getScheme(), tabPath.toUri() .getAuthority(), partPath.toUri().getPath()); - if ("har".equalsIgnoreCase(dest_path.toUri().getScheme())) { - throw new SemanticException(ErrorMsg.OVERWRITE_ARCHIVED_PART - .getMsg(qb.getParseInfo().getDestForClause(dest))); - } queryTmpdir = ctx.getExternalTmpFileURI(dest_path.toUri()); table_desc = Utilities.getTableDesc(dest_tab); @@ -7442,6 +7445,7 @@ @Override public void validate() throws SemanticException { + LOG.debug("validation start"); // Validate inputs and outputs have right protectmode to execute the query for (ReadEntity readEntity: getInputs()) { ReadEntity.Type type = readEntity.getType(); @@ -7474,8 +7478,30 @@ for (WriteEntity writeEntity: getOutputs()) { WriteEntity.Type type = writeEntity.getType(); + + if(type == WriteEntity.Type.PARTITION || type == WriteEntity.Type.DUMMYPARTITION) { + String conflictingArchive; + try { + Partition usedp = writeEntity.getPartition(); + Table tbl = usedp.getTable(); + + LOG.debug("validated " + usedp.getName()); + LOG.debug(usedp.getTable()); + conflictingArchive = ArchiveUtils + .conflictingArchiveNameOrNull(db, tbl, usedp.getSpec()); + } catch (HiveException e) { + throw new SemanticException(e); + } + if(conflictingArchive != null) { + String message = String.format("Insert conflict with existing archive: %s", + conflictingArchive); + throw new SemanticException(message); + } + } + if (type != WriteEntity.Type.TABLE && type != WriteEntity.Type.PARTITION) { + LOG.debug("not validating writeEntity, because entity is neither table nor partition"); continue; } @@ -7507,8 +7533,9 @@ " Table " + tbl.getTableName() + " Partition " + p.getName())); } - } - else { + + } else { + LOG.debug("Not a partition."); tbl = writeEntity.getTable(); } Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (revision 1153271) +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (working copy) @@ -51,6 +51,7 @@ import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.exec.ArchiveUtils; import org.apache.hadoop.hive.ql.exec.FetchTask; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; @@ -67,6 +68,10 @@ import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeCreateCommonVars; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.RowFormatParams; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.StorageFormat; +import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec; import org.apache.hadoop.hive.ql.plan.AddPartitionDesc; import org.apache.hadoop.hive.ql.plan.AlterDatabaseDesc; import org.apache.hadoop.hive.ql.plan.AlterIndexDesc; @@ -1158,7 +1163,7 @@ } bucketCols = part.getBucketCols(); inputFormatClass = part.getInputFormatClass(); - isArchived = Utilities.isArchived(part); + isArchived = ArchiveUtils.isArchived(part); tblPartLoc = part.getDataLocation().toString(); } } else { @@ -1732,7 +1737,7 @@ // configured not to fail silently boolean throwException = !ifExists && !HiveConf.getBoolVar(conf, ConfVars.DROPIGNORESNONEXISTENT); - addTablePartsOutputs(tblName, partSpecs, throwException, ast); + addTablePartsOutputs(tblName, partSpecs, throwException, false, ast); } rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), @@ -1927,15 +1932,16 @@ // partition name to value List> partSpecs = getPartitionSpecs(ast); + Table tab = null; try { - Table tab = db.getTable(db.getCurrentDatabase(), tblName, false); + tab = db.getTable(db.getCurrentDatabase(), tblName, false); if (tab != null) { inputs.add(new ReadEntity(tab)); } } catch (HiveException e) { throw new SemanticException(ErrorMsg.INVALID_TABLE.getMsg(tblName)); } - addTablePartsOutputs(tblName, partSpecs); + addTablePartsOutputs(tblName, partSpecs, true); if (partSpecs.size() > 1 ) { throw new SemanticException(isUnArchive ? @@ -1946,13 +1952,18 @@ throw new SemanticException(ErrorMsg.ARCHIVE_ON_TABLE.getMsg()); } - Map partSpec = partSpecs.get(0); - AlterTableSimpleDesc archiveDesc = new AlterTableSimpleDesc( - db.getCurrentDatabase(), tblName, partSpec, - (isUnArchive ? AlterTableTypes.UNARCHIVE : AlterTableTypes.ARCHIVE)); - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), - archiveDesc), conf)); - + // it's always LinkedHashMap, it is only casted to Map + LinkedHashMap partSpec = (LinkedHashMap) partSpecs.get(0); + try { + tab.isValidPrefixSpec(partSpec); + } catch(HiveException e) { + throw new SemanticException(e.getMessage(), e); + } + AlterTableSimpleDesc archiveDesc = new AlterTableSimpleDesc( + db.getCurrentDatabase(), tblName, partSpec, + (isUnArchive ? AlterTableTypes.UNARCHIVE : AlterTableTypes.ARCHIVE)); + rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), + archiveDesc), conf)); } /** @@ -2037,16 +2048,25 @@ */ private void addTablePartsOutputs(String tblName, List> partSpecs) throws SemanticException { - addTablePartsOutputs(tblName, partSpecs, false, null); + addTablePartsOutputs(tblName, partSpecs, false, false, null); } /** * Add the table partitions to be modified in the output, so that it is available for the + * pre-execution hook. If the partition does not exist, no error is thrown. + */ + private void addTablePartsOutputs(String tblName, List> partSpecs, boolean allowMany) + throws SemanticException { + addTablePartsOutputs(tblName, partSpecs, false, allowMany, null); + } + + /** + * Add the table partitions to be modified in the output, so that it is available for the * pre-execution hook. If the partition does not exist, throw an error if * throwIfNonExistent is true, otherwise ignore it. */ private void addTablePartsOutputs(String tblName, List> partSpecs, - boolean throwIfNonExistent, ASTNode ast) + boolean throwIfNonExistent, boolean allowMany, ASTNode ast) throws SemanticException { Table tab; try { @@ -2058,19 +2078,33 @@ Iterator> i; int index; for (i = partSpecs.iterator(), index = 1; i.hasNext(); ++index) { - Map partSpec = i.next(); - try { - Partition part = db.getPartition(tab, partSpec, false); - if (part == null) { - continue; + Map partSpec = i.next(); + List parts = null; + if(allowMany) { + try { + parts = db.getPartitions(tab, partSpec); + } catch (HiveException e) { + LOG.error("Got HiveException during obtaining list of partitions"); } - outputs.add(new WriteEntity(part)); - } catch (HiveException e) { - // Ignore the error if the partition does not exist - if (throwIfNonExistent) { - throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(ast.getChild(index))); - } + } else { + parts = new ArrayList(); + try { + Partition p = db.getPartition(tab, partSpec, false); + if(p != null) { + parts.add(p); + } + } catch(HiveException e) { + LOG.debug("Wrong specification"); + } } + if (parts.isEmpty()) { + if(throwIfNonExistent) { + throw new SemanticException(ErrorMsg.INVALID_PARTITION.getMsg(ast.getChild(index))); + } + } + for(Partition p: parts) { + outputs.add(new WriteEntity(p)); + } } } Index: ql/src/java/org/apache/hadoop/hive/ql/Driver.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java (revision 1153271) +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java (working copy) @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -680,14 +681,19 @@ String partialName = ""; String[] partns = name.split("/"); int len = p instanceof DummyPartition ? partns.length : partns.length - 1; + Map partialSpec = new LinkedHashMap(); for (int idx = 0; idx < len; idx++) { String partn = partns[idx]; partialName += partn; + String[] nameValue = partn.split("="); + assert(nameValue.length == 2); + partialSpec.put(nameValue[0], nameValue[1]); try { locks.add(new HiveLockObj( new HiveLockObject(new DummyPartition(p.getTable(), p.getTable().getDbName() + "/" + p.getTable().getTableName() - + "/" + partialName), lockData), mode)); + + "/" + partialName, + partialSpec), lockData), mode)); partialName += "/"; } catch (HiveException e) { throw new SemanticException(e.getMessage());