Index: metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
===================================================================
--- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (revision 1153271)
+++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (working copy)
@@ -27,8 +27,8 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Map.Entry;
-import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -52,9 +52,9 @@
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
import org.apache.hadoop.util.StringUtils;
Index: metastore/src/gen/thrift/gen-py/hive_metastore/constants.py
===================================================================
--- metastore/src/gen/thrift/gen-py/hive_metastore/constants.py (revision 1153271)
+++ metastore/src/gen/thrift/gen-py/hive_metastore/constants.py (working copy)
@@ -13,6 +13,8 @@
HIVE_FILTER_FIELD_LAST_ACCESS = "hive_filter_field_last_access__"
IS_ARCHIVED = "is_archived"
ORIGINAL_LOCATION = "original_location"
+ARCHIVING_LEVEL = "archiving_level"
+CONTAINS_ARCHIVED_PARTITIONS = "contains_archived_partitions"
META_TABLE_COLUMNS = "columns"
META_TABLE_COLUMN_TYPES = "columns.types"
BUCKET_FIELD_NAME = "bucket_field_name"
Index: metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h
===================================================================
--- metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h (revision 1153271)
+++ metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.h (working copy)
@@ -20,6 +20,8 @@
std::string HIVE_FILTER_FIELD_LAST_ACCESS;
std::string IS_ARCHIVED;
std::string ORIGINAL_LOCATION;
+ std::string ARCHIVING_LEVEL;
+ std::string CONTAINS_ARCHIVED_PARTITIONS;
std::string META_TABLE_COLUMNS;
std::string META_TABLE_COLUMN_TYPES;
std::string BUCKET_FIELD_NAME;
Index: metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp
===================================================================
--- metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp (revision 1153271)
+++ metastore/src/gen/thrift/gen-cpp/hive_metastore_constants.cpp (working copy)
@@ -22,6 +22,10 @@
ORIGINAL_LOCATION = "original_location";
+ ARCHIVING_LEVEL = "archiving_level";
+
+ CONTAINS_ARCHIVED_PARTITIONS = "contains_archived_partitions";
+
META_TABLE_COLUMNS = "columns";
META_TABLE_COLUMN_TYPES = "columns.types";
Index: metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb
===================================================================
--- metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb (revision 1153271)
+++ metastore/src/gen/thrift/gen-rb/hive_metastore_constants.rb (working copy)
@@ -18,6 +18,10 @@
ORIGINAL_LOCATION = %q"original_location"
+ARCHIVING_LEVEL = %q"archiving_level"
+
+CONTAINS_ARCHIVED_PARTITIONS = %q"contains_archived_partitions"
+
META_TABLE_COLUMNS = %q"columns"
META_TABLE_COLUMN_TYPES = %q"columns.types"
Index: metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Constants.java
===================================================================
--- metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Constants.java (revision 1153271)
+++ metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Constants.java (working copy)
@@ -34,6 +34,10 @@
public static final String ORIGINAL_LOCATION = "original_location";
+ public static final String ARCHIVING_LEVEL = "archiving_level";
+
+ public static final String CONTAINS_ARCHIVED_PARTITIONS = "contains_archived_partitions";
+
public static final String META_TABLE_COLUMNS = "columns";
public static final String META_TABLE_COLUMN_TYPES = "columns.types";
Index: metastore/src/gen/thrift/gen-php/hive_metastore/hive_metastore_constants.php
===================================================================
--- metastore/src/gen/thrift/gen-php/hive_metastore/hive_metastore_constants.php (revision 1153271)
+++ metastore/src/gen/thrift/gen-php/hive_metastore/hive_metastore_constants.php (working copy)
@@ -20,6 +20,10 @@
$GLOBALS['hive_metastore_CONSTANTS']['ORIGINAL_LOCATION'] = "original_location";
+$GLOBALS['hive_metastore_CONSTANTS']['ARCHIVING_LEVEL'] = "archiving_level";
+
+$GLOBALS['hive_metastore_CONSTANTS']['CONTAINS_ARCHIVED_PARTITIONS'] = "contains_archived_partitions";
+
$GLOBALS['hive_metastore_CONSTANTS']['META_TABLE_COLUMNS'] = "columns";
$GLOBALS['hive_metastore_CONSTANTS']['META_TABLE_COLUMN_TYPES'] = "columns.types";
Index: metastore/if/hive_metastore.thrift
===================================================================
--- metastore/if/hive_metastore.thrift (revision 1153271)
+++ metastore/if/hive_metastore.thrift (working copy)
@@ -454,6 +454,11 @@
// this directory will contain the archive. When the partition
// is dropped, this directory will be deleted
const string ORIGINAL_LOCATION = "original_location",
+// Level of archiving, when table is partitioned by ds, hr then if
+// partition was archived at ds level it is 1 at hr level it is 2
+const string ARCHIVING_LEVEL = "archiving_level";
+// Counts archived partitions in table
+const string CONTAINS_ARCHIVED_PARTITIONS = "contains_archived_partitions";
// these should be needed only for backward compatibility with filestore
const string META_TABLE_COLUMNS = "columns",
@@ -469,6 +474,3 @@
const string FILE_INPUT_FORMAT = "file.inputformat",
const string FILE_OUTPUT_FORMAT = "file.outputformat",
const string META_TABLE_STORAGE = "storage_handler",
-
-
-
Index: data/conf/hive-site.xml
===================================================================
--- data/conf/hive-site.xml (revision 1153271)
+++ data/conf/hive-site.xml (working copy)
@@ -139,7 +139,7 @@
hive.exec.post.hooks
- org.apache.hadoop.hive.ql.hooks.PostExecutePrinter
+ org.apache.hadoop.hive.ql.hooks.UpdateArchivedCounter, org.apache.hadoop.hive.ql.hooks.PostExecutePrinter
Post Execute Hook for Tests
Index: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
===================================================================
--- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (revision 1153271)
+++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (working copy)
@@ -425,6 +425,7 @@
// For har files
HIVEARCHIVEENABLED("hive.archive.enabled", false),
HIVEHARPARENTDIRSETTABLE("hive.archive.har.parentdir.settable", false),
+ HIVEARCHIVECONTAINSARCHIVEDUPDATE("hive.archive.contains.archived.update", true),
HIVEOUTERJOINSUPPORTSFILTERS("hive.outerjoin.supports.filters", true),
// Serde for FetchTask
Index: ql/src/test/results/clientnegative/archive1.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive1.q.out (revision 1153271)
+++ ql/src/test/results/clientnegative/archive1.q.out (working copy)
@@ -35,5 +35,5 @@
PREHOOK: type: ALTERTABLE_ARCHIVE
PREHOOK: Input: default@srcpart_archived
PREHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12
-FAILED: Error in metadata: Specified partition is already archived
+FAILED: Error in metadata: Partition(s) already archived
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask
Index: ql/src/test/results/clientnegative/archive_multi1.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive_multi1.q.out (revision 0)
+++ ql/src/test/results/clientnegative/archive_multi1.q.out (revision 0)
@@ -0,0 +1,58 @@
+PREHOOK: query: -- Tests trying to archive a partition twice.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests trying to archive a partition twice.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrcpart
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+POSTHOOK: type: ALTERTABLE_ARCHIVE
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+FAILED: Error in metadata: Partition(s) already archived
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask
Index: ql/src/test/results/clientnegative/archive_insert3.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive_insert3.q.out (revision 0)
+++ ql/src/test/results/clientnegative/archive_insert3.q.out (revision 0)
@@ -0,0 +1,34 @@
+PREHOOK: query: -- Tests trying to create partition inside of archived directory.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests trying to create partition inside of archived directory.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrcpart
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+POSTHOOK: type: ALTERTABLE_ARCHIVE
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: Error in semantic analysis: Insert conflict with existing archive: ds=2008-04-08
Index: ql/src/test/results/clientnegative/archive_multi4.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive_multi4.q.out (revision 0)
+++ ql/src/test/results/clientnegative/archive_multi4.q.out (revision 0)
@@ -0,0 +1,57 @@
+PREHOOK: query: -- Tests trying to archive inner partition contained in archived partition group.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests trying to archive inner partition contained in archived partition group.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrcpart
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+POSTHOOK: type: ALTERTABLE_ARCHIVE
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+FAILED: Error in metadata: Conflict with existing archive ds=2008-04-08
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask
Index: ql/src/test/results/clientnegative/archive_partspec2.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive_partspec2.q.out (revision 0)
+++ ql/src/test/results/clientnegative/archive_partspec2.q.out (revision 0)
@@ -0,0 +1,24 @@
+PREHOOK: query: -- Tests trying to archive a partition twice.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE srcpart_archived LIKE srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests trying to archive a partition twice.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE srcpart_archived LIKE srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@srcpart_archived
+PREHOOK: query: INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12
+POSTHOOK: query: INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: srcpart_archived PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_archived PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: Error in semantic analysis: partition spec is not prefix: hr is bound while ds is not
Index: ql/src/test/results/clientnegative/archive_multi7.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive_multi7.q.out (revision 0)
+++ ql/src/test/results/clientnegative/archive_multi7.q.out (revision 0)
@@ -0,0 +1,41 @@
+PREHOOK: query: -- Tests trying to archive a partition group with custom locations.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests trying to archive a partition group with custom locations.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrcpart
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart ADD PARTITION (ds='2008-04-08', hr='12')
+LOCATION "/tmp/tstsrc"
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: default@tstsrcpart
+POSTHOOK: query: ALTER TABLE tstsrcpart ADD PARTITION (ds='2008-04-08', hr='12')
+LOCATION "/tmp/tstsrc"
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+FAILED: Error in metadata: ARCHIVE cannot run for partition groups with custom locations like pfile:/tmp/tstsrc
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask
Index: ql/src/test/results/clientnegative/archive_insert1.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive_insert1.q.out (revision 0)
+++ ql/src/test/results/clientnegative/archive_insert1.q.out (revision 0)
@@ -0,0 +1,34 @@
+PREHOOK: query: -- Tests trying to insert into archived partition.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests trying to insert into archived partition.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrcpart
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12')
+POSTHOOK: type: ALTERTABLE_ARCHIVE
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: Error in semantic analysis: Insert conflict with existing archive: ds=2008-04-08/hr=12
Index: ql/src/test/results/clientnegative/archive2.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive2.q.out (revision 1153271)
+++ ql/src/test/results/clientnegative/archive2.q.out (working copy)
@@ -29,5 +29,5 @@
PREHOOK: type: ALTERTABLE_UNARCHIVE
PREHOOK: Input: default@tstsrcpart
PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
-FAILED: Error in metadata: Specified partition is not archived
+FAILED: Error in metadata: Partition(s) not archived
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask
Index: ql/src/test/results/clientnegative/archive_multi2.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive_multi2.q.out (revision 0)
+++ ql/src/test/results/clientnegative/archive_multi2.q.out (revision 0)
@@ -0,0 +1,47 @@
+PREHOOK: query: -- Tests trying to unarchive a non-archived partition group
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+drop table tstsrcpart
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: -- Tests trying to unarchive a non-archived partition group
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+drop table tstsrcpart
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table tstsrcpart like srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table tstsrcpart like srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrcpart
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12')
+select key, value from srcpart where ds='2008-04-08' and hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12')
+select key, value from srcpart where ds='2008-04-08' and hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr='12')
+PREHOOK: type: ALTERTABLE_UNARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+FAILED: Error in metadata: Partition(s) not archived
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask
Index: ql/src/test/results/clientnegative/archive_insert4.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive_insert4.q.out (revision 0)
+++ ql/src/test/results/clientnegative/archive_insert4.q.out (revision 0)
@@ -0,0 +1,34 @@
+PREHOOK: query: -- Tests trying to (possible) dynamic insert into archived partition.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests trying to (possible) dynamic insert into archived partition.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrcpart
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12')
+POSTHOOK: type: ALTERTABLE_ARCHIVE
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: Error in semantic analysis: Insert conflict with existing archive: ds=2008-04-08/hr=12
Index: ql/src/test/results/clientnegative/archive_multi5.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive_multi5.q.out (revision 0)
+++ ql/src/test/results/clientnegative/archive_multi5.q.out (revision 0)
@@ -0,0 +1,56 @@
+PREHOOK: query: -- Tests trying to unarchive outer partition group containing other partition inside.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests trying to unarchive outer partition group containing other partition inside.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrcpart
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12')
+POSTHOOK: type: ALTERTABLE_ARCHIVE
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08')
+PREHOOK: type: ALTERTABLE_UNARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+FAILED: Error in metadata: Archiving was at different level, use ds=2008-04-08/hr=12
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask
Index: ql/src/test/results/clientnegative/archive_partspec3.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive_partspec3.q.out (revision 0)
+++ ql/src/test/results/clientnegative/archive_partspec3.q.out (revision 0)
@@ -0,0 +1,25 @@
+PREHOOK: query: -- Tests trying to archive a partition twice.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE srcpart_archived LIKE srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests trying to archive a partition twice.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE srcpart_archived LIKE srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@srcpart_archived
+PREHOOK: query: INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12
+POSTHOOK: query: INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: srcpart_archived PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_archived PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: Parse Error: line 3:48 mismatched input ')' expecting Identifier near '(' in archive statement
+
Index: ql/src/test/results/clientnegative/archive_insert2.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive_insert2.q.out (revision 0)
+++ ql/src/test/results/clientnegative/archive_insert2.q.out (revision 0)
@@ -0,0 +1,34 @@
+PREHOOK: query: -- Tests trying to insert into archived partition.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests trying to insert into archived partition.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrcpart
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+POSTHOOK: type: ALTERTABLE_ARCHIVE
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: Error in semantic analysis: Insert conflict with existing archive: ds=2008-04-08
Index: ql/src/test/results/clientnegative/archive_multi3.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive_multi3.q.out (revision 0)
+++ ql/src/test/results/clientnegative/archive_multi3.q.out (revision 0)
@@ -0,0 +1,56 @@
+PREHOOK: query: -- Tests trying to archive outer partition group containing other partition inside.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests trying to archive outer partition group containing other partition inside.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrcpart
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12')
+POSTHOOK: type: ALTERTABLE_ARCHIVE
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+FAILED: Error in metadata: Conflict with existing archive ds=2008-04-08/hr=12
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask
Index: ql/src/test/results/clientnegative/archive_partspec1.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive_partspec1.q.out (revision 0)
+++ ql/src/test/results/clientnegative/archive_partspec1.q.out (revision 0)
@@ -0,0 +1,24 @@
+PREHOOK: query: -- Tests trying to archive a partition twice.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE srcpart_archived LIKE srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests trying to archive a partition twice.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE srcpart_archived LIKE srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@srcpart_archived
+PREHOOK: query: INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12
+POSTHOOK: query: INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: srcpart_archived PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcpart_archived PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+FAILED: Error in semantic analysis: partition spec is not valid: {ds=2008-04-08, nonexistingpart=12}
Index: ql/src/test/results/clientnegative/archive_multi6.q.out
===================================================================
--- ql/src/test/results/clientnegative/archive_multi6.q.out (revision 0)
+++ ql/src/test/results/clientnegative/archive_multi6.q.out (revision 0)
@@ -0,0 +1,57 @@
+PREHOOK: query: -- Tests trying to unarchive inner partition contained in archived partition group.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: -- Tests trying to unarchive inner partition contained in archived partition group.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrcpart
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+POSTHOOK: type: ALTERTABLE_ARCHIVE
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr='12')
+PREHOOK: type: ALTERTABLE_UNARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+FAILED: Error in metadata: Archiving was at different level, use ds=2008-04-08
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask
Index: ql/src/test/results/clientpositive/archive_corrupt.q.out
===================================================================
--- ql/src/test/results/clientpositive/archive_corrupt.q.out (revision 0)
+++ ql/src/test/results/clientpositive/archive_corrupt.q.out (revision 0)
@@ -0,0 +1,122 @@
+PREHOOK: query: drop table tstsrcpart
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table tstsrcpart
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table tstsrcpart like srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table tstsrcpart like srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrcpart
+PREHOOK: query: load data local inpath '../data/files/archive_corrupt.rc' overwrite into table tstsrcpart partition (ds='2008-04-08', hr='11')
+PREHOOK: type: LOAD
+PREHOOK: Output: default@tstsrcpart
+POSTHOOK: query: load data local inpath '../data/files/archive_corrupt.rc' overwrite into table tstsrcpart partition (ds='2008-04-08', hr='11')
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12')
+select key, value from srcpart where ds='2008-04-08' and hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12')
+select key, value from srcpart where ds='2008-04-08' and hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11')
+select key, value from srcpart where ds='2008-04-09' and hr='11'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=11
+POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11')
+select key, value from srcpart where ds='2008-04-09' and hr='11'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=11
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12')
+select key, value from srcpart where ds='2008-04-09' and hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=12
+POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12')
+select key, value from srcpart where ds='2008-04-09' and hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=12
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+describe extended tstsrcpart partition (ds='2008-04-08', hr='11')
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+describe extended tstsrcpart partition (ds='2008-04-08', hr='11')
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+key string default
+value string default
+ds string
+hr string
+
+Detailed Partition Information Partition(values:[2008-04-08, 11], dbName:default, tableName:tstsrcpart, createTime:1312317091, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:pfile:/data/users/marcin/hive/build/ql/test/data/warehouse/tstsrcpart/ds=2008-04-08/hr=11, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{transient_lastDdlTime=1312317091})
+PREHOOK: query: alter table tstsrcpart archive partition (ds='2008-04-08', hr='11')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: query: alter table tstsrcpart archive partition (ds='2008-04-08', hr='11')
+POSTHOOK: type: ALTERTABLE_ARCHIVE
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: describe extended tstsrcpart partition (ds='2008-04-08', hr='11')
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe extended tstsrcpart partition (ds='2008-04-08', hr='11')
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+key string default
+value string default
+ds string
+hr string
+
+Detailed Partition Information Partition(values:[2008-04-08, 11], dbName:default, tableName:tstsrcpart, createTime:1312317091, lastAccessTime:0, sd:StorageDescriptor(cols:[FieldSchema(name:key, type:string, comment:default), FieldSchema(name:value, type:string, comment:default)], location:har://pfile/data/users/marcin/hive/build/ql/test/data/warehouse/tstsrcpart/ds=2008-04-08/hr=11/data.har/data/users/marcin/hive/build/ql/test/data/warehouse/tstsrcpart/ds=2008-04-08/hr=11/, inputFormat:org.apache.hadoop.mapred.TextInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null, serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{serialization.format=1}), bucketCols:[], sortCols:[], parameters:{}), parameters:{archiving_offset=2, transient_lastDdlTime=1312317109, is_archived=true, original_location=pfile:/data/users/marcin/hive/build/ql/test/data/warehouse/tstsrcpart/ds=2008-04-08/hr=11})
+PREHOOK: query: alter table tstsrcpart unarchive partition (ds='2008-04-08', hr='11')
+PREHOOK: type: ALTERTABLE_UNARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: query: alter table tstsrcpart unarchive partition (ds='2008-04-08', hr='11')
+POSTHOOK: type: ALTERTABLE_UNARCHIVE
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
Index: ql/src/test/results/clientpositive/archive_multi.q.out
===================================================================
--- ql/src/test/results/clientpositive/archive_multi.q.out (revision 0)
+++ ql/src/test/results/clientpositive/archive_multi.q.out (revision 0)
@@ -0,0 +1,261 @@
+PREHOOK: query: drop table tstsrc
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table tstsrc
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table tstsrcpart
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table tstsrcpart
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table tstsrc like src
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table tstsrc like src
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrc
+PREHOOK: query: insert overwrite table tstsrc select key, value from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@tstsrc
+POSTHOOK: query: insert overwrite table tstsrc select key, value from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tstsrc
+POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: create table tstsrcpart like srcpart
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table tstsrcpart like srcpart
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tstsrcpart
+POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='11')
+select key, value from srcpart where ds='2008-04-08' and hr='11'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='11')
+select key, value from srcpart where ds='2008-04-08' and hr='11'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12')
+select key, value from srcpart where ds='2008-04-08' and hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12')
+select key, value from srcpart where ds='2008-04-08' and hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11')
+select key, value from srcpart where ds='2008-04-09' and hr='11'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=11
+POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11')
+select key, value from srcpart where ds='2008-04-09' and hr='11'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=11
+POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12')
+select key, value from srcpart where ds='2008-04-09' and hr='12'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=12
+POSTHOOK: query: insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12')
+select key, value from srcpart where ds='2008-04-09' and hr='12'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-09/hr=12
+POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col
+FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-21_228_2560964695734311201/-mr-10000
+POSTHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col
+FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-21_228_2560964695734311201/-mr-10000
+POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+48479881068
+PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+PREHOOK: type: ALTERTABLE_ARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08')
+POSTHOOK: type: ALTERTABLE_ARCHIVE
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col
+FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-28_034_424539810414173134/-mr-10000
+POSTHOOK: query: SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col
+FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-28_034_424539810414173134/-mr-10000
+POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+48479881068
+PREHOOK: query: SELECT key, count(1) FROM tstsrcpart WHERE ds='2008-04-08' AND hr='12' AND key='0' GROUP BY key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-31_642_7853891038069299073/-mr-10000
+POSTHOOK: query: SELECT key, count(1) FROM tstsrcpart WHERE ds='2008-04-08' AND hr='12' AND key='0' GROUP BY key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-31_642_7853891038069299073/-mr-10000
+POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+0 3
+PREHOOK: query: SELECT * FROM tstsrcpart a JOIN tstsrc b ON a.key=b.key
+WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tstsrc
+PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-35_365_2814905405536965778/-mr-10000
+POSTHOOK: query: SELECT * FROM tstsrcpart a JOIN tstsrc b ON a.key=b.key
+WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tstsrc
+POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-35_365_2814905405536965778/-mr-10000
+POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+0 val_0 2008-04-08 12 0 val_0
+0 val_0 2008-04-08 12 0 val_0
+0 val_0 2008-04-08 12 0 val_0
+0 val_0 2008-04-08 12 0 val_0
+0 val_0 2008-04-08 12 0 val_0
+0 val_0 2008-04-08 12 0 val_0
+0 val_0 2008-04-08 12 0 val_0
+0 val_0 2008-04-08 12 0 val_0
+0 val_0 2008-04-08 12 0 val_0
+PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08')
+PREHOOK: type: ALTERTABLE_UNARCHIVE
+PREHOOK: Input: default@tstsrcpart
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08')
+POSTHOOK: type: ALTERTABLE_UNARCHIVE
+POSTHOOK: Input: default@tstsrcpart
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col
+FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-40_944_4568299913242010267/-mr-10000
+POSTHOOK: query: SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col
+FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@tstsrcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: file:/tmp/marcin/hive_2011-07-26_19-45-40_944_4568299913242010267/-mr-10000
+POSTHOOK: Lineage: tstsrc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-09,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
+48479881068
Index: ql/src/test/queries/clientnegative/archive_multi5.q
===================================================================
--- ql/src/test/queries/clientnegative/archive_multi5.q (revision 0)
+++ ql/src/test/queries/clientnegative/archive_multi5.q (revision 0)
@@ -0,0 +1,13 @@
+set hive.archive.enabled = true;
+-- Tests trying to unarchive outer partition group containing other partition inside.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart;
+
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11';
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12';
+
+ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12');
+ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08');
Index: ql/src/test/queries/clientnegative/archive_multi6.q
===================================================================
--- ql/src/test/queries/clientnegative/archive_multi6.q (revision 0)
+++ ql/src/test/queries/clientnegative/archive_multi6.q (revision 0)
@@ -0,0 +1,13 @@
+set hive.archive.enabled = true;
+-- Tests trying to unarchive inner partition contained in archived partition group.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart;
+
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11';
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12';
+
+ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08');
+ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr='12');
Index: ql/src/test/queries/clientnegative/archive_multi7.q
===================================================================
--- ql/src/test/queries/clientnegative/archive_multi7.q (revision 0)
+++ ql/src/test/queries/clientnegative/archive_multi7.q (revision 0)
@@ -0,0 +1,12 @@
+set hive.archive.enabled = true;
+-- Tests trying to archive a partition group with custom locations.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart;
+
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11';
+ALTER TABLE tstsrcpart ADD PARTITION (ds='2008-04-08', hr='12')
+LOCATION "/tmp/tstsrc";
+
+ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08');
Index: ql/src/test/queries/clientnegative/archive_insert1.q
===================================================================
--- ql/src/test/queries/clientnegative/archive_insert1.q (revision 0)
+++ ql/src/test/queries/clientnegative/archive_insert1.q (revision 0)
@@ -0,0 +1,13 @@
+set hive.archive.enabled = true;
+-- Tests trying to insert into archived partition.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart;
+
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12';
+
+ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12');
+
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12';
Index: ql/src/test/queries/clientnegative/archive_insert2.q
===================================================================
--- ql/src/test/queries/clientnegative/archive_insert2.q (revision 0)
+++ ql/src/test/queries/clientnegative/archive_insert2.q (revision 0)
@@ -0,0 +1,13 @@
+set hive.archive.enabled = true;
+-- Tests trying to insert into archived partition.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart;
+
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12';
+
+ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08');
+
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12';
Index: ql/src/test/queries/clientnegative/archive_multi1.q
===================================================================
--- ql/src/test/queries/clientnegative/archive_multi1.q (revision 0)
+++ ql/src/test/queries/clientnegative/archive_multi1.q (revision 0)
@@ -0,0 +1,13 @@
+set hive.archive.enabled = true;
+-- Tests trying to archive a partition twice.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart;
+
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11';
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12';
+
+ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08');
+ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08');
Index: ql/src/test/queries/clientnegative/archive_partspec1.q
===================================================================
--- ql/src/test/queries/clientnegative/archive_partspec1.q (revision 0)
+++ ql/src/test/queries/clientnegative/archive_partspec1.q (revision 0)
@@ -0,0 +1,10 @@
+set hive.archive.enabled = true;
+-- Tests trying to archive a partition twice.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE srcpart_archived LIKE srcpart;
+
+INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12';
+
+ALTER TABLE srcpart_archived ARCHIVE PARTITION (ds='2008-04-08', nonexistingpart='12');
Index: ql/src/test/queries/clientnegative/archive_insert3.q
===================================================================
--- ql/src/test/queries/clientnegative/archive_insert3.q (revision 0)
+++ ql/src/test/queries/clientnegative/archive_insert3.q (revision 0)
@@ -0,0 +1,13 @@
+set hive.archive.enabled = true;
+-- Tests trying to create partition inside of archived directory.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart;
+
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12';
+
+ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08');
+
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11';
Index: ql/src/test/queries/clientnegative/archive_multi2.q
===================================================================
--- ql/src/test/queries/clientnegative/archive_multi2.q (revision 0)
+++ ql/src/test/queries/clientnegative/archive_multi2.q (revision 0)
@@ -0,0 +1,12 @@
+set hive.archive.enabled = true;
+-- Tests trying to unarchive a non-archived partition group
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+drop table tstsrcpart;
+create table tstsrcpart like srcpart;
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11';
+insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12')
+select key, value from srcpart where ds='2008-04-08' and hr='12';
+
+ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr='12');
Index: ql/src/test/queries/clientnegative/archive_partspec2.q
===================================================================
--- ql/src/test/queries/clientnegative/archive_partspec2.q (revision 0)
+++ ql/src/test/queries/clientnegative/archive_partspec2.q (revision 0)
@@ -0,0 +1,10 @@
+set hive.archive.enabled = true;
+-- Tests trying to archive a partition twice.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE srcpart_archived LIKE srcpart;
+
+INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12';
+
+ALTER TABLE srcpart_archived ARCHIVE PARTITION (hr='12');
Index: ql/src/test/queries/clientnegative/archive_insert4.q
===================================================================
--- ql/src/test/queries/clientnegative/archive_insert4.q (revision 0)
+++ ql/src/test/queries/clientnegative/archive_insert4.q (revision 0)
@@ -0,0 +1,15 @@
+set hive.archive.enabled = true;
+-- Tests trying to (possible) dynamic insert into archived partition.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart;
+
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12';
+
+ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12');
+
+SET hive.exec.dynamic.partition=true;
+
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr)
+SELECT key, value, hr FROM srcpart WHERE ds='2008-04-08' AND hr='12';
Index: ql/src/test/queries/clientnegative/archive_multi3.q
===================================================================
--- ql/src/test/queries/clientnegative/archive_multi3.q (revision 0)
+++ ql/src/test/queries/clientnegative/archive_multi3.q (revision 0)
@@ -0,0 +1,13 @@
+set hive.archive.enabled = true;
+-- Tests trying to archive outer partition group containing other partition inside.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart;
+
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11';
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12';
+
+ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12');
+ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08');
Index: ql/src/test/queries/clientnegative/archive_partspec3.q
===================================================================
--- ql/src/test/queries/clientnegative/archive_partspec3.q (revision 0)
+++ ql/src/test/queries/clientnegative/archive_partspec3.q (revision 0)
@@ -0,0 +1,10 @@
+set hive.archive.enabled = true;
+-- Tests trying to archive a partition twice.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE srcpart_archived LIKE srcpart;
+
+INSERT OVERWRITE TABLE srcpart_archived PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12';
+
+ALTER TABLE srcpart_archived ARCHIVE PARTITION ();
Index: ql/src/test/queries/clientnegative/archive_multi4.q
===================================================================
--- ql/src/test/queries/clientnegative/archive_multi4.q (revision 0)
+++ ql/src/test/queries/clientnegative/archive_multi4.q (revision 0)
@@ -0,0 +1,13 @@
+set hive.archive.enabled = true;
+-- Tests trying to archive inner partition contained in archived partition group.
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+CREATE TABLE tstsrcpart LIKE srcpart;
+
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='11')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='11';
+INSERT OVERWRITE TABLE tstsrcpart PARTITION (ds='2008-04-08', hr='12')
+SELECT key, value FROM srcpart WHERE ds='2008-04-08' AND hr='12';
+
+ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08');
+ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='12');
Index: ql/src/test/queries/clientpositive/archive_multi.q
===================================================================
--- ql/src/test/queries/clientpositive/archive_multi.q (revision 0)
+++ ql/src/test/queries/clientpositive/archive_multi.q (revision 0)
@@ -0,0 +1,42 @@
+set hive.archive.enabled = true;
+set hive.enforce.bucketing = true;
+
+drop table tstsrc;
+drop table tstsrcpart;
+
+create table tstsrc like src;
+insert overwrite table tstsrc select key, value from src;
+
+create table tstsrcpart like srcpart;
+
+insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='11')
+select key, value from srcpart where ds='2008-04-08' and hr='11';
+
+insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12')
+select key, value from srcpart where ds='2008-04-08' and hr='12';
+
+insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11')
+select key, value from srcpart where ds='2008-04-09' and hr='11';
+
+insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12')
+select key, value from srcpart where ds='2008-04-09' and hr='12';
+
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col
+FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2;
+
+ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08');
+
+SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col
+FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2;
+
+SELECT key, count(1) FROM tstsrcpart WHERE ds='2008-04-08' AND hr='12' AND key='0' GROUP BY key;
+
+SELECT * FROM tstsrcpart a JOIN tstsrc b ON a.key=b.key
+WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0';
+
+ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08');
+
+SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col
+FROM (SELECT * FROM tstsrcpart WHERE ds='2008-04-08') subq1) subq2;
Index: ql/src/test/queries/clientpositive/archive_corrupt.q
===================================================================
--- ql/src/test/queries/clientpositive/archive_corrupt.q (revision 0)
+++ ql/src/test/queries/clientpositive/archive_corrupt.q (revision 0)
@@ -0,0 +1,27 @@
+set hive.archive.enabled = true;
+set hive.enforce.bucketing = true;
+
+drop table tstsrcpart;
+
+create table tstsrcpart like srcpart;
+
+load data local inpath '../data/files/archive_corrupt.rc' overwrite into table tstsrcpart partition (ds='2008-04-08', hr='11');
+
+insert overwrite table tstsrcpart partition (ds='2008-04-08', hr='12')
+select key, value from srcpart where ds='2008-04-08' and hr='12';
+
+insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='11')
+select key, value from srcpart where ds='2008-04-09' and hr='11';
+
+insert overwrite table tstsrcpart partition (ds='2008-04-09', hr='12')
+select key, value from srcpart where ds='2008-04-09' and hr='12';
+
+-- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.17, 0.18, 0.19)
+
+describe extended tstsrcpart partition (ds='2008-04-08', hr='11');
+
+alter table tstsrcpart archive partition (ds='2008-04-08', hr='11');
+
+describe extended tstsrcpart partition (ds='2008-04-08', hr='11');
+
+alter table tstsrcpart unarchive partition (ds='2008-04-08', hr='11');
Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java (revision 1153271)
+++ ql/src/java/org/apache/hadoop/hive/ql/metadata/DummyPartition.java (working copy)
@@ -18,8 +18,14 @@
package org.apache.hadoop.hive.ql.metadata;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
/**
* A Hive Table Partition: is a fundamental storage unit within a Table. Currently, Hive does not support
@@ -35,14 +41,26 @@
.getLog("hive.ql.metadata.DummyPartition");
private String name;
+ private LinkedHashMap partSpec;
+ private Table table;
+
public DummyPartition() {
}
- public DummyPartition(Table tbl, String name) throws HiveException {
+ public DummyPartition(Table tbl, String name, Map partSpec)
+ throws HiveException {
setTable(tbl);
this.name = name;
+ this.partSpec = new LinkedHashMap(partSpec);
+ this.table = tbl;
}
+ @Override
+ public Table getTable() {
+ return table;
+ }
+
+ @Override
public String getName() {
return name;
}
@@ -51,7 +69,24 @@
this.name = name;
}
+ @Override
public String getCompleteName() {
return getName();
}
+
+ @Override
+ public LinkedHashMap getSpec() {
+ return partSpec;
+ }
+
+ @Override
+ public List getValues() {
+ List values = new ArrayList();
+ for(FieldSchema fs: table.getPartCols()) {
+ values.add(partSpec.get(fs.getName()));
+ }
+ return values;
+ }
+
+
}
Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (revision 1153271)
+++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java (working copy)
@@ -355,6 +355,55 @@
return true;
}
+ /**
+ * Checks if given specification is proper specification for prefix of
+ * partition cols, for table partitioned by ds, hr, min valid ones are
+ * (ds='2008-04-08'), (ds='2008-04-08', hr='12'), (ds='2008-04-08', hr='12', min='30')
+ * invalid one is for example (ds='2008-04-08', min='30')
+ * @param spec specification key-value map
+ * @return true if the specification is prefix; never returns false, but throws
+ * @throws HiveException
+ */
+ final public boolean isValidPrefixSpec(Map spec)
+ throws HiveException {
+
+ // TODO - types need to be checked.
+ List partCols = tTable.getPartitionKeys();
+ if (partCols == null || (partCols.size() == 0)) {
+ if (spec != null) {
+ throw new HiveException(
+ "table is not partitioned but partition spec exists: " + spec);
+ } else {
+ return true;
+ }
+ }
+
+ if(spec == null) {
+ throw new HiveException("partition spec is not specified");
+ }
+
+ String unboundKey = null;
+ int boundKeys = 0;
+ for (FieldSchema field : partCols) {
+ if (spec.get(field.getName()) == null) {
+ unboundKey = field.getName();
+ } else if(unboundKey != null) {
+ throw new HiveException("partition spec is not prefix: "
+ + field.getName() + " is bound while " + unboundKey + " is not");
+ } else {
+ boundKeys += 1;
+ }
+ }
+
+ if (spec.size() != boundKeys) {
+ throw new HiveException(
+ "partition spec is not valid: "
+ + spec);
+ }
+
+ return true;
+ }
+
public void setProperty(String name, String value) {
tTable.getParameters().put(name, value);
}
@@ -709,7 +758,7 @@
public boolean isView() {
return TableType.VIRTUAL_VIEW.equals(getTableType());
}
-
+
/**
* @return whether this table is actually an index table
*/
Index: ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (revision 1153271)
+++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (working copy)
@@ -1506,14 +1506,16 @@
*
* @param tbl
* object for which partition is needed. Must be partitioned.
+ * @param limit number of partitions to return
* @return list of partition objects
* @throws HiveException
*/
- public List getPartitions(Table tbl, Map partialPartSpec)
+ public List getPartitions(Table tbl, Map partialPartSpec,
+ short limit)
throws HiveException {
if (!tbl.isPartitioned()) {
throw new HiveException("Partition spec should only be supplied for a " +
- "partitioned table");
+ "partitioned table");
}
List partialPvals = getPvals(tbl.getPartCols(), partialPartSpec);
@@ -1521,7 +1523,7 @@
List partitions = null;
try {
partitions = getMSC().listPartitionsWithAuthInfo(tbl.getDbName(), tbl.getTableName(),
- partialPvals, (short) -1, getUserName(), getGroupNames());
+ partialPvals, limit, getUserName(), getGroupNames());
} catch (Exception e) {
throw new HiveException(e);
}
@@ -1541,6 +1543,21 @@
*
* @param tbl
* object for which partition is needed. Must be partitioned.
+ * @return list of partition objects
+ * @throws HiveException
+ */
+ public List getPartitions(Table tbl, Map partialPartSpec)
+ throws HiveException {
+ return getPartitions(tbl, partialPartSpec, (short)-1);
+ }
+
+ /**
+ * get all the partitions of the table that matches the given partial
+ * specification. partition columns whose value is can be anything should be
+ * an empty string.
+ *
+ * @param tbl
+ * object for which partition is needed. Must be partitioned.
* @param partialPartSpec
* partial partition specification (some subpartitions can be empty).
* @return list of partition objects
@@ -1816,7 +1833,7 @@
// Note: there are race conditions here, but I don't believe
// they're worse than what was already present.
int counter = 1;
-
+
// Strip off the file type, if any so we don't make:
// 000000_0.gz -> 000000_0.gz_copy_1
String name = itemStaging.getName();
@@ -1828,10 +1845,10 @@
} else {
filetype = "";
}
-
+
Path itemDest = new Path(destf, itemStaging.getName());
Path itemStagingBase = new Path(itemStaging.getParent(), name);
-
+
while (fs.exists(itemDest)) {
Path proposedStaging = itemStagingBase
.suffix("_copy_" + counter++).suffix(filetype);
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java (revision 0)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/ArchiveUtils.java (revision 0)
@@ -0,0 +1,366 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+/**
+ * MetaUtils.
+ *
+ */
+@SuppressWarnings("nls")
+public final class ArchiveUtils {
+ private static final Log LOG = LogFactory.getLog(ArchiveUtils.class.getName());
+
+ /**
+ * PartSpecInfo keeps fields and values extracted from partial partition info
+ * which is prefix of the full info.
+ */
+ public static class PartSpecInfo {
+ public List fields;
+ public List values;
+ private PartSpecInfo(List fields, List values) {
+ this.fields = fields;
+ this.values = values;
+ }
+
+ /**
+ * Extract partial prefix specification from table and key-value map
+ *
+ * @param tbl table in which partition is
+ * @param partSpec specification of partition
+ * @return extracted specification
+ */
+ static public PartSpecInfo create(Table tbl, Map partSpec)
+ throws HiveException {
+ // we have to check if we receive prefix of partition keys so in table
+ // scheme like table/ds=2011-01-02/hr=13/
+ // ARCHIVE PARTITION (ds='2011-01-02') will work and
+ // ARCHIVE PARTITION(hr='13') won't
+ List prefixFields = new ArrayList();
+ List prefixValues = new ArrayList();
+ String unboundKey = null;
+ for(FieldSchema fs: tbl.getPartCols()) {
+ if(partSpec.containsKey(fs.getName())) {
+ if(unboundKey != null) {
+ break;
+ }
+ else {
+ prefixFields.add(fs);
+ prefixValues.add(partSpec.get(fs.getName()));
+ }
+ }
+ else if(unboundKey == null) {
+ unboundKey = fs.getName();
+ }
+ }
+ return new PartSpecInfo(prefixFields, prefixValues);
+ }
+
+ /**
+ * Creates path where partitions matching prefix should lie in filesystem
+ * @param tbl table in which partition is
+ * @return expected location of partitions matching prefix in filesystem
+ */
+ public Path createPath(Table tbl) throws HiveException {
+ String prefixSubdir;
+ try {
+ prefixSubdir = Warehouse.makePartName(fields, values);
+ } catch (MetaException e) {
+ throw new HiveException("Unable to get partitions directories prefix", e);
+ }
+ URI tableDir = tbl.getDataLocation();
+ if(tableDir == null) {
+ throw new HiveException("Table has no location set");
+ }
+ return new Path(tableDir.toString(), prefixSubdir);
+ }
+ /**
+ * Generates name for prefix partial partition specification.
+ */
+ public String getName() throws HiveException {
+ try {
+ return Warehouse.makePartName(fields, values);
+ } catch (MetaException e) {
+ throw new HiveException("Unable to create partial name", e);
+ }
+ }
+ }
+
+ /**
+ * HarPathHelper helps to create har:/ URIs for locations inside of archive.
+ */
+ public static class HarPathHelper {
+ boolean parentSettable;
+ private final URI base, originalBase;
+
+ /**
+ * Creates helper for archive.
+ * @param archive absolute location of archive in underlying filesystem
+ * @param originalBase directory for which Hadoop archive was created
+ */
+ public HarPathHelper(HiveConf hconf, URI archive, URI originalBase) throws HiveException {
+ parentSettable = hconf.getBoolVar(HiveConf.ConfVars.HIVEHARPARENTDIRSETTABLE);
+ this.originalBase = addSlash(originalBase);
+ String parentHost = archive.getHost();
+ String harHost = null;
+ if (parentHost == null) {
+ harHost = archive.getScheme();
+ } else {
+ harHost = archive.getScheme() + "-" + parentHost;
+ }
+
+ // have to make sure there's slash after .har, otherwise resolve doesn't work
+ String path = addSlash(archive.getPath());
+ if(!path.endsWith(".har/")) {
+ throw new HiveException("HAR archive path must end with .har");
+ }
+ // harUri is used to access the partition's files, which are in the archive
+ // The format of the RI is something like:
+ // har://underlyingfsscheme-host:port/archivepath
+ try {
+ base = new URI("har", archive.getUserInfo(), harHost, archive.getPort(),
+ path, archive.getQuery(), archive.getFragment());
+ } catch (URISyntaxException e) {
+ throw new HiveException("Couldn't create har URI from archive URI", e);
+ }
+ }
+
+ /**
+ * Creates har URI for file/directory that was put there when creating HAR.
+ *
+ * With older versions of Hadoop, archiving a directory would produce
+ * the same directory structure, reflecting absoulute paths.
+ * If you created myArchive.har of /tmp/myDir the files in /tmp/myDir
+ * will be located under myArchive.har/tmp/myDir/*
+ *
+ * With newer versions, the parent directory can be specified. Assuming
+ * the parent directory was set to /tmp/myDir when creating the archive,
+ * the files can be found under myArchive.har/*
+ *
+ * This is why originalBase is argument - with new versions we can
+ * relativize URI, in older we keep absolute one.
+ *
+ * @param original file/directory path
+ * @return absolute HAR uri
+ */
+ public URI getHarUri(URI original) throws HiveException {
+ URI relative = null;
+ if (!parentSettable) {
+ String dirInArchive = original.getPath();
+ if(dirInArchive.length() > 1 && dirInArchive.charAt(0)=='/') {
+ dirInArchive = dirInArchive.substring(1);
+ }
+ try {
+ relative = new URI(null, null, dirInArchive, null);
+ } catch (URISyntaxException e) {
+ throw new HiveException("Couldn't create har URI for location");
+ } // relative URI with path only
+ }
+ else {
+ relative = originalBase.relativize(original);
+ if(relative.isAbsolute()) {
+ throw new HiveException("Unable to relativize URI");
+ }
+ }
+ return base.resolve(relative);
+ }
+ }
+
+ public static String addSlash(String s) {
+ return s.endsWith("/") ? s : s + "/";
+ }
+
+ /**
+ * Makes sure, that URI points to directory by adding slash to it.
+ * Useful in relativizing URIs.
+ */
+ public static URI addSlash(URI u) throws HiveException {
+ if(u.getPath().endsWith("/")) {
+ return u;
+ } else {
+ try {
+ return new URI(u.getScheme(), u.getAuthority(), u.getPath() + "/", u.getQuery(), u.getFragment());
+ } catch (URISyntaxException e) {
+ // It cannot happen (can it?)
+ throw new HiveException("Couldn't append slash to a URI", e);
+ }
+ }
+ }
+
+ /**
+ * Determines whether a partition has been archived
+ *
+ * @param p
+ * @return
+ */
+ public static boolean isArchived(Partition p) {
+ Map params = p.getParameters();
+ if ("true".equalsIgnoreCase(params.get(
+ org.apache.hadoop.hive.metastore.api.Constants.IS_ARCHIVED))) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Returns archiving level, which is how many fields were set in partial
+ * specification ARCHIVE was run for
+ */
+ public static int getArchivingLevel(Partition p) throws HiveException {
+ if(!isArchived(p)) {
+ throw new HiveException("Getting level of unarchived partition");
+ }
+
+ Map params = p.getParameters();
+ String lv = params.get(
+ org.apache.hadoop.hive.metastore.api.Constants.ARCHIVING_LEVEL);
+ if(lv != null) {
+ return Integer.parseInt(lv);
+ } else { // partitions archived before introducing multiple archiving
+ return p.getValues().size();
+ }
+ }
+
+ /**
+ * Checks if table is flagged as containing archived partition.
+ * @param tbl Table object
+ * @return Counter value
+ */
+ public static boolean containsArchivedPartitions(Table tbl) {
+ String val = tbl.getProperty(org.apache.hadoop.hive.metastore
+ .api.Constants.CONTAINS_ARCHIVED_PARTITIONS);
+ return "true".equals(val);
+ }
+
+ public static String getPartialName(Partition p, int level) throws HiveException {
+ List ffields = p.getTable().getPartCols();
+ List fvalues = p.getValues();
+ List fields = new ArrayList(level);
+ List values = new ArrayList(level);
+ int lev = level;
+ for(String s: fvalues) {
+ values.add(s);
+ if(--lev == 0) {
+ break;
+ }
+ }
+ lev = level;
+ for(FieldSchema fs: ffields) {
+ fields.add(fs);
+ if(--lev == 0) {
+ break;
+ }
+ }
+ try {
+ return Warehouse.makePartName(fields, values);
+ } catch (MetaException e) {
+ throw new HiveException("Wasn't able to generate name" +
+ " for partial specification");
+ }
+ }
+
+ /**
+ * Determines if one can insert into partition(s), or there's a conflict with
+ * archive. It can be because partition is itself archived or it is to be
+ * created inside existing archive. The second case is when partition doesn't
+ * exist yet, but it existed it would be inside of an archive. This one is
+ * quite tricky to check, we need to find at least one partition inside of
+ * the parent directory. If it is archived and archiving level tells that
+ * the archival was done of directory partition is in it means we cannot
+ * insert; otherwise we can.
+ * This method works both for full specifications and partial ones - in second
+ * case it checks if any partition that could possibly match such
+ * specification is inside archive.
+ *
+ * @param db - Hive object
+ * @param tbl - table where partition is
+ * @param partSpec - partition specification with possible nulls in case of
+ * dynamic partiton inserts
+ * @return null if partition can be inserted, string with colliding archive
+ * name when it can't
+ * @throws HiveException
+ */
+ public static String conflictingArchiveNameOrNull(Hive db, Table tbl,
+ LinkedHashMap partSpec)
+ throws HiveException {
+ if(!containsArchivedPartitions(tbl)) {
+ return null;
+ }
+ List partitions = db.getPartitions(tbl, partSpec);
+ if(!partitions.isEmpty()) {
+ LOG.debug("Found some partitions matching the provided specification");
+ for(Partition p: partitions) {
+ if(isArchived(p)) {
+ return getPartialName(p, getArchivingLevel(p));
+ }
+ }
+ } else {
+ LOG.debug("Haven't found any partitions matching the provided specification");
+ Map spec = new HashMap(partSpec);
+ List reversedKeys = new LinkedList();
+ for(FieldSchema fs: tbl.getPartCols()) {
+ if(spec.containsKey(fs.getName())) {
+ reversedKeys.add(0, fs.getName());
+ }
+ }
+ for(String rk: reversedKeys) {
+ spec.remove(rk);
+ List parts = db.getPartitions(tbl, spec, (short)1);
+ if(parts.size() != 0) {
+ Partition p = parts.get(0);
+ if(!isArchived(p)) {
+ // if archiving was done at this or at upper level, every matched
+ // partition would be archived, so it not being archived means
+ // no archiving was done neither at this nor at upper level
+ return null;
+ } else if(getArchivingLevel(p) > spec.size()) {
+ // if archiving was done at this or at upper level its level
+ // would be lesser or equal to specification size
+ // it is not, which means no archiving at this or upper level
+ return null;
+ } else {
+ return getPartialName(p, getArchivingLevel(p));
+ }
+ }
+ }
+ }
+ return null;
+ }
+}
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (revision 1153271)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (working copy)
@@ -1929,22 +1929,6 @@
}
/**
- * Determines whether a partition has been archived
- *
- * @param p
- * @return
- */
- public static boolean isArchived(Partition p) {
- Map params = p.getParameters();
- if ("true".equalsIgnoreCase(params.get(
- org.apache.hadoop.hive.metastore.api.Constants.IS_ARCHIVED))) {
- return true;
- } else {
- return false;
- }
- }
-
- /**
* Check if a function can be pushed down to JDO.
* Now only {=, AND, OR} are supported.
* @param func a generic function.
Index: ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision 1153271)
+++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (working copy)
@@ -73,6 +73,7 @@
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.DriverContext;
import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.exec.ArchiveUtils.PartSpecInfo;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.io.rcfile.merge.BlockMergeTask;
@@ -969,22 +970,37 @@
return 0;
}
- private void setIsArchived(Partition p, boolean state) {
+ /**
+ * Sets archiving flag locally; it has to be pushed into metastore
+ * @param p partition to set flag
+ * @param state desired state of IS_ARCHIVED flag
+ * @param level desired level for state == true, anything for false
+ */
+ private void setIsArchived(Partition p, boolean state, int level) {
Map params = p.getParameters();
if (state) {
params.put(org.apache.hadoop.hive.metastore.api.Constants.IS_ARCHIVED,
"true");
+ params.put(org.apache.hadoop.hive.metastore.api.Constants.ARCHIVING_LEVEL,
+ Integer.toString(level));
} else {
params.remove(org.apache.hadoop.hive.metastore.api.Constants.IS_ARCHIVED);
+ params.remove(org.apache.hadoop.hive.metastore.api.Constants.ARCHIVING_LEVEL);
}
}
+ /**
+ * Returns original partition of archived partition, null for unarchived one
+ */
private String getOriginalLocation(Partition p) {
Map params = p.getParameters();
return params.get(
org.apache.hadoop.hive.metastore.api.Constants.ORIGINAL_LOCATION);
}
+ /**
+ * Sets original location of partition which is to be archived
+ */
private void setOriginalLocation(Partition p, String loc) {
Map params = p.getParameters();
if (loc == null) {
@@ -994,56 +1010,65 @@
}
}
- // Returns only the path component of the URI
- private String getArchiveDirOnly(Path parentDir, String archiveName) {
- URI parentUri = parentDir.toUri();
- Path harDir = new Path(parentUri.getPath(), archiveName);
- return harDir.toString();
- }
-
/**
* Sets the appropriate attributes in the supplied Partition object to mark
* it as archived. Note that the metastore is not touched - a separate
* call to alter_partition is needed.
*
* @param p - the partition object to modify
- * @param parentDir - the parent directory of the archive, which is the
- * original directory that the partition's files resided in
- * @param dirInArchive - the directory within the archive file that contains
- * the partitions files
- * @param archiveName - the name of the archive
- * @throws URISyntaxException
+ * @param harPath - new location of partition (har schema URI)
*/
- private void setArchived(Partition p, Path parentDir, String dirInArchive, String archiveName)
- throws URISyntaxException {
- assert(Utilities.isArchived(p) == false);
- Map params = p.getParameters();
+ private void setArchived(Partition p, Path harPath, int level) {
+ assert(ArchiveUtils.isArchived(p) == false);
- URI parentUri = parentDir.toUri();
- String parentHost = parentUri.getHost();
- String harHost = null;
- if (parentHost == null) {
- harHost = "";
- } else {
- harHost = parentUri.getScheme() + "-" + parentHost;
+ setIsArchived(p, true, level);
+ setOriginalLocation(p, p.getLocation());
+ p.setLocation(harPath.toString());
+ }
+
+ /**
+ * Sets CONTAINS_ARCHIVED_PARTITION flag to true.
+ * @param db Hive object for database
+ * @param t table to change the flag
+ */
+ private void setContainsArchived(Hive db, Table t) throws HiveException {
+ Table tbl = db.getTable(t.getTableName());
+ String key = org.apache.hadoop.hive.metastore.api.Constants.CONTAINS_ARCHIVED_PARTITIONS;
+ tbl.getParameters().put(key, "true");
+ try {
+ db.alterTable(t.getTableName(), tbl);
+ } catch (Exception e) {
+ throw new HiveException(e);
}
+ }
- // harUri is used to access the partition's files, which are in the archive
- // The format of the RI is something like:
- // har://underlyingfsscheme-host:port/archivepath
- URI harUri = null;
- if (dirInArchive.length() == 0) {
- harUri = new URI("har", parentUri.getUserInfo(), harHost, parentUri.getPort(),
- getArchiveDirOnly(parentDir, archiveName),
- parentUri.getQuery(), parentUri.getFragment());
- } else {
- harUri = new URI("har", parentUri.getUserInfo(), harHost, parentUri.getPort(),
- new Path(getArchiveDirOnly(parentDir, archiveName), dirInArchive).toUri().getPath(),
- parentUri.getQuery(), parentUri.getFragment());
+ /**
+ * Checks if there are any archived partitions in table and sets
+ * CONTAINS_ARCHIVED_PARTITION flag properly.
+ * Omited if hive.archive.contains.archived.update is false.
+ * @param db Hive object for database
+ * @param t table to change the flag
+ */
+ private void updateContainsArchived(Hive db, Table t) throws HiveException {
+ if(!conf.getBoolVar(HiveConf.ConfVars.HIVEARCHIVECONTAINSARCHIVEDUPDATE)) {
+ return;
}
- setIsArchived(p, true);
- setOriginalLocation(p, parentDir.toString());
- p.setLocation(harUri.toString());
+ Table tbl = db.getTable(t.getTableName());
+ List partitions = db.getPartitions(tbl);
+ String val = "false";
+ for(Partition p: partitions) {
+ if(ArchiveUtils.isArchived(p)) {
+ val = "true";
+ break;
+ }
+ }
+ String key = org.apache.hadoop.hive.metastore.api.Constants.CONTAINS_ARCHIVED_PARTITIONS;
+ tbl.getParameters().put(key, val);
+ try {
+ db.alterTable(t.getTableName(), tbl);
+ } catch (Exception e) {
+ throw new HiveException(e);
+ }
}
/**
@@ -1054,9 +1079,9 @@
* @param p - the partition to modify
*/
private void setUnArchived(Partition p) {
- assert(Utilities.isArchived(p) == true);
+ assert(ArchiveUtils.isArchived(p) == true);
String parentDir = getOriginalLocation(p);
- setIsArchived(p, false);
+ setIsArchived(p, false, 0);
setOriginalLocation(p, null);
assert(parentDir != null);
p.setLocation(parentDir);
@@ -1090,6 +1115,33 @@
}
}
+ /**
+ * Checks in partition is in custom (not-standard) location.
+ * @param tbl - table in which partition is
+ * @param p - partition
+ * @return true if partition location is custom, false if it is standard
+ */
+ boolean partitionInCustomLocation(Table tbl, Partition p)
+ throws HiveException {
+ String subdir = null;
+ try {
+ subdir = Warehouse.makePartName(tbl.getPartCols(), p.getValues());
+ } catch (MetaException e) {
+ throw new HiveException("Unable to get partition's directory", e);
+ }
+ URI tableDir = tbl.getDataLocation();
+ if(tableDir == null) {
+ throw new HiveException("Table has no location set");
+ }
+
+ String standardLocation = (new Path(tableDir.toString(), subdir)).toString();
+ if(ArchiveUtils.isArchived(p)) {
+ return !getOriginalLocation(p).equals(standardLocation);
+ } else {
+ return !p.getLocation().equals(standardLocation);
+ }
+ }
+
private int archive(Hive db, AlterTableSimpleDesc simpleDesc, DriverContext driverContext)
throws HiveException {
String dbName = simpleDesc.getDbName();
@@ -1098,35 +1150,43 @@
Table tbl = db.getTable(dbName, tblName);
validateAlterTableType(tbl, AlterTableDesc.AlterTableTypes.ARCHIVE);
- Map partSpec = simpleDesc.getPartSpec();
- Partition p = db.getPartition(tbl, partSpec, false);
-
if (tbl.getTableType() != TableType.MANAGED_TABLE) {
throw new HiveException("ARCHIVE can only be performed on managed tables");
}
- if (p == null) {
- throw new HiveException("Specified partition does not exist");
- }
+ Map partSpec = simpleDesc.getPartSpec();
+ PartSpecInfo partSpecInfo = PartSpecInfo.create(tbl, partSpec);
+ List partitions = db.getPartitions(tbl, partSpec);
- if (Utilities.isArchived(p)) {
- // If there were a failure right after the metadata was updated in an
- // archiving operation, it's possible that the original, unarchived files
- // weren't deleted.
- Path originalDir = new Path(getOriginalLocation(p));
- Path leftOverIntermediateOriginal = new Path(originalDir.getParent(),
- originalDir.getName() + INTERMEDIATE_ORIGINAL_DIR_SUFFIX);
+ Path originalDir = null;
- if (pathExists(leftOverIntermediateOriginal)) {
- console.printInfo("Deleting " + leftOverIntermediateOriginal +
- " left over from a previous archiving operation");
- deleteDir(leftOverIntermediateOriginal);
+ // when we have partial partitions specification we must assume partitions
+ // lie in standard place - if they were in custom locations putting
+ // them into one archive would involve mass amount of copying
+ // in full partition specification case we allow custom locations
+ // to keep backward compatibility
+ if (partitions.isEmpty()) {
+ throw new HiveException("No partition matches the specification");
+ } else if(partSpecInfo.values.size() != tbl.getPartCols().size()) {
+ // for partial specifications we need partitions to follow the scheme
+ for(Partition p: partitions){
+ if(partitionInCustomLocation(tbl, p)) {
+ String message = String.format("ARCHIVE cannot run for partition " +
+ "groups with custom locations like %s", p.getLocation());
+ throw new HiveException(message);
+ }
}
-
- throw new HiveException("Specified partition is already archived");
+ originalDir = partSpecInfo.createPath(tbl);
+ } else {
+ Partition p = partitions.get(0);
+ // partition can be archived if during recovery
+ if(ArchiveUtils.isArchived(p)) {
+ originalDir = new Path(getOriginalLocation(p));
+ } else {
+ originalDir = p.getPartitionPath();
+ }
}
- Path originalDir = p.getPartitionPath();
Path intermediateArchivedDir = new Path(originalDir.getParent(),
originalDir.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX);
Path intermediateOriginalDir = new Path(originalDir.getParent(),
@@ -1139,6 +1199,36 @@
throw new HiveException(e);
}
+ URI originalUri = ArchiveUtils.addSlash(originalDir.toUri());
+ URI archiveUri = (new Path(originalDir, archiveName)).toUri();
+ ArchiveUtils.HarPathHelper harHelper = new ArchiveUtils.HarPathHelper(
+ conf, archiveUri, originalUri);
+
+ // we checked if partitions matching specification are marked as archived
+ // in the metadata; if they are and their levels are the same as we would
+ // set it later it means previous run failed and we have to do the recovery;
+ // if they are different, we throw an error
+ boolean anyUnarchived = false;
+ for(Partition p: partitions) {
+ if(ArchiveUtils.isArchived(p)) {
+ if(ArchiveUtils.getArchivingLevel(p) != partSpecInfo.values.size()) {
+ String name = ArchiveUtils.getPartialName(p, ArchiveUtils.getArchivingLevel(p));
+ String m = String.format("Conflict with existing archive %s", name);
+ throw new HiveException(m);
+ }
+ } else {
+ anyUnarchived = true;
+ }
+ }
+
+ boolean recovery = false;
+ if(pathExists(intermediateArchivedDir) || pathExists(intermediateOriginalDir)) {
+ recovery = true;
+ console.printInfo("Starting recovery after failed ARCHIVE");
+ } else if(!anyUnarchived) {
+ throw new HiveException("Partition(s) already archived");
+ }
+
// The following steps seem roundabout, but they are meant to aid in
// recovery if a failure occurs and to keep a consistent state in the FS
@@ -1166,10 +1256,12 @@
// First create the archive in a tmp dir so that if the job fails, the
// bad files don't pollute the filesystem
- Path tmpDir = new Path(driverContext.getCtx().getExternalTmpFileURI(originalDir.toUri()), "partlevel");
+ Path tmpPath = new Path(driverContext.getCtx()
+ .getExternalTmpFileURI(originalDir.toUri()), "partlevel");
- console.printInfo("Creating " + archiveName + " for " + originalDir.toString());
- console.printInfo("in " + tmpDir);
+ console.printInfo("Creating " + archiveName +
+ " for " + originalDir.toString());
+ console.printInfo("in " + tmpPath);
console.printInfo("Please wait... (this may take a while)");
// Create the Hadoop archive
@@ -1178,10 +1270,10 @@
try {
int maxJobNameLen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
String jobname = String.format("Archiving %s@%s",
- tbl.getTableName(), p.getName());
+ tbl.getTableName(), partSpecInfo.getName());
jobname = Utilities.abbreviate(jobname, maxJobNameLen - 6);
conf.setVar(HiveConf.ConfVars.HADOOPJOBNAME, jobname);
- ret = shim.createHadoopArchive(conf, originalDir, tmpDir, archiveName);
+ ret = shim.createHadoopArchive(conf, originalDir, tmpPath, archiveName);
} catch (Exception e) {
throw new HiveException(e);
}
@@ -1191,11 +1283,11 @@
// Move from the tmp dir to an intermediate directory, in the same level as
// the partition directory. e.g. .../hr=12-intermediate-archived
try {
- console.printInfo("Moving " + tmpDir + " to " + intermediateArchivedDir);
+ console.printInfo("Moving " + tmpPath + " to " + intermediateArchivedDir);
if (pathExists(intermediateArchivedDir)) {
throw new HiveException("The intermediate archive directory already exists.");
}
- fs.rename(tmpDir, intermediateArchivedDir);
+ fs.rename(tmpPath, intermediateArchivedDir);
} catch (IOException e) {
throw new HiveException("Error while moving tmp directory");
}
@@ -1239,39 +1331,31 @@
// Record this change in the metastore
try {
- boolean parentSettable =
- conf.getBoolVar(HiveConf.ConfVars.HIVEHARPARENTDIRSETTABLE);
-
- // dirInArchive is the directory within the archive that has all the files
- // for this partition. With older versions of Hadoop, archiving a
- // a directory would produce the same directory structure
- // in the archive. So if you created myArchive.har of /tmp/myDir, the
- // files in /tmp/myDir would be located under myArchive.har/tmp/myDir/*
- // In this case, dirInArchive should be tmp/myDir
-
- // With newer versions of Hadoop, the parent directory could be specified.
- // Assuming the parent directory was set to /tmp/myDir when creating the
- // archive, the files can be found under myArchive.har/*
- // In this case, dirInArchive should be empty
-
- String dirInArchive = "";
- if (!parentSettable) {
- dirInArchive = originalDir.toUri().getPath();
- if(dirInArchive.length() > 1 && dirInArchive.charAt(0)=='/') {
- dirInArchive = dirInArchive.substring(1);
- }
+ for(Partition p: partitions) {
+ URI originalPartitionUri = ArchiveUtils.addSlash(p.getPartitionPath().toUri());
+ URI harPartitionDir = harHelper.getHarUri(originalPartitionUri);
+ Path harPath = new Path(harPartitionDir.getScheme(),
+ harPartitionDir.getAuthority(),
+ harPartitionDir.getPath()); // make in Path to ensure no slash at the end
+ setArchived(p, harPath, partSpecInfo.values.size());
+ db.alterPartition(tblName, p);
}
- setArchived(p, originalDir, dirInArchive, archiveName);
- db.alterPartition(tblName, p);
} catch (Exception e) {
throw new HiveException("Unable to change the partition info for HAR", e);
}
// If a failure occurs here, the directory containing the original files
// will not be deleted. The user will run ARCHIVE again to clear this up
- deleteDir(intermediateOriginalDir);
+ if(pathExists(intermediateOriginalDir)) {
+ deleteDir(intermediateOriginalDir);
+ }
+ setContainsArchived(db, tbl);
+ if(recovery) {
+ console.printInfo("Recovery after ARCHIVE succeeded");
+ }
+
return 0;
}
@@ -1285,95 +1369,140 @@
// Means user specified a table, not a partition
if (simpleDesc.getPartSpec() == null) {
- throw new HiveException("ARCHIVE is for partitions only");
+ throw new HiveException("UNARCHIVE is for partitions only");
}
- Map partSpec = simpleDesc.getPartSpec();
- Partition p = db.getPartition(tbl, partSpec, false);
-
if (tbl.getTableType() != TableType.MANAGED_TABLE) {
throw new HiveException("UNARCHIVE can only be performed on managed tables");
}
- if (p == null) {
- throw new HiveException("Specified partition does not exist");
+ Map partSpec = simpleDesc.getPartSpec();
+ PartSpecInfo partSpecInfo = PartSpecInfo.create(tbl, partSpec);
+ List partitions = db.getPartitions(tbl, partSpec);
+
+ Path originalDir = null;
+
+ // when we have partial partitions specification we must assume partitions
+ // lie in standard place - if they were in custom locations putting
+ // them into one archive would involve mass amount of copying
+ // in full partition specification case we allow custom locations
+ // to keep backward compatibility
+ if (partitions.isEmpty()) {
+ throw new HiveException("No partition matches the specification");
+ } else if(partSpecInfo.values.size() != tbl.getPartCols().size()) {
+ // for partial specifications we need partitions to follow the scheme
+ for(Partition p: partitions){
+ if(partitionInCustomLocation(tbl, p)) {
+ String message = String.format("UNARCHIVE cannot run for partition " +
+ "groups with custom locations like %s", p.getLocation());
+ throw new HiveException(message);
+ }
+ }
+ originalDir = partSpecInfo.createPath(tbl);
+ } else {
+ Partition p = partitions.get(0);
+ if(ArchiveUtils.isArchived(p)) {
+ originalDir = new Path(getOriginalLocation(p));
+ } else {
+ originalDir = p.getPartitionPath();
+ }
}
- if (!Utilities.isArchived(p)) {
- Path location = new Path(p.getLocation());
- Path leftOverArchiveDir = new Path(location.getParent(),
- location.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX);
+ URI originalUri = ArchiveUtils.addSlash(originalDir.toUri());
+ Path intermediateArchivedDir = new Path(originalDir.getParent(),
+ originalDir.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX);
+ Path intermediateExtractedDir = new Path(originalDir.getParent(),
+ originalDir.getName() + INTERMEDIATE_EXTRACTED_DIR_SUFFIX);
+ String archiveName = "data.har";
+ FileSystem fs = null;
+ try {
+ fs = originalDir.getFileSystem(conf);
+ } catch (IOException e) {
+ throw new HiveException(e);
+ }
- if (pathExists(leftOverArchiveDir)) {
- console.printInfo("Deleting " + leftOverArchiveDir + " left over " +
- "from a previous unarchiving operation");
- deleteDir(leftOverArchiveDir);
+ // if nothing is archived we throw an error saying partition not archived
+ // if only some partitions are archived we check if these are archived
+ // in right place and if so, it means we are during the recovery
+ boolean anyArchived = false;
+ for(Partition p: partitions) {
+ if(ArchiveUtils.isArchived(p)) {
+ anyArchived = true;
}
+ }
- throw new HiveException("Specified partition is not archived");
+ // assume the archive is in the original dir, check if it exists
+ Path archivePath = new Path(originalDir, archiveName);
+ URI archiveUri = archivePath.toUri();
+ ArchiveUtils.HarPathHelper harHelper = new ArchiveUtils.HarPathHelper(conf,
+ archiveUri, originalUri);
+ URI sourceUri = harHelper.getHarUri(originalUri);
+ Path sourceDir = new Path(sourceUri.getScheme(), sourceUri.getAuthority(), sourceUri.getPath());
+
+ // check if all partitions' metadata tells they lie in the archive
+ for(Partition p: partitions) {
+ if(ArchiveUtils.isArchived(p)){
+ if(ArchiveUtils.getArchivingLevel(p) != partSpecInfo.values.size()) {
+ String name = ArchiveUtils.getPartialName(p, ArchiveUtils.getArchivingLevel(p));
+ String m = String.format("Archiving was at different level, use %s",
+ name);
+ throw new HiveException(m);
+ }
+ }
}
- Path originalLocation = new Path(getOriginalLocation(p));
- Path sourceDir = new Path(p.getLocation());
- Path intermediateArchiveDir = new Path(originalLocation.getParent(),
- originalLocation.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX);
- Path intermediateExtractedDir = new Path(originalLocation.getParent(),
- originalLocation.getName() + INTERMEDIATE_EXTRACTED_DIR_SUFFIX);
+ boolean recovery = false;
+ if(pathExists(intermediateArchivedDir) || pathExists(intermediateExtractedDir)) {
+ recovery = true;
+ console.printInfo("Starting recovery after failed UNARCHIVE");
+ } else if(!anyArchived) {
+ throw new HiveException("Partition(s) not archived");
+ }
- Path tmpDir = new Path(driverContext
+ if(!pathExists(intermediateArchivedDir) && !pathExists(archivePath)) {
+ throw new HiveException("Haven't found any archive where it should be");
+ }
+
+ Path tmpPath = new Path(driverContext
.getCtx()
- .getExternalTmpFileURI(originalLocation.toUri()));
+ .getExternalTmpFileURI(originalDir.toUri()));
- FileSystem fs = null;
try {
- fs = tmpDir.getFileSystem(conf);
- // Verify that there are no files in the tmp dir, because if there are, it
- // would be copied to the partition
- FileStatus [] filesInTmpDir = fs.listStatus(tmpDir);
- if (filesInTmpDir != null && filesInTmpDir.length != 0) {
- for (FileStatus file : filesInTmpDir) {
- console.printInfo(file.getPath().toString());
- }
- throw new HiveException("Temporary directory " + tmpDir + " is not empty");
- }
-
+ fs = tmpPath.getFileSystem(conf);
} catch (IOException e) {
throw new HiveException(e);
}
// Some sanity checks
- if (originalLocation == null) {
+ if (originalDir == null) {
throw new HiveException("Missing archive data in the partition");
}
- if (!"har".equals(sourceDir.toUri().getScheme())) {
- throw new HiveException("Location should refer to a HAR");
- }
// Clarification of terms:
- // - The originalLocation directory represents the original directory of the
- // partition's files. They now contain an archived version of those files
+ // - The originalDir directory represents the original directory of the
+ // partitions' files. They now contain an archived version of those files
// eg. hdfs:/warehouse/myTable/ds=1/
// - The source directory is the directory containing all the files that
- // should be in the partition. e.g. har:/warehouse/myTable/ds=1/myTable.har/
+ // should be in the partitions. e.g. har:/warehouse/myTable/ds=1/myTable.har/
// Note the har:/ scheme
// Steps:
// 1. Extract the archive in a temporary folder
// 2. Move the archive dir to an intermediate dir that is in at the same
// dir as originalLocation. Call the new dir intermediate-extracted.
- // 3. Rename the original partition dir to an intermediate dir. Call the
+ // 3. Rename the original partitions dir to an intermediate dir. Call the
// renamed dir intermediate-archive
- // 4. Rename intermediate-extracted to the original partition dir
+ // 4. Rename intermediate-extracted to the original partitions dir
// 5. Change the metadata
- // 6. Delete the archived partition files in intermediate-archive
+ // 6. Delete the archived partitions files in intermediate-archive
if (!pathExists(intermediateExtractedDir) &&
- !pathExists(intermediateArchiveDir)) {
+ !pathExists(intermediateArchivedDir)) {
try {
// Copy the files out of the archive into the temporary directory
String copySource = sourceDir.toString();
- String copyDest = tmpDir.toString();
+ String copyDest = tmpPath.toString();
List args = new ArrayList();
args.add("-cp");
args.add(copySource);
@@ -1391,12 +1520,12 @@
throw new HiveException("Error while copying files from archive");
}
- console.printInfo("Moving " + tmpDir + " to " + intermediateExtractedDir);
+ console.printInfo("Moving " + tmpPath + " to " + intermediateExtractedDir);
if (fs.exists(intermediateExtractedDir)) {
throw new HiveException("Invalid state: the intermediate extracted " +
"directory already exists.");
}
- fs.rename(tmpDir, intermediateExtractedDir);
+ fs.rename(tmpPath, intermediateExtractedDir);
} catch (Exception e) {
throw new HiveException(e);
}
@@ -1405,15 +1534,15 @@
// At this point, we know that the extracted files are in the intermediate
// extracted dir, or in the the original directory.
- if (!pathExists(intermediateArchiveDir)) {
+ if (!pathExists(intermediateArchivedDir)) {
try {
- console.printInfo("Moving " + originalLocation + " to " + intermediateArchiveDir);
- fs.rename(originalLocation, intermediateArchiveDir);
+ console.printInfo("Moving " + originalDir + " to " + intermediateArchivedDir);
+ fs.rename(originalDir, intermediateArchivedDir);
} catch (IOException e) {
throw new HiveException(e);
}
} else {
- console.printInfo(intermediateArchiveDir + " already exists. " +
+ console.printInfo(intermediateArchivedDir + " already exists. " +
"Assuming it contains the archived version of the partition");
}
@@ -1423,28 +1552,39 @@
// If the original location exists here, then it must be the extracted files
// because in the previous step, we moved the previous original location
// (containing the archived version of the files) to intermediateArchiveDir
- if (!pathExists(originalLocation)) {
+ if (!pathExists(originalDir)) {
try {
- console.printInfo("Moving " + intermediateExtractedDir + " to " + originalLocation);
- fs.rename(intermediateExtractedDir, originalLocation);
+ console.printInfo("Moving " + intermediateExtractedDir + " to " + originalDir);
+ fs.rename(intermediateExtractedDir, originalDir);
} catch (IOException e) {
throw new HiveException(e);
}
} else {
- console.printInfo(originalLocation + " already exists. " +
+ console.printInfo(originalDir + " already exists. " +
"Assuming it contains the extracted files in the partition");
}
- setUnArchived(p);
- try {
- db.alterPartition(tblName, p);
- } catch (InvalidOperationException e) {
- throw new HiveException(e);
+ for(Partition p: partitions) {
+ setUnArchived(p);
+ try {
+ db.alterPartition(tblName, p);
+ } catch (InvalidOperationException e) {
+ throw new HiveException(e);
+ }
}
+
// If a failure happens here, the intermediate archive files won't be
// deleted. The user will need to call unarchive again to clear those up.
- deleteDir(intermediateArchiveDir);
+ if(pathExists(intermediateArchivedDir)) {
+ deleteDir(intermediateArchivedDir);
+ }
+ updateContainsArchived(db, tbl);
+
+ if(recovery) {
+ console.printInfo("Recovery after UNARCHIVE succeeded");
+ }
+
return 0;
}
@@ -2840,9 +2980,9 @@
} else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ALTERLOCATION) {
String newLocation = alterTbl.getNewLocation();
try {
- URI locURI = new URI(newLocation);
- if (!locURI.isAbsolute() || locURI.getScheme() == null
- || locURI.getScheme().trim().equals("")) {
+ URI locUri = new URI(newLocation);
+ if (!locUri.isAbsolute() || locUri.getScheme() == null
+ || locUri.getScheme().trim().equals("")) {
throw new HiveException(
newLocation
+ " is not absolute or has no scheme information. "
@@ -2851,7 +2991,7 @@
if (part != null) {
part.setLocation(newLocation);
} else {
- tbl.setDataLocation(locURI);
+ tbl.setDataLocation(locUri);
}
} catch (URISyntaxException e) {
throw new HiveException(e);
@@ -2983,14 +3123,47 @@
List partsToDelete = new ArrayList();
for (Map partSpec : dropTbl.getPartSpecs()) {
List partitions = db.getPartitions(tbl, partSpec);
+ // calculated archivingLevel is to be minimal of partitions' archiving
+ // levels if there are any and -1 otherwise
+ int archivingLevel = -1;
for (Partition p : partitions) {
if (!p.canDrop()) {
throw new HiveException("Table " + tbl.getTableName() +
" Partition " + p.getName() +
" is protected from being dropped");
+ } else if(ArchiveUtils.isArchived(p)) {
+ if(archivingLevel == -1 ||
+ ArchiveUtils.getArchivingLevel(p) < archivingLevel) {
+ archivingLevel = ArchiveUtils.getArchivingLevel(p);
+ }
}
- partsToDelete.add(p);
}
+ if(archivingLevel != -1) {
+ // dropped archived partition
+ int tail = 0; // how many unbound partition keys are at the end of spec
+ for(FieldSchema fs: tbl.getPartCols()) {
+ if(partSpec.get(fs.getName()) == null) {
+ tail += 1;
+ } else {
+ tail = 0;
+ }
+ }
+
+ // let's say table is partitioned by ds, hr, min and user
+ // requested drop WHERE ds='1' AND hr='2'
+ // then tail = 1 and tbl.getPartCols().size() - tail = 2
+ // this difference is always number of last set column
+ // if it's bigger that minimal archiving level it means we are
+ // choosing some partitions of archive with smaller level
+ // which is kind of wrong and impossible with current implementation
+ if(tbl.getPartCols().size() - tail > archivingLevel) {
+ throw new HiveException("Cannot drop a subset of partitions in an archive");
+ } else if(tbl.getPartCols().size() > archivingLevel) {
+ throw new HiveException("Dropping archives containing many partitions" +
+ "not yet implemented");
+ }
+ updateContainsArchived(db, tbl);
+ }
}
// drop all existing partitions from the list
Index: ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java (revision 1153271)
+++ ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java (working copy)
@@ -603,7 +603,7 @@
}
if (partn == null) {
- return new HiveLockObject(new DummyPartition(tab, path), data);
+ return new HiveLockObject(new DummyPartition(tab, path, partSpec), data);
}
return new HiveLockObject(partn, data);
Index: ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (revision 1153271)
+++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (working copy)
@@ -27,9 +27,9 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
+import java.util.Map.Entry;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
@@ -57,6 +57,7 @@
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.MapRedTask;
+import org.apache.hadoop.hive.ql.exec.ArchiveUtils;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorFactory;
import org.apache.hadoop.hive.ql.exec.RecordReader;
@@ -92,7 +93,6 @@
import org.apache.hadoop.hive.ql.optimizer.GenMRFileSink1;
import org.apache.hadoop.hive.ql.optimizer.GenMROperator;
import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext;
-import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx;
import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink1;
import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink2;
import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink3;
@@ -102,10 +102,14 @@
import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils;
import org.apache.hadoop.hive.ql.optimizer.MapJoinFactory;
import org.apache.hadoop.hive.ql.optimizer.Optimizer;
+import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx;
import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext;
import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalOptimizer;
import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeCreateCommonVars;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.RowFormatParams;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.StorageFormat;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec.SpecType;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
@@ -122,7 +126,6 @@
import org.apache.hadoop.hive.ql.plan.FetchWork;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.FilterDesc;
-import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc;
import org.apache.hadoop.hive.ql.plan.ForwardDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
@@ -145,12 +148,13 @@
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
import org.apache.hadoop.hive.ql.plan.UDTFDesc;
import org.apache.hadoop.hive.ql.plan.UnionDesc;
+import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.ResourceType;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
import org.apache.hadoop.hive.serde.Constants;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
@@ -158,9 +162,9 @@
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
@@ -3773,7 +3777,9 @@
ppath = ppath.substring(0, ppath.length()-1);
DummyPartition p =
new DummyPartition(dest_tab,
- dest_tab.getDbName() + "@" + dest_tab.getTableName() + "@" + ppath);
+ dest_tab.getDbName() + "@"
+ + dest_tab.getTableName() + "@" + ppath,
+ partSpec);
outputs.add(new WriteEntity(p, false));
} catch (HiveException e) {
throw new SemanticException(e.getMessage());
@@ -3795,10 +3801,6 @@
dest_path = new Path(tabPath.toUri().getScheme(), tabPath.toUri()
.getAuthority(), partPath.toUri().getPath());
- if ("har".equalsIgnoreCase(dest_path.toUri().getScheme())) {
- throw new SemanticException(ErrorMsg.OVERWRITE_ARCHIVED_PART
- .getMsg(qb.getParseInfo().getDestForClause(dest)));
- }
queryTmpdir = ctx.getExternalTmpFileURI(dest_path.toUri());
table_desc = Utilities.getTableDesc(dest_tab);
@@ -7442,6 +7444,7 @@
@Override
public void validate() throws SemanticException {
+ LOG.debug("validation start");
// Validate inputs and outputs have right protectmode to execute the query
for (ReadEntity readEntity: getInputs()) {
ReadEntity.Type type = readEntity.getType();
@@ -7474,8 +7477,30 @@
for (WriteEntity writeEntity: getOutputs()) {
WriteEntity.Type type = writeEntity.getType();
+
+ if(type == WriteEntity.Type.PARTITION || type == WriteEntity.Type.DUMMYPARTITION) {
+ String conflictingArchive;
+ try {
+ Partition usedp = writeEntity.getPartition();
+ Table tbl = usedp.getTable();
+
+ LOG.debug("validated " + usedp.getName());
+ LOG.debug(usedp.getTable());
+ conflictingArchive = ArchiveUtils
+ .conflictingArchiveNameOrNull(db, tbl, usedp.getSpec());
+ } catch (HiveException e) {
+ throw new SemanticException(e);
+ }
+ if(conflictingArchive != null) {
+ String message = String.format("Insert conflict with existing archive: %s",
+ conflictingArchive);
+ throw new SemanticException(message);
+ }
+ }
+
if (type != WriteEntity.Type.TABLE &&
type != WriteEntity.Type.PARTITION) {
+ LOG.debug("not validating writeEntity, because entity is neither table nor partition");
continue;
}
@@ -7507,8 +7532,10 @@
" Table " + tbl.getTableName() +
" Partition " + p.getName()));
}
+
}
else {
+ LOG.debug("Not a partition.");
tbl = writeEntity.getTable();
}
Index: ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (revision 1153271)
+++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (working copy)
@@ -51,6 +51,7 @@
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.exec.ArchiveUtils;
import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
@@ -1158,7 +1159,7 @@
}
bucketCols = part.getBucketCols();
inputFormatClass = part.getInputFormatClass();
- isArchived = Utilities.isArchived(part);
+ isArchived = ArchiveUtils.isArchived(part);
tblPartLoc = part.getDataLocation().toString();
}
} else {
@@ -1732,7 +1733,7 @@
// configured not to fail silently
boolean throwException =
!ifExists && !HiveConf.getBoolVar(conf, ConfVars.DROPIGNORESNONEXISTENT);
- addTablePartsOutputs(tblName, partSpecs, throwException, ast);
+ addTablePartsOutputs(tblName, partSpecs, throwException, false, ast);
}
rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),
@@ -1927,15 +1928,16 @@
// partition name to value
List