diff --git accumulo-handler/src/test/results/positive/accumulo_queries.q.out accumulo-handler/src/test/results/positive/accumulo_queries.q.out index c5379c7348..32a6e44d20 100644 --- accumulo-handler/src/test/results/positive/accumulo_queries.q.out +++ accumulo-handler/src/test/results/positive/accumulo_queries.q.out @@ -51,12 +51,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - Alter Table Operator: - Alter Table - type: drop props - old name: default.accumulo_table_1 - properties: - COLUMN_STATS_ACCURATE + Unset Properties + properties: + COLUMN_STATS_ACCURATE + table name: default.accumulo_table_1 Stage: Stage-1 Pre-Insert task @@ -534,12 +532,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - Alter Table Operator: - Alter Table - type: drop props - old name: default.accumulo_table_3 - properties: - COLUMN_STATS_ACCURATE + Unset Properties + properties: + COLUMN_STATS_ACCURATE + table name: default.accumulo_table_3 Stage: Stage-1 Pre-Insert task diff --git accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out index 1a5dde0602..12343d5a5f 100644 --- accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out +++ accumulo-handler/src/test/results/positive/accumulo_single_sourced_multi_insert.q.out @@ -54,12 +54,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - Alter Table Operator: - Alter Table - type: drop props - old name: default.src_x2 - properties: - COLUMN_STATS_ACCURATE + Unset Properties + properties: + COLUMN_STATS_ACCURATE + table name: default.src_x2 Stage: Stage-4 Stats Work diff --git druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java index 254d0a39a6..6e707e2841 100644 --- druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java +++ druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java @@ -137,7 +137,8 @@ private static final HttpClient HTTP_CLIENT; - private static final List ALLOWED_ALTER_TYPES = ImmutableList.of("ADDPROPS", "DROPPROPS", "ADD_COLUMNS"); + private static final List ALLOWED_ALTER_TYPES = + ImmutableList.of("SET_PROPERTIES", "UNSET_PROPERTIES", "ADD_COLUMNS"); static { final Lifecycle lifecycle = new Lifecycle(); diff --git hbase-handler/src/test/results/negative/hbase_ddl.q.out hbase-handler/src/test/results/negative/hbase_ddl.q.out index 4646def667..b577873519 100644 --- hbase-handler/src/test/results/negative/hbase_ddl.q.out +++ hbase-handler/src/test/results/negative/hbase_ddl.q.out @@ -26,4 +26,4 @@ key int It is a column key value string It is the column string value #### A masked pattern was here #### -FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [ADDPROPS, DROPPROPS, ADD_COLUMNS] to a non-native table hbase_table_1 +FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [SET_PROPERTIES, UNSET_PROPERTIES, ADD_COLUMNS] to a non-native table hbase_table_1 diff --git hbase-handler/src/test/results/positive/hbase_ddl.q.out hbase-handler/src/test/results/positive/hbase_ddl.q.out index 779ca4d16a..db3ef9ee3b 100644 --- hbase-handler/src/test/results/positive/hbase_ddl.q.out +++ hbase-handler/src/test/results/positive/hbase_ddl.q.out @@ -49,12 +49,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - Alter Table Operator: - Alter Table - type: drop props - old name: default.hbase_table_1 - properties: - COLUMN_STATS_ACCURATE + Unset Properties + properties: + COLUMN_STATS_ACCURATE + table name: default.hbase_table_1 Stage: Stage-1 Pre-Insert task diff --git hbase-handler/src/test/results/positive/hbase_queries.q.out hbase-handler/src/test/results/positive/hbase_queries.q.out index adf8864363..0c21d6d748 100644 --- hbase-handler/src/test/results/positive/hbase_queries.q.out +++ hbase-handler/src/test/results/positive/hbase_queries.q.out @@ -49,12 +49,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - Alter Table Operator: - Alter Table - type: drop props - old name: default.hbase_table_1 - properties: - COLUMN_STATS_ACCURATE + Unset Properties + properties: + COLUMN_STATS_ACCURATE + table name: default.hbase_table_1 Stage: Stage-1 Pre-Insert task @@ -534,12 +532,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - Alter Table Operator: - Alter Table - type: drop props - old name: default.hbase_table_3 - properties: - COLUMN_STATS_ACCURATE + Unset Properties + properties: + COLUMN_STATS_ACCURATE + table name: default.hbase_table_3 Stage: Stage-1 Pre-Insert task diff --git hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out index d474b4d065..e3d0b9ddb9 100644 --- hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out +++ hbase-handler/src/test/results/positive/hbase_single_sourced_multi_insert.q.out @@ -54,12 +54,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - Alter Table Operator: - Alter Table - type: drop props - old name: default.src_x2 - properties: - COLUMN_STATS_ACCURATE + Unset Properties + properties: + COLUMN_STATS_ACCURATE + table name: default.src_x2 Stage: Stage-4 Stats Work diff --git hbase-handler/src/test/results/positive/hbasestats.q.out hbase-handler/src/test/results/positive/hbasestats.q.out index 367b479556..7b1f152e34 100644 --- hbase-handler/src/test/results/positive/hbasestats.q.out +++ hbase-handler/src/test/results/positive/hbasestats.q.out @@ -76,12 +76,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - Alter Table Operator: - Alter Table - type: drop props - old name: default.users - properties: - COLUMN_STATS_ACCURATE + Unset Properties + properties: + COLUMN_STATS_ACCURATE + table name: default.users Stage: Stage-1 Pre-Insert task diff --git ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 554df3c6bf..91ecb414fc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -23,9 +23,9 @@ import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.UnresolvedPathException; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.parse.ASTNode; import org.apache.hadoop.hive.ql.parse.ASTNodeOrigin; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.security.AccessControlException; import java.io.FileNotFoundException; @@ -214,7 +214,8 @@ ALTER_COMMAND_FOR_VIEWS(10131, "To alter a view you need to use the ALTER VIEW command."), ALTER_COMMAND_FOR_TABLES(10132, "To alter a base table you need to use the ALTER TABLE command."), ALTER_VIEW_DISALLOWED_OP(10133, "Cannot use this form of ALTER on a view"), - ALTER_TABLE_NON_NATIVE(10134, "ALTER TABLE can only be used for " + AlterTableTypes.nonNativeTableAllowedTypes + " to a non-native table "), + ALTER_TABLE_NON_NATIVE(10134, "ALTER TABLE can only be used for " + AlterTableTypes.NON_NATIVE_TABLE_ALLOWED + + " to a non-native table "), SORTMERGE_MAPJOIN_FAILED(10135, "Sort merge bucketed join could not be performed. " + "If you really want to perform the operation, either set " + diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableDesc.java index 432779b3f4..963a7bc275 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableDesc.java @@ -21,12 +21,12 @@ import java.io.Serializable; import java.util.Map; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.ql.ddl.DDLDesc; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.Explain; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.DDLDesc.DDLDescWithWriteId; import org.apache.hadoop.hive.ql.plan.Explain.Level; @@ -50,7 +50,7 @@ public AbstractAlterTableDesc(AlterTableTypes type, String tableName, Map props) throws SemanticException { this.type = type; - this.tableName = String.join(".", Utilities.getDbTableName(tableName)); + this.tableName = tableName.contains(".") ? tableName : String.join(".", Utilities.getDbTableName(tableName)); this.partitionSpec = partitionSpec; this.replicationSpec = replicationSpec; this.isCascade = isCascade; @@ -86,11 +86,15 @@ public boolean expectView() { return expectView; } - @Explain(displayName = "props", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + @Explain(displayName = "properties", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) public Map getProps() { return props; } + public EnvironmentContext getEnvironmentContext() { + return null; + }; + @Override public String getFullTableName() { return tableName; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableOperation.java index baf98da37a..221510d603 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableOperation.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableOperation.java @@ -42,7 +42,6 @@ import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.session.SessionState; /** @@ -74,7 +73,7 @@ public int execute() throws HiveException { // Don't change the table object returned by the metastore, as we'll mess with it's caches. Table table = oldTable.copy(); - environmentContext = initializeEnvironmentContext(null); + environmentContext = initializeEnvironmentContext(desc.getEnvironmentContext()); if (partitions == null) { doAlteration(table, null); @@ -154,11 +153,11 @@ public void finalizeAlterTableWithWriteIdOp(Table table, Table oldTable, List props = alterTable.getProps(); diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableWithConstraintsDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableWithConstraintsDesc.java index 9babf2a1a9..aa953a9430 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableWithConstraintsDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AbstractAlterTableWithConstraintsDesc.java @@ -23,7 +23,6 @@ import org.apache.hadoop.hive.ql.ddl.table.constaint.Constraints; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; /** * Abstract ancestor of all ALTER TABLE descriptors that are handled by the AlterTableWithWriteIdOperations framework diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableTypes.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableTypes.java new file mode 100644 index 0000000000..a64a63d01f --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableTypes.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table; + +import java.util.List; +import java.util.Set; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +/** + * Enumeration of alter table command types. + */ +public enum AlterTableTypes { + // column + ADD_COLUMNS("add columns"), + REPLACE_COLUMNS("replace columns"), + RENAME_COLUMN("rename column"), + UPDATE_COLUMNS("update columns"), + // partition + ADDPARTITION("add partition"), + DROPPARTITION("drop partition"), + RENAMEPARTITION("rename partition"), // Note: used in RenamePartitionDesc, not here. + ALTERPARTITION("alter partition"), // Note: this is never used in AlterTableDesc. + // constraint + ADD_CONSTRAINT("add constraint"), + DROP_CONSTRAINT("drop constraint"), + // storage + SET_SERDE("set serde"), + SET_SERDE_PROPS("set serde props"), + SET_FILE_FORMAT("add fileformat"), + CLUSTERED_BY("clustered by"), + NOT_SORTED("not sorted"), + NOT_CLUSTERED("not clustered"), + SET_LOCATION("set location"), + SKEWED_BY("skewed by"), + NOT_SKEWED("not skewed"), + SET_SKEWED_LOCATION("alter skew location"), + INTO_BUCKETS("alter bucket number"), + // misc + SET_PROPERTIES("set properties"), + UNSET_PROPERTIES("unset properties"), + TOUCH("touch"), + RENAME("rename"), + OWNER("set owner"), + ARCHIVE("archieve"), + UNARCHIVE("unarchieve"), + COMPACT("compact"), + TRUNCATE("truncate"), + MERGEFILES("merge files"), + UPDATESTATS("update stats"); // Note: used in ColumnStatsUpdateWork, not here. + + private final String name; + + AlterTableTypes(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public static final List NON_NATIVE_TABLE_ALLOWED = + ImmutableList.of(SET_PROPERTIES, UNSET_PROPERTIES, ADD_COLUMNS); + + public static final Set SUPPORT_PARTIAL_PARTITION_SPEC = + ImmutableSet.of(ADD_COLUMNS, REPLACE_COLUMNS, RENAME_COLUMN, SET_PROPERTIES, UNSET_PROPERTIES, SET_SERDE, + SET_SERDE_PROPS, SET_FILE_FORMAT); +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableAddColumnsDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableAddColumnsDesc.java index e40ba1819d..57d01c4ac6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableAddColumnsDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableAddColumnsDesc.java @@ -23,9 +23,9 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableChangeColumnDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableChangeColumnDesc.java index ce3b97eb68..a9bca6005e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableChangeColumnDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableChangeColumnDesc.java @@ -21,9 +21,9 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableWithConstraintsDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.ddl.table.constaint.Constraints; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.DDLDesc.DDLDescWithWriteId; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableReplaceColumnsDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableReplaceColumnsDesc.java index 3975f6682a..32c21150bc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableReplaceColumnsDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableReplaceColumnsDesc.java @@ -23,9 +23,9 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.DDLDesc.DDLDescWithWriteId; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableUpdateColumnsDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableUpdateColumnsDesc.java index 18485c9a81..50028f5d8a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableUpdateColumnsDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/AlterTableUpdateColumnsDesc.java @@ -22,8 +22,8 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.DDLDesc.DDLDescWithWriteId; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constaint/AlterTableAddConstraintDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constaint/AlterTableAddConstraintDesc.java index 2077c7d7e6..825768a184 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constaint/AlterTableAddConstraintDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/constaint/AlterTableAddConstraintDesc.java @@ -19,9 +19,9 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableWithConstraintsDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableRenameDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableRenameDesc.java new file mode 100644 index 0000000000..7906f8299d --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableRenameDesc.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.misc; + +import org.apache.hadoop.hive.ql.ddl.DDLTask2; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; +import org.apache.hadoop.hive.ql.parse.ReplicationSpec; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.Explain.Level; + +/** + * DDL task description for ALTER TABLE ... RENAME TO ... commands. + */ +@Explain(displayName = "Rename Table", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +public class AlterTableRenameDesc extends AbstractAlterTableDesc { + private static final long serialVersionUID = 1L; + + static { + DDLTask2.registerOperation(AlterTableRenameDesc.class, AlterTableRenameOperation.class); + } + + private final String newName; + + public AlterTableRenameDesc(String tableName, ReplicationSpec replicationSpec, boolean expectView, String newName) + throws SemanticException { + super(AlterTableTypes.RENAME, tableName, null, replicationSpec, false, expectView, null); + this.newName = newName; + } + + @Explain(displayName = "new table name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public String getNewName() { + return newName; + } + + @Override + public boolean mayNeedWriteId() { + return true; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableRenameOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableRenameOperation.java new file mode 100644 index 0000000000..4345c9e6f5 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableRenameOperation.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.misc; + +import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableOperation; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.repl.dump.Utils; + +/** + * Operation process of renaming a table. + */ +public class AlterTableRenameOperation extends AbstractAlterTableOperation { + private final AlterTableRenameDesc desc; + + public AlterTableRenameOperation(DDLOperationContext context, AlterTableRenameDesc desc) { + super(context, desc); + this.desc = desc; + } + + @Override + public int execute() throws HiveException { + String[] names = Utilities.getDbTableName(desc.getTableName()); + if (Utils.isBootstrapDumpInProgress(context.getDb(), names[0])) { + LOG.error("DDLTask: Rename Table not allowed as bootstrap dump in progress"); + throw new HiveException("Rename Table: Not allowed as bootstrap dump in progress"); + } + + return super.execute(); + } + + @Override + protected void doAlteration(Table table, Partition partition) throws HiveException { + table.setDbName(Utilities.getDatabaseName(desc.getNewName())); + table.setTableName(Utilities.getTableName(desc.getNewName())); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetOwnerDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetOwnerDesc.java new file mode 100644 index 0000000000..191626f967 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetOwnerDesc.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.misc; + +import org.apache.hadoop.hive.ql.ddl.DDLTask2; +import org.apache.hadoop.hive.ql.ddl.privilege.PrincipalDesc; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.Explain.Level; + +/** + * DDL task description for ALTER TABLE ... SET OWNER ... commands. + */ +@Explain(displayName = "Set Owner", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +public class AlterTableSetOwnerDesc extends AbstractAlterTableDesc { + private static final long serialVersionUID = 1L; + + static { + DDLTask2.registerOperation(AlterTableSetOwnerDesc.class, AlterTableSetOwnerOperation.class); + } + + private final PrincipalDesc ownerPrincipal; + + public AlterTableSetOwnerDesc(String tableName, PrincipalDesc ownerPrincipal) throws SemanticException { + super(AlterTableTypes.OWNER, tableName, null, null, false, false, null); + this.ownerPrincipal = ownerPrincipal; + } + + @Explain(displayName = "owner principal", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public PrincipalDesc getOwnerPrincipal() { + return ownerPrincipal; + } + + @Override + public boolean mayNeedWriteId() { + return false; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetOwnerOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetOwnerOperation.java new file mode 100644 index 0000000000..eb7be24bba --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetOwnerOperation.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.misc; + +import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableOperation; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; + +/** + * Operation process of setting the owner of a table. + */ +public class AlterTableSetOwnerOperation extends AbstractAlterTableOperation { + private final AlterTableSetOwnerDesc desc; + + public AlterTableSetOwnerOperation(DDLOperationContext context, AlterTableSetOwnerDesc desc) { + super(context, desc); + this.desc = desc; + } + + @Override + protected void doAlteration(Table table, Partition partition) throws HiveException { + if (desc.getOwnerPrincipal() != null) { + table.setOwner(desc.getOwnerPrincipal().getName()); + table.setOwnerType(desc.getOwnerPrincipal().getType()); + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetPropertiesDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetPropertiesDesc.java new file mode 100644 index 0000000000..cfdffb73de --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetPropertiesDesc.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.misc; + +import java.util.Map; + +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.ql.ddl.DDLTask2; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; +import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.parse.ReplicationSpec; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.Explain.Level; + +/** + * DDL task description for ALTER TABLE ... SET TBLPROPERTIES ... commands. + */ +@Explain(displayName = "Set Properties", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +public class AlterTableSetPropertiesDesc extends AbstractAlterTableDesc { + private static final long serialVersionUID = 1L; + + static { + DDLTask2.registerOperation(AlterTableSetPropertiesDesc.class, AlterTableSetPropertiesOperation.class); + } + + private final boolean isExplicitStatsUpdate; + private final boolean isFullAcidConversion; + private final EnvironmentContext environmentContext; + + public AlterTableSetPropertiesDesc(String tableName, Map partitionSpec, + ReplicationSpec replicationSpec, boolean expectView, Map props, boolean isExplicitStatsUpdate, + boolean isFullAcidConversion, EnvironmentContext environmentContext) throws SemanticException { + super(AlterTableTypes.SET_PROPERTIES, tableName, partitionSpec, replicationSpec, false, expectView, props); + this.isExplicitStatsUpdate = isExplicitStatsUpdate; + this.isFullAcidConversion = isFullAcidConversion; + this.environmentContext = environmentContext; + } + + @Override + public EnvironmentContext getEnvironmentContext() { + return environmentContext; + } + + @Override + public boolean mayNeedWriteId() { + return isExplicitStatsUpdate || AcidUtils.isToInsertOnlyTable(null, getProps()) || + (AcidUtils.isTransactionalTable(getProps()) && !isFullAcidConversion); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetPropertiesOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetPropertiesOperation.java new file mode 100644 index 0000000000..b0a763fc7b --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableSetPropertiesOperation.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.misc; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang.BooleanUtils; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableOperation; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.PartitionIterable; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.plan.LoadMultiFilesDesc; +import org.apache.hadoop.hive.ql.plan.MoveWork; + +import com.google.common.collect.Lists; + +/** + * Operation process of setting properties of a table. + */ +public class AlterTableSetPropertiesOperation extends AbstractAlterTableOperation { + private final AlterTableSetPropertiesDesc desc; + + public AlterTableSetPropertiesOperation(DDLOperationContext context, AlterTableSetPropertiesDesc desc) { + super(context, desc); + this.desc = desc; + } + + @Override + protected void doAlteration(Table table, Partition partition) throws HiveException { + if (StatsSetupConst.USER.equals(environmentContext.getProperties().get(StatsSetupConst.STATS_GENERATED))) { + environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS); + } + + if (partition != null) { + partition.getTPartition().getParameters().putAll(desc.getProps()); + } else { + boolean isFromMmTable = AcidUtils.isInsertOnlyTable(table.getParameters()); + Boolean isToMmTable = AcidUtils.isToInsertOnlyTable(table, desc.getProps()); + if (!isFromMmTable && BooleanUtils.isTrue(isToMmTable)) { + if (!HiveConf.getBoolVar(context.getConf(), ConfVars.HIVE_MM_ALLOW_ORIGINALS)) { + List> mmTasks = generateAddMmTasks(table, desc.getWriteId()); + for (Task mmTask : mmTasks) { + context.getTask().addDependentTask(mmTask); + } + } else { + if (!table.getPartitionKeys().isEmpty()) { + PartitionIterable parts = new PartitionIterable(context.getDb(), table, null, + HiveConf.getIntVar(context.getConf(), ConfVars.METASTORE_BATCH_RETRIEVE_MAX)); + for (Partition part : parts) { + checkMmLb(part); + } + } else { + checkMmLb(table); + } + } + } else if (isFromMmTable && BooleanUtils.isFalse(isToMmTable)) { + throw new HiveException("Cannot convert an ACID table to non-ACID"); + } + + // Converting to/from external table + String externalProp = desc.getProps().get("EXTERNAL"); + if (externalProp != null) { + if (Boolean.parseBoolean(externalProp) && table.getTableType() == TableType.MANAGED_TABLE) { + table.setTableType(TableType.EXTERNAL_TABLE); + } else if (!Boolean.parseBoolean(externalProp) && table.getTableType() == TableType.EXTERNAL_TABLE) { + table.setTableType(TableType.MANAGED_TABLE); + } + } + + table.getTTable().getParameters().putAll(desc.getProps()); + } + } + + + private List> generateAddMmTasks(Table table, Long writeId) throws HiveException { + // We will move all the files in the table/partition directories into the first MM + // directory, then commit the first write ID. + if (writeId == null) { + throw new HiveException("Internal error - write ID not set for MM conversion"); + } + + List sources = new ArrayList<>(); + List targets = new ArrayList<>(); + + int stmtId = 0; + String mmDir = AcidUtils.deltaSubdir(writeId, writeId, stmtId); + + if (!table.getPartitionKeys().isEmpty()) { + PartitionIterable parts = new PartitionIterable(context.getDb(), table, null, + HiveConf.getIntVar(context.getConf(), ConfVars.METASTORE_BATCH_RETRIEVE_MAX)); + for (Partition part : parts) { + checkMmLb(part); + Path source = part.getDataLocation(); + Path target = new Path(source, mmDir); + sources.add(source); + targets.add(target); + Utilities.FILE_OP_LOGGER.trace("Will move " + source + " to " + target); + } + } else { + checkMmLb(table); + Path source = table.getDataLocation(); + Path target = new Path(source, mmDir); + sources.add(source); + targets.add(target); + Utilities.FILE_OP_LOGGER.trace("Will move " + source + " to " + target); + } + + // Don't set inputs and outputs - the locks have already been taken so it's pointless. + MoveWork mw = new MoveWork(null, null, null, null, false); + mw.setMultiFilesDesc(new LoadMultiFilesDesc(sources, targets, true, null, null)); + return Lists.>newArrayList(TaskFactory.get(mw)); + } + + private void checkMmLb(Table table) throws HiveException { + if (!table.isStoredAsSubDirectories()) { + return; + } + // TODO [MM gap?]: by design; no-one seems to use LB tables. They will work, but not convert. + // It's possible to work around this by re-creating and re-inserting the table. + throw new HiveException("Converting list bucketed tables stored as subdirectories " + + " to MM is not supported. Please re-create a table in the desired format."); + } + + private void checkMmLb(Partition partition) throws HiveException { + if (!partition.isStoredAsSubDirectories()) { + return; + } + throw new HiveException("Converting list bucketed tables stored as subdirectories " + + " to MM is not supported. Please re-create a table in the desired format."); + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableTouchDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableTouchDesc.java new file mode 100644 index 0000000000..92f498d550 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableTouchDesc.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.misc; + +import java.util.Map; + +import org.apache.hadoop.hive.ql.ddl.DDLDesc; +import org.apache.hadoop.hive.ql.ddl.DDLTask2; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.Explain.Level; + +/** + * DDL task description for ALTER TABLE ... TOUCH [PARTITION ...] commands. + */ +@Explain(displayName = "Touch", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +public class AlterTableTouchDesc implements DDLDesc { + static { + DDLTask2.registerOperation(AlterTableTouchDesc.class, AlterTableTouchOperation.class); + } + + private final String tableName; + private Map partitionSpec; + + public AlterTableTouchDesc(String tableName, Map partitionSpec) { + this.tableName = tableName; + this.partitionSpec = partitionSpec; + } + + @Explain(displayName = "table name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public String getTableName() { + return tableName; + } + + @Explain(displayName = "partition spec", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public Map getPartitionSpec() { + return partitionSpec; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableTouchOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableTouchOperation.java new file mode 100644 index 0000000000..4180f622a3 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableTouchOperation.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.misc; + +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.ql.ddl.DDLOperation; +import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.ddl.DDLUtils; +import org.apache.hadoop.hive.ql.hooks.ReadEntity; +import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; + +/** + * Operation process of touching a table. + */ +public class AlterTableTouchOperation extends DDLOperation { + private final AlterTableTouchDesc desc; + + public AlterTableTouchOperation(DDLOperationContext context, AlterTableTouchDesc desc) { + super(context); + this.desc = desc; + } + + @Override + public int execute() throws HiveException { + // TODO: catalog + Table table = context.getDb().getTable(desc.getTableName()); + EnvironmentContext environmentContext = new EnvironmentContext(); + environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); + + if (desc.getPartitionSpec() == null) { + context.getDb().alterTable(table, false, environmentContext, true); + context.getWork().getInputs().add(new ReadEntity(table)); + DDLUtils.addIfAbsentByName(new WriteEntity(table, WriteEntity.WriteType.DDL_NO_LOCK), context); + } else { + Partition part = context.getDb().getPartition(table, desc.getPartitionSpec(), false); + if (part == null) { + throw new HiveException("Specified partition does not exist"); + } + try { + context.getDb().alterPartition(table.getCatalogName(), table.getDbName(), table.getTableName(), part, + environmentContext, true); + } catch (InvalidOperationException e) { + throw new HiveException(e); + } + context.getWork().getInputs().add(new ReadEntity(part)); + DDLUtils.addIfAbsentByName(new WriteEntity(part, WriteEntity.WriteType.DDL_NO_LOCK), context); + } + return 0; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableUnsetPropertiesDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableUnsetPropertiesDesc.java new file mode 100644 index 0000000000..eb70fb3fb8 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableUnsetPropertiesDesc.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.misc; + +import java.util.Map; + +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.ql.ddl.DDLTask2; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; +import org.apache.hadoop.hive.ql.parse.ReplicationSpec; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.Explain.Level; + +/** + * DDL task description for ALTER TABLE ... UNSET TBLPROPERTIES [IF EXISTS] ... commands. + */ +@Explain(displayName = "Unset Properties", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +public class AlterTableUnsetPropertiesDesc extends AbstractAlterTableDesc { + private static final long serialVersionUID = 1L; + + static { + DDLTask2.registerOperation(AlterTableUnsetPropertiesDesc.class, AlterTableUnsetPropertiesOperation.class); + } + + private final boolean isExplicitStatsUpdate; + private final EnvironmentContext environmentContext; + + public AlterTableUnsetPropertiesDesc(String tableName, Map partitionSpec, + ReplicationSpec replicationSpec, boolean expectView, Map props, boolean isExplicitStatsUpdate, + EnvironmentContext environmentContext) throws SemanticException { + super(AlterTableTypes.UNSET_PROPERTIES, tableName, partitionSpec, replicationSpec, false, expectView, props); + this.isExplicitStatsUpdate = isExplicitStatsUpdate; + this.environmentContext = environmentContext; + } + + @Override + public EnvironmentContext getEnvironmentContext() { + return environmentContext; + } + + @Override + public boolean mayNeedWriteId() { + return isExplicitStatsUpdate; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableUnsetPropertiesOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableUnsetPropertiesOperation.java new file mode 100644 index 0000000000..0e69a9a0d4 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/AlterTableUnsetPropertiesOperation.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.misc; + +import java.util.Set; + +import org.apache.hadoop.hive.common.StatsSetupConst; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableOperation; +import org.apache.hadoop.hive.ql.io.AcidUtils; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; + +/** + * Operation process of unsetting properties of a table. + */ +public class AlterTableUnsetPropertiesOperation extends AbstractAlterTableOperation { + private final AlterTableUnsetPropertiesDesc desc; + + public AlterTableUnsetPropertiesOperation(DDLOperationContext context, AlterTableUnsetPropertiesDesc desc) { + super(context, desc); + this.desc = desc; + } + + @Override + protected void doAlteration(Table table, Partition partition) throws HiveException { + if (StatsSetupConst.USER.equals(environmentContext.getProperties().get(StatsSetupConst.STATS_GENERATED))) { + // drop a stats parameter, which triggers recompute stats update automatically + environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS); + } + + if (partition == null) { + Set removedSet = desc.getProps().keySet(); + boolean isFromMmTable = AcidUtils.isInsertOnlyTable(table.getParameters()); + boolean isRemoved = AcidUtils.isRemovedInsertOnlyTable(removedSet); + if (isFromMmTable && isRemoved) { + throw new HiveException("Cannot convert an ACID table to non-ACID"); + } + + // Check if external table property being removed + if (removedSet.contains("EXTERNAL") && table.getTableType() == TableType.EXTERNAL_TABLE) { + table.setTableType(TableType.MANAGED_TABLE); + } + } + for (String key : desc.getProps().keySet()) { + if (partition != null) { + partition.getTPartition().getParameters().remove(key); + } else { + table.getTTable().getParameters().remove(key); + } + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveDesc.java new file mode 100644 index 0000000000..67aabd5f57 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveDesc.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.storage; + +import java.util.Map; + +import org.apache.hadoop.hive.ql.ddl.DDLDesc; +import org.apache.hadoop.hive.ql.ddl.DDLTask2; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.Explain.Level; + +/** + * DDL task description for ALTER TABLE ... ARCHIVE [PARTITION ...] commands. + */ +@Explain(displayName = "Archive", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +public class AlterTableArchiveDesc implements DDLDesc { + static { + DDLTask2.registerOperation(AlterTableArchiveDesc.class, AlterTableArchiveOperation.class); + } + + private final String tableName; + private final Map partitionSpec; + + public AlterTableArchiveDesc(String tableName, Map partitionSpec) { + this.tableName = tableName; + this.partitionSpec = partitionSpec; + } + + @Explain(displayName = "table name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public String getTableName() { + return tableName; + } + + @Explain(displayName = "partition spec", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public Map getPartitionSpec() { + return partitionSpec; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveOperation.java new file mode 100644 index 0000000000..4aea760f20 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveOperation.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.storage; + +import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.exec.ArchiveUtils; +import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.exec.ArchiveUtils.PartSpecInfo; + +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.ql.ddl.DDLOperation; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.tools.HadoopArchives; +import org.apache.hadoop.util.ToolRunner; + +import com.google.common.collect.ImmutableList; + +import static org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableArchiveUtils.ARCHIVE_NAME; + +/** + * Operation process of archiving a table. + */ +public class AlterTableArchiveOperation extends DDLOperation { + private final AlterTableArchiveDesc desc; + + public AlterTableArchiveOperation(DDLOperationContext context, AlterTableArchiveDesc desc) { + super(context); + this.desc = desc; + } + + @Override + public int execute() throws HiveException { + Table table = context.getDb().getTable(desc.getTableName()); + if (table.getTableType() != TableType.MANAGED_TABLE) { + throw new HiveException("ARCHIVE can only be performed on managed tables"); + } + + Map partitionSpec = desc.getPartitionSpec(); + PartSpecInfo partitionSpecInfo = PartSpecInfo.create(table, partitionSpec); + List partitions = context.getDb().getPartitions(table, partitionSpec); + + Path originalDir = getOriginalDir(table, partitionSpecInfo, partitions); + Path intermediateArchivedDir = AlterTableArchiveUtils.getInterMediateDir(originalDir, context.getConf(), + ConfVars.METASTORE_INT_ARCHIVED); + Path intermediateOriginalDir = AlterTableArchiveUtils.getInterMediateDir(originalDir, context.getConf(), + ConfVars.METASTORE_INT_ORIGINAL); + + context.getConsole().printInfo("intermediate.archived is " + intermediateArchivedDir.toString()); + context.getConsole().printInfo("intermediate.original is " + intermediateOriginalDir.toString()); + + checkIfAlreadyArchived(partitionSpecInfo, partitions); + boolean recovery = isRecovery(intermediateArchivedDir, intermediateOriginalDir); + + FileSystem fs = null; + try { + fs = originalDir.getFileSystem(context.getConf()); + } catch (IOException e) { + throw new HiveException(e); + } + + // The following steps seem roundabout, but they are meant to aid in recovery if a failure occurs and to keep a + // consistent state in the FS + + // If the intermediate directory exists, we assume the dir is good to use as it's creation is atomic (move) + if (!recovery) { + Path tmpPath = createArchiveInTmpDir(table, partitionSpecInfo, originalDir); + moveTmpDirToIntermediateDir(intermediateArchivedDir, fs, tmpPath); + } else { + if (AlterTableArchiveUtils.pathExists(intermediateArchivedDir, context.getConf())) { + context.getConsole().printInfo("Intermediate archive directory " + intermediateArchivedDir + + " already exists. Assuming it contains an archived version of the partition"); + } + } + + moveOriginalDirToIntermediateDir(originalDir, intermediateOriginalDir, fs); + + // If there's a failure from here to when the metadata is updated, there will be no data in the partition, or an + // error while trying to read the partition (if the archive files have been moved to the original partition + // directory.) But re-running the archive command will allow recovery + + moveIntermediateArchivedDirToOriginalParent(originalDir, intermediateArchivedDir, fs); + + writeArchivationToMetastore(partitionSpecInfo, partitions, originalDir); + + // If a failure occurs here, the directory containing the original files will not be deleted. The user will run + // ARCHIVE again to clear this up. The presence of these files are used to indicate whether the original partition + // directory contains archived or unarchived files. + + deleteIntermediateOriginalDir(table, intermediateOriginalDir); + + if (recovery) { + context.getConsole().printInfo("Recovery after ARCHIVE succeeded"); + } + + return 0; + } + + private Path getOriginalDir(Table table, PartSpecInfo partitionSpecInfo, List partitions) + throws HiveException { + // when we have partial partitions specification we must assume partitions lie in standard place - + // if they were in custom locations putting them into one archive would involve mass amount of copying + // in full partition specification case we allow custom locations to keep backward compatibility + if (partitions.isEmpty()) { + throw new HiveException("No partition matches the specification"); + } else if (partitionSpecInfo.values.size() != table.getPartCols().size()) { + // for partial specifications we need partitions to follow the scheme + for (Partition partition : partitions){ + if (AlterTableArchiveUtils.partitionInCustomLocation(table, partition)) { + throw new HiveException(String.format("ARCHIVE cannot run for partition groups with custom locations like %s", + partition.getLocation())); + } + } + return partitionSpecInfo.createPath(table); + } else { + Partition p = partitions.get(0); + // partition can be archived if during recovery + if (ArchiveUtils.isArchived(p)) { + return new Path(AlterTableArchiveUtils.getOriginalLocation(p)); + } else { + return p.getDataLocation(); + } + } + } + + private void checkIfAlreadyArchived(PartSpecInfo partitionSpecInfo, List partitions) throws HiveException { + // we checked if partitions matching specification are marked as archived in the metadata; if they are then + // throw an exception + for (Partition partition : partitions) { + if (ArchiveUtils.isArchived(partition)) { + if (ArchiveUtils.getArchivingLevel(partition) != partitionSpecInfo.values.size()) { + String name = ArchiveUtils.getPartialName(partition, ArchiveUtils.getArchivingLevel(partition)); + throw new HiveException(String.format("Conflict with existing archive %s", name)); + } else { + throw new HiveException("Partition(s) already archived"); + } + } + } + } + + private boolean isRecovery(Path intermediateArchivedDir, Path intermediateOriginalDir) throws HiveException { + if (AlterTableArchiveUtils.pathExists(intermediateArchivedDir, context.getConf()) || + AlterTableArchiveUtils.pathExists(intermediateOriginalDir, context.getConf())) { + context.getConsole().printInfo("Starting recovery after failed ARCHIVE"); + return true; + } + return false; + } + + private Path createArchiveInTmpDir(Table table, PartSpecInfo partitionSpecInfo, Path originalDir) + throws HiveException { + // First create the archive in a tmp dir so that if the job fails, the bad files don't pollute the filesystem + Path tmpPath = new Path(context.getDriverContext().getCtx().getExternalTmpPath(originalDir), "partlevel"); + + // Create the Hadoop archive + context.getConsole().printInfo("Creating " + ARCHIVE_NAME + " for " + originalDir.toString() + " in " + tmpPath); + context.getConsole().printInfo("Please wait... (this may take a while)"); + try { + int maxJobNameLength = context.getConf().getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH); + String jobName = String.format("Archiving %s@%s", table.getTableName(), partitionSpecInfo.getName()); + jobName = Utilities.abbreviate(jobName, maxJobNameLength - 6); + context.getConf().set(MRJobConfig.JOB_NAME, jobName); + + HadoopArchives har = new HadoopArchives(context.getConf()); + List args = ImmutableList.of("-archiveName", ARCHIVE_NAME, "-p", originalDir.toString(), + tmpPath.toString()); + + int ret = ToolRunner.run(har, args.toArray(new String[0])); + if (ret != 0) { + throw new HiveException("Error while creating HAR"); + } + } catch (Exception e) { + throw new HiveException(e); + } + + return tmpPath; + } + + /** + * Move from the tmp dir to an intermediate directory, in the same level as the partition directory. + * e.g. .../hr=12-intermediate-archived + */ + private void moveTmpDirToIntermediateDir(Path intermediateArchivedDir, FileSystem fs, Path tmpPath) + throws HiveException { + try { + context.getConsole().printInfo("Moving " + tmpPath + " to " + intermediateArchivedDir); + if (AlterTableArchiveUtils.pathExists(intermediateArchivedDir, context.getConf())) { + throw new HiveException("The intermediate archive directory already exists."); + } + fs.rename(tmpPath, intermediateArchivedDir); + } catch (IOException e) { + throw new HiveException("Error while moving tmp directory"); + } + } + + /** + * Move the original parent directory to the intermediate original directory if the move hasn't been made already. + */ + private void moveOriginalDirToIntermediateDir(Path originalDir, Path intermediateOriginalDir, FileSystem fs) + throws HiveException { + // If we get to here, we know that we've archived the partition files, but they may be in the original partition + // location, or in the intermediate original dir. + if (!AlterTableArchiveUtils.pathExists(intermediateOriginalDir, context.getConf())) { + context.getConsole().printInfo("Moving " + originalDir + " to " + intermediateOriginalDir); + moveDir(fs, originalDir, intermediateOriginalDir); + } else { + context.getConsole().printInfo(intermediateOriginalDir + " already exists. " + + "Assuming it contains the original files in the partition"); + } + } + + /** + * Move the intermediate archived directory to the original parent directory. + */ + private void moveIntermediateArchivedDirToOriginalParent(Path originalDir, Path intermediateArchivedDir, + FileSystem fs) throws HiveException { + if (!AlterTableArchiveUtils.pathExists(originalDir, context.getConf())) { + context.getConsole().printInfo("Moving " + intermediateArchivedDir + " to " + originalDir); + moveDir(fs, intermediateArchivedDir, originalDir); + } else { + context.getConsole().printInfo(originalDir + " already exists. Assuming it contains the archived version of " + + "the partition"); + } + } + + private void moveDir(FileSystem fs, Path from, Path to) throws HiveException { + try { + if (!fs.rename(from, to)) { + throw new HiveException("Moving " + from + " to " + to + " failed!"); + } + } catch (IOException e) { + throw new HiveException(e); + } + } + + /** + * Record the previous changes in the metastore. + */ + private void writeArchivationToMetastore(PartSpecInfo partitionSpecInfo, List partitions, Path originalDir) + throws HiveException { + try { + URI archiveUri = new Path(originalDir, ARCHIVE_NAME).toUri(); + URI originalUri = ArchiveUtils.addSlash(originalDir.toUri()); + ArchiveUtils.HarPathHelper harHelper = new ArchiveUtils.HarPathHelper(context.getConf(), archiveUri, originalUri); + + for (Partition partition : partitions) { + URI originalPartitionUri = ArchiveUtils.addSlash(partition.getDataLocation().toUri()); + URI harPartitionDir = harHelper.getHarUri(originalPartitionUri); + StringBuilder authority = new StringBuilder(); + if (harPartitionDir.getUserInfo() != null) { + authority.append(harPartitionDir.getUserInfo()).append("@"); + } + authority.append(harPartitionDir.getHost()); + if (harPartitionDir.getPort() != -1) { + authority.append(":").append(harPartitionDir.getPort()); + } + + // make in Path to ensure no slash at the end + Path harPath = new Path(harPartitionDir.getScheme(), authority.toString(), harPartitionDir.getPath()); + setArchived(partition, harPath, partitionSpecInfo.values.size()); + // TODO: catalog + context.getDb().alterPartition(desc.getTableName(), partition, null, true); + } + } catch (Exception e) { + throw new HiveException("Unable to change the partition info for HAR", e); + } + } + + /** + * Sets the appropriate attributes in the supplied Partition object to mark + * it as archived. Note that the metastore is not touched - a separate + * call to alter_partition is needed. + * + * @param p - the partition object to modify + * @param harPath - new location of partition (har schema URI) + */ + private void setArchived(Partition p, Path harPath, int level) { + assert(!ArchiveUtils.isArchived(p)); + AlterTableArchiveUtils.setIsArchived(p, true, level); + AlterTableArchiveUtils.setOriginalLocation(p, p.getLocation()); + p.setLocation(harPath.toString()); + } + + private void deleteIntermediateOriginalDir(Table table, Path intermediateOriginalDir) throws HiveException { + if (AlterTableArchiveUtils.pathExists(intermediateOriginalDir, context.getConf())) { + AlterTableArchiveUtils.deleteDir(intermediateOriginalDir, context.getDb().getDatabase(table.getDbName()), + context.getConf()); + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveUtils.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveUtils.java new file mode 100644 index 0000000000..1652602c97 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableArchiveUtils.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.storage; + +import java.io.IOException; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.ql.exec.ArchiveUtils; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; + +/** + * Utilities for archiving. + */ +final class AlterTableArchiveUtils { + private AlterTableArchiveUtils() { + throw new UnsupportedOperationException("ArchiveUtils should not be instantiated"); + } + + static final String ARCHIVE_NAME = "data.har"; + + /** + * Returns original partition of archived partition, null for unarchived one. + */ + static String getOriginalLocation(Partition partition) { + Map params = partition.getParameters(); + return params.get(hive_metastoreConstants.ORIGINAL_LOCATION); + } + + /** + * Sets original location of partition which is to be archived. + */ + static void setOriginalLocation(Partition partition, String loc) { + Map params = partition.getParameters(); + if (loc == null) { + params.remove(hive_metastoreConstants.ORIGINAL_LOCATION); + } else { + params.put(hive_metastoreConstants.ORIGINAL_LOCATION, loc); + } + } + + /** + * Checks in partition is in custom (not-standard) location. + * @param table - table in which partition is + * @param partition - partition + * @return true if partition location is custom, false if it is standard + */ + static boolean partitionInCustomLocation(Table table, Partition partition) throws HiveException { + String subdir = null; + try { + subdir = Warehouse.makePartName(table.getPartCols(), partition.getValues()); + } catch (MetaException e) { + throw new HiveException("Unable to get partition's directory", e); + } + + Path tableDir = table.getDataLocation(); + if (tableDir == null) { + throw new HiveException("Table has no location set"); + } + + String standardLocation = new Path(tableDir, subdir).toString(); + if (ArchiveUtils.isArchived(partition)) { + return !getOriginalLocation(partition).equals(standardLocation); + } else { + return !partition.getLocation().equals(standardLocation); + } + } + + static Path getInterMediateDir(Path dir, Configuration conf, ConfVars suffixConfig) { + String intermediateDirSuffix = HiveConf.getVar(conf, suffixConfig); + return new Path(dir.getParent(), dir.getName() + intermediateDirSuffix); + } + + static boolean pathExists(Path p, Configuration conf) throws HiveException { + try { + FileSystem fs = p.getFileSystem(conf); + return fs.exists(p); + } catch (IOException e) { + throw new HiveException(e); + } + } + + static void deleteDir(Path dir, Database db, Configuration conf) throws HiveException { + try { + Warehouse wh = new Warehouse(conf); + wh.deleteDir(dir, true, db); + } catch (MetaException e) { + throw new HiveException(e); + } + } + + /** + * Sets archiving flag locally; it has to be pushed into metastore. + * @param partition partition to set flag + * @param state desired state of IS_ARCHIVED flag + * @param level desired level for state == true, anything for false + */ + static void setIsArchived(Partition partition, boolean state, int level) { + Map params = partition.getParameters(); + if (state) { + params.put(hive_metastoreConstants.IS_ARCHIVED, "true"); + params.put(ArchiveUtils.ARCHIVING_LEVEL, Integer.toString(level)); + } else { + params.remove(hive_metastoreConstants.IS_ARCHIVED); + params.remove(ArchiveUtils.ARCHIVING_LEVEL); + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableClusteredByDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableClusteredByDesc.java index 8aab47b840..ade2d26927 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableClusteredByDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableClusteredByDesc.java @@ -25,8 +25,8 @@ import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; import org.apache.hadoop.hive.ql.util.DirectionUtils; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableCompactDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableCompactDesc.java new file mode 100644 index 0000000000..3ac9051865 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableCompactDesc.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.storage; + +import java.util.Map; + +import org.apache.hadoop.hive.ql.ddl.DDLDesc; +import org.apache.hadoop.hive.ql.ddl.DDLTask2; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.Explain.Level; + +/** + * DDL task description for ALTER TABLE ... [PARTITION ... ] COMPACT 'major|minor' [WITH OVERWRITE TBLPROPERTIES ...] + * commands. + */ +@Explain(displayName = "Compact", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +public class AlterTableCompactDesc implements DDLDesc { + static { + DDLTask2.registerOperation(AlterTableCompactDesc.class, AlterTableCompactOperation.class); + } + + private final String tableName; + private final Map partitionSpec; + private final String compactionType; + private final boolean isBlocking; + private final Map properties; + + public AlterTableCompactDesc(String tableName, Map partitionSpec, String compactionType, + boolean isBlocking, Map properties) { + this.tableName = tableName; + this.partitionSpec = partitionSpec; + this.compactionType = compactionType; + this.isBlocking = isBlocking; + this.properties = properties; + } + + @Explain(displayName = "table name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public String getTableName() { + return tableName; + } + + @Explain(displayName = "partition spec", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public Map getPartitionSpec() { + return partitionSpec; + } + + @Explain(displayName = "compaction type", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public String getCompactionType() { + return compactionType; + } + + @Explain(displayName = "blocking", displayOnlyOnTrue = true, + explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public boolean isBlocking() { + return isBlocking; + } + + @Explain(displayName = "properties", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public Map getProperties() { + return properties; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableCompactOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableCompactOperation.java new file mode 100644 index 0000000000..4e3e2c5033 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableCompactOperation.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.storage; + +import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.io.AcidUtils; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hive.metastore.api.CompactionResponse; +import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; +import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement; +import org.apache.hadoop.hive.metastore.txn.TxnStore; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.ddl.DDLOperation; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; + +/** + * Operation process of compacting a table. + */ +public class AlterTableCompactOperation extends DDLOperation { + private static final int FIVE_MINUTES_IN_MILLIES = 5*60*1000; + private final AlterTableCompactDesc desc; + + public AlterTableCompactOperation(DDLOperationContext context, AlterTableCompactDesc desc) { + super(context); + this.desc = desc; + } + + @Override + public int execute() throws HiveException { + Table table = context.getDb().getTable(desc.getTableName()); + if (!AcidUtils.isTransactionalTable(table)) { + throw new HiveException(ErrorMsg.NONACID_COMPACTION_NOT_SUPPORTED, table.getDbName(), table.getTableName()); + } + + String partitionName = getPartitionName(table); + + CompactionResponse resp = compact(table, partitionName); + + if (desc.isBlocking() && resp.isAccepted()) { + waitForCompactionToFinish(resp); + } + + return 0; + } + + private String getPartitionName(Table table) throws HiveException { + String partitionName = null; + if (desc.getPartitionSpec() == null) { + if (table.isPartitioned()) { // Compaction can only be done on the whole table if the table is non-partitioned. + throw new HiveException(ErrorMsg.NO_COMPACTION_PARTITION); + } + } else { + Map partitionSpec = desc.getPartitionSpec(); + List partitions = context.getDb().getPartitions(table, partitionSpec); + if (partitions.size() > 1) { + throw new HiveException(ErrorMsg.TOO_MANY_COMPACTION_PARTITIONS); + } else if (partitions.size() == 0) { + throw new HiveException(ErrorMsg.INVALID_PARTITION_SPEC); + } + partitionName = partitions.get(0).getName(); + } + return partitionName; + } + + private CompactionResponse compact(Table table, String partitionName) throws HiveException { + CompactionResponse resp = context.getDb().compact2(table.getDbName(), table.getTableName(), partitionName, + desc.getCompactionType(), desc.getProperties()); + if (resp.isAccepted()) { + context.getConsole().printInfo("Compaction enqueued with id " + resp.getId()); + } else { + context.getConsole().printInfo("Compaction already enqueued with id " + resp.getId() + "; State is " + + resp.getState()); + } + return resp; + } + + private void waitForCompactionToFinish(CompactionResponse resp) throws HiveException { + StringBuilder progressDots = new StringBuilder(); + long waitTimeMs = 1000; + wait: while (true) { + //double wait time until 5min + waitTimeMs = waitTimeMs*2; + waitTimeMs = Math.max(waitTimeMs, FIVE_MINUTES_IN_MILLIES); + try { + Thread.sleep(waitTimeMs); + } catch (InterruptedException ex) { + context.getConsole().printInfo("Interrupted while waiting for compaction with id=" + resp.getId()); + break; + } + + //this could be expensive when there are a lot of compactions.... + //todo: update to search by ID once HIVE-13353 is done + ShowCompactResponse allCompactions = context.getDb().showCompactions(); + for (ShowCompactResponseElement compaction : allCompactions.getCompacts()) { + if (resp.getId() != compaction.getId()) { + continue; + } + + switch (compaction.getState()) { + case TxnStore.WORKING_RESPONSE: + case TxnStore.INITIATED_RESPONSE: + //still working + context.getConsole().printInfo(progressDots.toString()); + progressDots.append("."); + continue wait; + default: + //done + context.getConsole().printInfo("Compaction with id " + resp.getId() + " finished with status: " + + compaction.getState()); + break wait; + } + } + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateDesc.java similarity index 79% rename from ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java rename to ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateDesc.java index 215da93d15..6862b81a72 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/AlterTablePartMergeFilesDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateDesc.java @@ -16,25 +16,33 @@ * limitations under the License. */ -package org.apache.hadoop.hive.ql.parse; +package org.apache.hadoop.hive.ql.ddl.table.storage; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.ddl.DDLDesc; +import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.ListBucketingCtx; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.hive.ql.plan.Explain.Level; - -@Explain(displayName = "Alter Table Partition Merge Files", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) -public class AlterTablePartMergeFilesDesc { +/** + * DDL task description for ALTER TABLE ... [PARTITION ... ] CONCATENATE commands. + */ +@Explain(displayName = "Concatenate", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +public class AlterTableConcatenateDesc implements DDLDesc { + static { + DDLTask2.registerOperation(AlterTableConcatenateDesc.class, AlterTableConcatenateOperation.class); + } private String tableName; - private HashMap partSpec; + private Map partSpec; private ListBucketingCtx lbCtx; // context for list bucketing. private List inputDir = new ArrayList(); @@ -42,8 +50,8 @@ private Class inputFormatClass; private TableDesc tableDesc; - public AlterTablePartMergeFilesDesc(String tableName, - HashMap partSpec) { + public AlterTableConcatenateDesc(String tableName, + Map partSpec) { this.tableName = tableName; this.partSpec = partSpec; } @@ -58,7 +66,7 @@ public void setTableName(String tableName) { } @Explain(displayName = "partition desc") - public HashMap getPartSpec() { + public Map getPartSpec() { return partSpec; } diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateOperation.java new file mode 100644 index 0000000000..1f1ee692bf --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableConcatenateOperation.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.storage; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.LinkedHashMap; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.CompilationOpContext; +import org.apache.hadoop.hive.ql.DriverContext; +import org.apache.hadoop.hive.ql.ddl.DDLOperation; +import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.hadoop.hive.ql.exec.OperatorFactory; +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.tez.TezTask; +import org.apache.hadoop.hive.ql.io.RCFileInputFormat; +import org.apache.hadoop.hive.ql.io.merge.MergeFileTask; +import org.apache.hadoop.hive.ql.io.merge.MergeFileWork; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.FileMergeDesc; +import org.apache.hadoop.hive.ql.plan.ListBucketingCtx; +import org.apache.hadoop.hive.ql.plan.OperatorDesc; +import org.apache.hadoop.hive.ql.plan.OrcFileMergeDesc; +import org.apache.hadoop.hive.ql.plan.RCFileMergeDesc; +import org.apache.hadoop.hive.ql.plan.TezWork; + +/** + * Operation process of concatenating the files of a table/partition. + */ +public class AlterTableConcatenateOperation extends DDLOperation { + private final AlterTableConcatenateDesc desc; + + public AlterTableConcatenateOperation(DDLOperationContext context, AlterTableConcatenateDesc desc) { + super(context); + this.desc = desc; + } + + @Override + public int execute() throws HiveException { + ListBucketingCtx lbCtx = desc.getLbCtx(); + boolean lbatc = lbCtx == null ? false : lbCtx.isSkewedStoredAsDir(); + int lbd = lbCtx == null ? 0 : lbCtx.calculateListBucketingLevel(); + + // merge work only needs input and output. + MergeFileWork mergeWork = new MergeFileWork(desc.getInputDir(), desc.getOutputDir(), + desc.getInputFormatClass().getName(), desc.getTableDesc()); + LinkedHashMap> pathToAliases = new LinkedHashMap<>(); + ArrayList inputDirstr = new ArrayList(1); + inputDirstr.add(desc.getInputDir().toString()); + pathToAliases.put(desc.getInputDir().get(0), inputDirstr); + mergeWork.setPathToAliases(pathToAliases); + mergeWork.setListBucketingCtx(desc.getLbCtx()); + mergeWork.resolveConcatenateMerge(context.getDb().getConf()); + mergeWork.setMapperCannotSpanPartns(true); + mergeWork.setSourceTableInputFormat(desc.getInputFormatClass().getName()); + final FileMergeDesc fmd; + if (desc.getInputFormatClass().equals(RCFileInputFormat.class)) { + fmd = new RCFileMergeDesc(); + } else { + // safe to assume else is ORC as semantic analyzer will check for RC/ORC + fmd = new OrcFileMergeDesc(); + } + + fmd.setDpCtx(null); + fmd.setHasDynamicPartitions(false); + fmd.setListBucketingAlterTableConcatenate(lbatc); + fmd.setListBucketingDepth(lbd); + fmd.setOutputPath(desc.getOutputDir()); + + CompilationOpContext opContext = context.getDriverContext().getCtx().getOpContext(); + Operator mergeOp = OperatorFactory.get(opContext, fmd); + + LinkedHashMap> aliasToWork = + new LinkedHashMap>(); + aliasToWork.put(desc.getInputDir().toString(), mergeOp); + mergeWork.setAliasToWork(aliasToWork); + DriverContext driverCxt = new DriverContext(); + Task task; + if (context.getConf().getVar(ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) { + TezWork tezWork = new TezWork(context.getQueryState().getQueryId(), context.getConf()); + mergeWork.setName("File Merge"); + tezWork.add(mergeWork); + task = new TezTask(); + ((TezTask) task).setWork(tezWork); + } else { + task = new MergeFileTask(); + ((MergeFileTask) task).setWork(mergeWork); + } + + // initialize the task and execute + task.initialize(context.getQueryState(), context.getQueryPlan(), driverCxt, opContext); + Task subtask = task; + int ret = task.execute(driverCxt); + if (subtask.getException() != null) { + context.getTask().setException(subtask.getException()); + } + return ret; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableIntoBucketsDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableIntoBucketsDesc.java index 680f31096e..daa33cb237 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableIntoBucketsDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableIntoBucketsDesc.java @@ -22,8 +22,8 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotClusteredDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotClusteredDesc.java index a335d0dbdc..48e573ff88 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotClusteredDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotClusteredDesc.java @@ -22,8 +22,8 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSkewedDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSkewedDesc.java index af67964c73..492e447014 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSkewedDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSkewedDesc.java @@ -20,8 +20,8 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java index 11e8bf37eb..2226495be5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableNotSortedDesc.java @@ -22,8 +22,8 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetFileFormatDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetFileFormatDesc.java index 89bbb17aec..ebf3e2ae77 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetFileFormatDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetFileFormatDesc.java @@ -22,8 +22,8 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationDesc.java index c918bb9870..f4cb818021 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetLocationDesc.java @@ -22,8 +22,8 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdeDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdeDesc.java index 861139d41b..481bd87253 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdeDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdeDesc.java @@ -22,8 +22,8 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdePropsDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdePropsDesc.java index 381b94f38a..70164a8edf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdePropsDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSerdePropsDesc.java @@ -22,8 +22,8 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSkewedLocationDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSkewedLocationDesc.java index afe2b0817b..5772b300de 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSkewedLocationDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSetSkewedLocationDesc.java @@ -24,8 +24,8 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSkewedByDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSkewedByDesc.java index 6a6f397ef7..4ec0ed43fd 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSkewedByDesc.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableSkewedByDesc.java @@ -23,8 +23,8 @@ import org.apache.hadoop.hive.ql.ddl.DDLTask2; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.Explain; import org.apache.hadoop.hive.ql.plan.Explain.Level; diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveDesc.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveDesc.java new file mode 100644 index 0000000000..b75299c56f --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveDesc.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.storage; + +import java.util.Map; + +import org.apache.hadoop.hive.ql.ddl.DDLDesc; +import org.apache.hadoop.hive.ql.ddl.DDLTask2; +import org.apache.hadoop.hive.ql.plan.Explain; +import org.apache.hadoop.hive.ql.plan.Explain.Level; + +/** + * DDL task description for ALTER TABLE ... UNARCHIVE [PARTITION ...] commands. + */ +@Explain(displayName = "Unarchive", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) +public class AlterTableUnarchiveDesc implements DDLDesc { + static { + DDLTask2.registerOperation(AlterTableUnarchiveDesc.class, AlterTableUnarchiveOperation.class); + } + + private final String tableName; + private final Map partitionSpec; + + public AlterTableUnarchiveDesc(String tableName, Map partitionSpec) { + this.tableName = tableName; + this.partitionSpec = partitionSpec; + } + + @Explain(displayName = "table name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public String getTableName() { + return tableName; + } + + @Explain(displayName = "partition spec", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) + public Map getPartitionSpec() { + return partitionSpec; + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveOperation.java new file mode 100644 index 0000000000..6c4a96df00 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/AlterTableUnarchiveOperation.java @@ -0,0 +1,303 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.ddl.table.storage; + +import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; +import org.apache.hadoop.hive.ql.exec.ArchiveUtils; +import org.apache.hadoop.hive.ql.exec.ArchiveUtils.PartSpecInfo; +import org.apache.hadoop.hive.ql.ddl.DDLOperation; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.util.ToolRunner; + +import com.google.common.collect.ImmutableList; + +import static org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableArchiveUtils.ARCHIVE_NAME; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FsShell; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.MetaException; + +/** + * Operation process of truncating a table. + */ +public class AlterTableUnarchiveOperation extends DDLOperation { + private final AlterTableUnarchiveDesc desc; + + public AlterTableUnarchiveOperation(DDLOperationContext context, AlterTableUnarchiveDesc desc) { + super(context); + this.desc = desc; + } + + @Override + public int execute() throws HiveException, URISyntaxException { + Table table = context.getDb().getTable(desc.getTableName()); + if (table.getTableType() != TableType.MANAGED_TABLE) { + throw new HiveException("UNARCHIVE can only be performed on managed tables"); + } + + Map partitionSpec = desc.getPartitionSpec(); + if (partitionSpec == null) { + throw new HiveException("UNARCHIVE is for partitions only"); + } + + PartSpecInfo partitionSpecInfo = PartSpecInfo.create(table, partitionSpec); + List partitions = context.getDb().getPartitions(table, partitionSpec); + + // The originalDir directory represents the original directory of the partitions' files. They now contain an + // archived version of those files eg. hdfs:/warehouse/myTable/ds=1/ + Path originalDir = getOriginalDir(table, partitionSpecInfo, partitions); + Path intermediateArchivedDir = AlterTableArchiveUtils.getInterMediateDir(originalDir, context.getConf(), + ConfVars.METASTORE_INT_ARCHIVED); + Path intermediateExtractedDir = AlterTableArchiveUtils.getInterMediateDir(originalDir, context.getConf(), + ConfVars.METASTORE_INT_EXTRACTED); + + boolean recovery = isRecovery(intermediateArchivedDir, intermediateExtractedDir); + + for (Partition partition : partitions) { + checkArchiveProperty(partitionSpec.size(), recovery, partition); + } + + // assume the archive is in the original dir, check if it exists + // The source directory is the directory containing all the files that should be in the partitions. + // e.g. har:/warehouse/myTable/ds=1/myTable.har/ - Note the har:/ scheme + Path archivePath = new Path(originalDir, ARCHIVE_NAME); + URI archiveUri = archivePath.toUri(); + URI originalUri = ArchiveUtils.addSlash(originalDir.toUri()); + ArchiveUtils.HarPathHelper harHelper = new ArchiveUtils.HarPathHelper(context.getConf(), archiveUri, originalUri); + URI sourceUri = harHelper.getHarUri(originalUri); + Path sourceDir = new Path(sourceUri.getScheme(), sourceUri.getAuthority(), sourceUri.getPath()); + + if (!AlterTableArchiveUtils.pathExists(intermediateArchivedDir, context.getConf()) && + !AlterTableArchiveUtils.pathExists(archivePath, context.getConf())) { + throw new HiveException("Haven't found any archive where it should be"); + } + + Path tmpPath = context.getDriverContext().getCtx().getExternalTmpPath(originalDir); + + FileSystem fs = null; + try { + fs = tmpPath.getFileSystem(context.getConf()); + } catch (IOException e) { + throw new HiveException(e); + } + + if (!recovery) { + extractArchiveToIntermediateDir(intermediateExtractedDir, sourceDir, tmpPath, fs); + } + + // At this point, we know that the extracted files are in the intermediate extracted dir, or in the the original dir + + moveOriginalDirToIntermediateDir(originalDir, intermediateArchivedDir, fs); + + // If there is a failure from here to until when the metadata is changed, the partition will be empty or throw + // errors on read. If the original location exists here, then it must be the extracted files because in the previous + // step, we moved the previous original location (containing the archived files) to intermediateArchiveDir + + moveIntermediateExtractedDirToOriginalParent(originalDir, intermediateExtractedDir, fs); + + writeUnarchivationToMetastore(partitions); + + // If a failure happens here, the intermediate archive files won't be + // deleted. The user will need to call unarchive again to clear those up. + + deleteIntermediateArchivedDir(table, intermediateArchivedDir); + + if (recovery) { + context.getConsole().printInfo("Recovery after UNARCHIVE succeeded"); + } + + return 0; + } + + private Path getOriginalDir(Table table, PartSpecInfo partitionSpecInfo, List partitions) + throws HiveException { + Path originalDir = null; + + // when we have partial partitions specification we must assume partitions + // lie in standard place - if they were in custom locations putting + // them into one archive would involve mass amount of copying + // in full partition specification case we allow custom locations + // to keep backward compatibility + if (partitions.isEmpty()) { + throw new HiveException("No partition matches the specification"); + } else if (partitionSpecInfo.values.size() != table.getPartCols().size()) { + // for partial specifications we need partitions to follow the scheme + for (Partition partition : partitions){ + if (AlterTableArchiveUtils.partitionInCustomLocation(table, partition)) { + String message = String.format("UNARCHIVE cannot run for partition groups with custom locations like %s", + partition.getLocation()); + throw new HiveException(message); + } + } + originalDir = partitionSpecInfo.createPath(table); + } else { + Partition partition = partitions.get(0); + if (ArchiveUtils.isArchived(partition)) { + originalDir = new Path(AlterTableArchiveUtils.getOriginalLocation(partition)); + } else { + originalDir = new Path(partition.getLocation()); + } + } + return originalDir; + } + + private boolean isRecovery(Path intermediateArchivedDir, Path intermediateExtractedDir) throws HiveException { + if (AlterTableArchiveUtils.pathExists(intermediateArchivedDir, context.getConf()) || + AlterTableArchiveUtils.pathExists(intermediateExtractedDir, context.getConf())) { + context.getConsole().printInfo("Starting recovery after failed UNARCHIVE"); + return true; + } + return false; + } + + private void checkArchiveProperty(int partitionSpecLevel, boolean recovery, Partition partition) + throws HiveException { + if (!ArchiveUtils.isArchived(partition) && !recovery) { + throw new HiveException("Partition " + partition.getName() + " is not archived."); + } + int archiveLevel = ArchiveUtils.getArchivingLevel(partition); + if (partitionSpecLevel > archiveLevel) { + throw new HiveException("Partition " + partition.getName() + " is archived at level " + archiveLevel + + ", and given partspec only has " + partitionSpecLevel + " specs."); + } + } + + private void extractArchiveToIntermediateDir(Path intermediateExtractedDir, Path sourceDir, Path tmpPath, + FileSystem fs) throws HiveException { + try { + // Copy the files out of the archive into the temporary directory + String copySource = sourceDir.toString(); + String copyDest = tmpPath.toString(); + List args = ImmutableList.of("-cp", copySource, copyDest); + + context.getConsole().printInfo("Copying " + copySource + " to " + copyDest); + FileSystem srcFs = FileSystem.get(sourceDir.toUri(), context.getConf()); + srcFs.initialize(sourceDir.toUri(), context.getConf()); + + FsShell fss = new FsShell(context.getConf()); + int ret = 0; + try { + ret = ToolRunner.run(fss, args.toArray(new String[0])); + } catch (Exception e) { + throw new HiveException(e); + } + + if (ret != 0) { + throw new HiveException("Error while copying files from archive, return code=" + ret); + } else { + context.getConsole().printInfo("Successfully Copied " + copySource + " to " + copyDest); + } + + context.getConsole().printInfo("Moving " + tmpPath + " to " + intermediateExtractedDir); + if (fs.exists(intermediateExtractedDir)) { + throw new HiveException("Invalid state: the intermediate extracted directory already exists."); + } + fs.rename(tmpPath, intermediateExtractedDir); + } catch (Exception e) { + throw new HiveException(e); + } + } + + private void moveOriginalDirToIntermediateDir(Path originalDir, Path intermediateArchivedDir, FileSystem fs) + throws HiveException { + if (!AlterTableArchiveUtils.pathExists(intermediateArchivedDir, context.getConf())) { + try { + context.getConsole().printInfo("Moving " + originalDir + " to " + intermediateArchivedDir); + fs.rename(originalDir, intermediateArchivedDir); + } catch (IOException e) { + throw new HiveException(e); + } + } else { + context.getConsole().printInfo(intermediateArchivedDir + " already exists. " + + "Assuming it contains the archived version of the partition"); + } + } + + private void moveIntermediateExtractedDirToOriginalParent(Path originalDir, Path intermediateExtractedDir, + FileSystem fs) throws HiveException { + if (!AlterTableArchiveUtils.pathExists(originalDir, context.getConf())) { + try { + context.getConsole().printInfo("Moving " + intermediateExtractedDir + " to " + originalDir); + fs.rename(intermediateExtractedDir, originalDir); + } catch (IOException e) { + throw new HiveException(e); + } + } else { + context.getConsole().printInfo(originalDir + " already exists. " + + "Assuming it contains the extracted files in the partition"); + } + } + + private void writeUnarchivationToMetastore(List partitions) throws HiveException { + for (Partition partition : partitions) { + setUnArchived(partition); + try { + // TODO: catalog + context.getDb().alterPartition(desc.getTableName(), partition, null, true); + } catch (InvalidOperationException e) { + throw new HiveException(e); + } + } + } + + /** + * Sets the appropriate attributes in the supplied Partition object to mark it as not archived. + * Note that the metastore is not touched - a separate call to alter_partition is needed. + * + * @param partition - the partition to modify + */ + private void setUnArchived(Partition partition) { + assert(ArchiveUtils.isArchived(partition)); + String parentDir = AlterTableArchiveUtils.getOriginalLocation(partition); + AlterTableArchiveUtils.setIsArchived(partition, false, 0); + AlterTableArchiveUtils.setOriginalLocation(partition, null); + assert(parentDir != null); + partition.setLocation(parentDir); + } + + private void deleteIntermediateArchivedDir(Table table, Path intermediateArchivedDir) throws HiveException { + if (AlterTableArchiveUtils.pathExists(intermediateArchivedDir, context.getConf())) { + deleteDir(intermediateArchivedDir, context.getDb().getDatabase(table.getDbName()), context.getConf()); + } + } + + private void deleteDir(Path dir, Database db, Configuration conf) throws HiveException { + try { + Warehouse wh = new Warehouse(conf); + wh.deleteDir(dir, true, db); + } catch (MetaException e) { + throw new HiveException(e); + } + } +} diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 88ea73f8d5..d9e12c136b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -19,88 +19,37 @@ package org.apache.hadoop.hive.ql.exec; import java.io.DataOutputStream; -import java.io.IOException; import java.io.Serializable; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Set; -import com.google.common.collect.Lists; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.DefaultHiveMetaHook; import org.apache.hadoop.hive.metastore.HiveMetaHook; import org.apache.hadoop.hive.metastore.Msck; import org.apache.hadoop.hive.metastore.MsckInfo; -import org.apache.hadoop.hive.metastore.TableType; -import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.CompactionResponse; import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.EnvironmentContext; -import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; -import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.txn.TxnStore; -import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.DriverContext; -import org.apache.hadoop.hive.ql.ErrorMsg; -import org.apache.hadoop.hive.ql.QueryPlan; -import org.apache.hadoop.hive.ql.QueryState; -import org.apache.hadoop.hive.ql.exec.ArchiveUtils.PartSpecInfo; -import org.apache.hadoop.hive.ql.exec.tez.TezTask; -import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; -import org.apache.hadoop.hive.ql.io.AcidUtils; -import org.apache.hadoop.hive.ql.io.RCFileInputFormat; -import org.apache.hadoop.hive.ql.io.merge.MergeFileTask; -import org.apache.hadoop.hive.ql.io.merge.MergeFileWork; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.Partition; -import org.apache.hadoop.hive.ql.metadata.PartitionIterable; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.parse.AlterTablePartMergeFilesDesc; -import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.AnalyzeState; -import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.repl.dump.Utils; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; -import org.apache.hadoop.hive.ql.plan.AlterTableSimpleDesc; import org.apache.hadoop.hive.ql.plan.CacheMetadataDesc; import org.apache.hadoop.hive.ql.plan.DDLWork; -import org.apache.hadoop.hive.ql.plan.FileMergeDesc; import org.apache.hadoop.hive.ql.plan.InsertCommitHookDesc; -import org.apache.hadoop.hive.ql.plan.ListBucketingCtx; -import org.apache.hadoop.hive.ql.plan.LoadMultiFilesDesc; -import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.MsckDesc; -import org.apache.hadoop.hive.ql.plan.OperatorDesc; -import org.apache.hadoop.hive.ql.plan.OrcFileMergeDesc; -import org.apache.hadoop.hive.ql.plan.RCFileMergeDesc; import org.apache.hadoop.hive.ql.plan.ReplRemoveFirstIncLoadPendFlagDesc; import org.apache.hadoop.hive.ql.plan.ShowConfDesc; -import org.apache.hadoop.hive.ql.plan.TezWork; import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.Deserializer; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.tools.HadoopArchives; -import org.apache.hadoop.util.ToolRunner; import org.apache.hive.common.util.ReflectionUtil; import org.apache.thrift.TException; import org.slf4j.Logger; @@ -117,12 +66,6 @@ private static final int separator = Utilities.tabCode; private static final int terminator = Utilities.newLineCode; - // These are suffixes attached to intermediate directory names used in the - // archiving / un-archiving process. - private static String INTERMEDIATE_ARCHIVED_DIR_SUFFIX; - private static String INTERMEDIATE_ORIGINAL_DIR_SUFFIX; - private static String INTERMEDIATE_EXTRACTED_DIR_SUFFIX; - @Override public boolean requireLock() { return this.work != null && this.work.getNeedLock(); @@ -132,21 +75,6 @@ public DDLTask() { super(); } - @Override - public void initialize(QueryState queryState, QueryPlan queryPlan, DriverContext ctx, - CompilationOpContext opContext) { - super.initialize(queryState, queryPlan, ctx, opContext); - - // Pick the formatter to use to display the results. Either the - // normal human readable output or a json object. - INTERMEDIATE_ARCHIVED_DIR_SUFFIX = - HiveConf.getVar(conf, ConfVars.METASTORE_INT_ARCHIVED); - INTERMEDIATE_ORIGINAL_DIR_SUFFIX = - HiveConf.getVar(conf, ConfVars.METASTORE_INT_ORIGINAL); - INTERMEDIATE_EXTRACTED_DIR_SUFFIX = - HiveConf.getVar(conf, ConfVars.METASTORE_INT_EXTRACTED); - } - @Override public int execute(DriverContext driverContext) { if (driverContext.getCtx().getExplainAnalyze() == AnalyzeState.RUNNING) { @@ -158,30 +86,6 @@ public int execute(DriverContext driverContext) { try { db = Hive.get(conf); - AlterTableDesc alterTbl = work.getAlterTblDesc(); - if (alterTbl != null) { - if (!allowOperationInReplicationScope(db, alterTbl.getOldName(), null, alterTbl.getReplicationSpec())) { - // no alter, the table is missing either due to drop/rename which follows the alter. - // or the existing table is newer than our update. - LOG.debug("DDLTask: Alter Table is skipped as table {} is newer than update", alterTbl.getOldName()); - return 0; - } - return alterTable(db, alterTbl); - } - - AlterTableSimpleDesc simpleDesc = work.getAlterTblSimpleDesc(); - if (simpleDesc != null) { - if (simpleDesc.getType() == AlterTableTypes.TOUCH) { - return touch(db, simpleDesc); - } else if (simpleDesc.getType() == AlterTableTypes.ARCHIVE) { - return archive(db, simpleDesc, driverContext); - } else if (simpleDesc.getType() == AlterTableTypes.UNARCHIVE) { - return unarchive(db, simpleDesc); - } else if (simpleDesc.getType() == AlterTableTypes.COMPACT) { - return compact(db, simpleDesc); - } - } - MsckDesc msckDesc = work.getMsckDesc(); if (msckDesc != null) { return msck(db, msckDesc); @@ -192,11 +96,6 @@ public int execute(DriverContext driverContext) { return showConf(db, showConf); } - AlterTablePartMergeFilesDesc mergeFilesDesc = work.getMergeFilesDesc(); - if (mergeFilesDesc != null) { - return mergeFiles(db, mergeFilesDesc, driverContext); - } - CacheMetadataDesc cacheMetadataDesc = work.getCacheMetadataDesc(); if (cacheMetadataDesc != null) { return cacheMetadata(db, cacheMetadataDesc); @@ -287,771 +186,6 @@ private DataOutputStream getOutputStream(Path outputFile) throws HiveException { } } - /** - * First, make sure the source table/partition is not - * archived/indexes/non-rcfile. If either of these is true, throw an - * exception. - * - * The way how it does the merge is to create a BlockMergeTask from the - * mergeFilesDesc. - * - * @param db - * @param mergeFilesDesc - * @return - * @throws HiveException - */ - private int mergeFiles(Hive db, AlterTablePartMergeFilesDesc mergeFilesDesc, - DriverContext driverContext) throws HiveException { - ListBucketingCtx lbCtx = mergeFilesDesc.getLbCtx(); - boolean lbatc = lbCtx == null ? false : lbCtx.isSkewedStoredAsDir(); - int lbd = lbCtx == null ? 0 : lbCtx.calculateListBucketingLevel(); - - // merge work only needs input and output. - MergeFileWork mergeWork = new MergeFileWork(mergeFilesDesc.getInputDir(), - mergeFilesDesc.getOutputDir(), mergeFilesDesc.getInputFormatClass().getName(), - mergeFilesDesc.getTableDesc()); - LinkedHashMap> pathToAliases = new LinkedHashMap<>(); - ArrayList inputDirstr = new ArrayList(1); - inputDirstr.add(mergeFilesDesc.getInputDir().toString()); - pathToAliases.put(mergeFilesDesc.getInputDir().get(0), inputDirstr); - mergeWork.setPathToAliases(pathToAliases); - mergeWork.setListBucketingCtx(mergeFilesDesc.getLbCtx()); - mergeWork.resolveConcatenateMerge(db.getConf()); - mergeWork.setMapperCannotSpanPartns(true); - mergeWork.setSourceTableInputFormat(mergeFilesDesc.getInputFormatClass().getName()); - final FileMergeDesc fmd; - if (mergeFilesDesc.getInputFormatClass().equals(RCFileInputFormat.class)) { - fmd = new RCFileMergeDesc(); - } else { - // safe to assume else is ORC as semantic analyzer will check for RC/ORC - fmd = new OrcFileMergeDesc(); - } - - fmd.setDpCtx(null); - fmd.setHasDynamicPartitions(false); - fmd.setListBucketingAlterTableConcatenate(lbatc); - fmd.setListBucketingDepth(lbd); - fmd.setOutputPath(mergeFilesDesc.getOutputDir()); - - CompilationOpContext opContext = driverContext.getCtx().getOpContext(); - Operator mergeOp = OperatorFactory.get(opContext, fmd); - - LinkedHashMap> aliasToWork = - new LinkedHashMap>(); - aliasToWork.put(mergeFilesDesc.getInputDir().toString(), mergeOp); - mergeWork.setAliasToWork(aliasToWork); - DriverContext driverCxt = new DriverContext(); - Task task; - if (conf.getVar(ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) { - TezWork tezWork = new TezWork(queryState.getQueryId(), conf); - mergeWork.setName("File Merge"); - tezWork.add(mergeWork); - task = new TezTask(); - ((TezTask) task).setWork(tezWork); - } else { - task = new MergeFileTask(); - ((MergeFileTask) task).setWork(mergeWork); - } - - // initialize the task and execute - task.initialize(queryState, getQueryPlan(), driverCxt, opContext); - Task subtask = task; - int ret = task.execute(driverCxt); - if (subtask.getException() != null) { - setException(subtask.getException()); - } - return ret; - } - - /** - * Rewrite the partition's metadata and force the pre/post execute hooks to - * be fired. - * - * @param db - * @param touchDesc - * @return - * @throws HiveException - */ - private int touch(Hive db, AlterTableSimpleDesc touchDesc) - throws HiveException { - // TODO: catalog - Table tbl = db.getTable(touchDesc.getTableName()); - EnvironmentContext environmentContext = new EnvironmentContext(); - environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); - - if (touchDesc.getPartSpec() == null) { - db.alterTable(tbl, false, environmentContext, true); - work.getInputs().add(new ReadEntity(tbl)); - addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK)); - } else { - Partition part = db.getPartition(tbl, touchDesc.getPartSpec(), false); - if (part == null) { - throw new HiveException("Specified partition does not exist"); - } - try { - db.alterPartition(tbl.getCatalogName(), tbl.getDbName(), tbl.getTableName(), - part, environmentContext, true); - } catch (InvalidOperationException e) { - throw new HiveException(e); - } - work.getInputs().add(new ReadEntity(part)); - addIfAbsentByName(new WriteEntity(part, WriteEntity.WriteType.DDL_NO_LOCK)); - } - return 0; - } - - /** - * Sets archiving flag locally; it has to be pushed into metastore - * @param p partition to set flag - * @param state desired state of IS_ARCHIVED flag - * @param level desired level for state == true, anything for false - */ - private void setIsArchived(Partition p, boolean state, int level) { - Map params = p.getParameters(); - if (state) { - params.put(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.IS_ARCHIVED, - "true"); - params.put(ArchiveUtils.ARCHIVING_LEVEL, Integer - .toString(level)); - } else { - params.remove(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.IS_ARCHIVED); - params.remove(ArchiveUtils.ARCHIVING_LEVEL); - } - } - - /** - * Returns original partition of archived partition, null for unarchived one - */ - private String getOriginalLocation(Partition p) { - Map params = p.getParameters(); - return params.get( - org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ORIGINAL_LOCATION); - } - - /** - * Sets original location of partition which is to be archived - */ - private void setOriginalLocation(Partition p, String loc) { - Map params = p.getParameters(); - if (loc == null) { - params.remove(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ORIGINAL_LOCATION); - } else { - params.put(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ORIGINAL_LOCATION, loc); - } - } - - /** - * Sets the appropriate attributes in the supplied Partition object to mark - * it as archived. Note that the metastore is not touched - a separate - * call to alter_partition is needed. - * - * @param p - the partition object to modify - * @param harPath - new location of partition (har schema URI) - */ - private void setArchived(Partition p, Path harPath, int level) { - assert(ArchiveUtils.isArchived(p) == false); - setIsArchived(p, true, level); - setOriginalLocation(p, p.getLocation()); - p.setLocation(harPath.toString()); - } - - /** - * Sets the appropriate attributes in the supplied Partition object to mark - * it as not archived. Note that the metastore is not touched - a separate - * call to alter_partition is needed. - * - * @param p - the partition to modify - */ - private void setUnArchived(Partition p) { - assert(ArchiveUtils.isArchived(p) == true); - String parentDir = getOriginalLocation(p); - setIsArchived(p, false, 0); - setOriginalLocation(p, null); - assert(parentDir != null); - p.setLocation(parentDir); - } - - private boolean pathExists(Path p) throws HiveException { - try { - FileSystem fs = p.getFileSystem(conf); - return fs.exists(p); - } catch (IOException e) { - throw new HiveException(e); - } - } - - private void moveDir(FileSystem fs, Path from, Path to) throws HiveException { - try { - if (!fs.rename(from, to)) { - throw new HiveException("Moving " + from + " to " + to + " failed!"); - } - } catch (IOException e) { - throw new HiveException(e); - } - } - - private void deleteDir(Path dir, Database db) throws HiveException { - try { - Warehouse wh = new Warehouse(conf); - wh.deleteDir(dir, true, db); - } catch (MetaException e) { - throw new HiveException(e); - } - } - - /** - * Checks in partition is in custom (not-standard) location. - * @param tbl - table in which partition is - * @param p - partition - * @return true if partition location is custom, false if it is standard - */ - boolean partitionInCustomLocation(Table tbl, Partition p) - throws HiveException { - String subdir = null; - try { - subdir = Warehouse.makePartName(tbl.getPartCols(), p.getValues()); - } catch (MetaException e) { - throw new HiveException("Unable to get partition's directory", e); - } - Path tableDir = tbl.getDataLocation(); - if(tableDir == null) { - throw new HiveException("Table has no location set"); - } - - String standardLocation = (new Path(tableDir, subdir)).toString(); - if(ArchiveUtils.isArchived(p)) { - return !getOriginalLocation(p).equals(standardLocation); - } else { - return !p.getLocation().equals(standardLocation); - } - } - - private int archive(Hive db, AlterTableSimpleDesc simpleDesc, - DriverContext driverContext) - throws HiveException { - - Table tbl = db.getTable(simpleDesc.getTableName()); - - if (tbl.getTableType() != TableType.MANAGED_TABLE) { - throw new HiveException("ARCHIVE can only be performed on managed tables"); - } - - Map partSpec = simpleDesc.getPartSpec(); - PartSpecInfo partSpecInfo = PartSpecInfo.create(tbl, partSpec); - List partitions = db.getPartitions(tbl, partSpec); - - Path originalDir = null; - - // when we have partial partitions specification we must assume partitions - // lie in standard place - if they were in custom locations putting - // them into one archive would involve mass amount of copying - // in full partition specification case we allow custom locations - // to keep backward compatibility - if (partitions.isEmpty()) { - throw new HiveException("No partition matches the specification"); - } else if(partSpecInfo.values.size() != tbl.getPartCols().size()) { - // for partial specifications we need partitions to follow the scheme - for(Partition p: partitions){ - if(partitionInCustomLocation(tbl, p)) { - String message = String.format("ARCHIVE cannot run for partition " + - "groups with custom locations like %s", p.getLocation()); - throw new HiveException(message); - } - } - originalDir = partSpecInfo.createPath(tbl); - } else { - Partition p = partitions.get(0); - // partition can be archived if during recovery - if(ArchiveUtils.isArchived(p)) { - originalDir = new Path(getOriginalLocation(p)); - } else { - originalDir = p.getDataLocation(); - } - } - - Path intermediateArchivedDir = new Path(originalDir.getParent(), - originalDir.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX); - Path intermediateOriginalDir = new Path(originalDir.getParent(), - originalDir.getName() + INTERMEDIATE_ORIGINAL_DIR_SUFFIX); - - console.printInfo("intermediate.archived is " + intermediateArchivedDir.toString()); - console.printInfo("intermediate.original is " + intermediateOriginalDir.toString()); - - String archiveName = "data.har"; - FileSystem fs = null; - try { - fs = originalDir.getFileSystem(conf); - } catch (IOException e) { - throw new HiveException(e); - } - - URI archiveUri = (new Path(originalDir, archiveName)).toUri(); - URI originalUri = ArchiveUtils.addSlash(originalDir.toUri()); - ArchiveUtils.HarPathHelper harHelper = new ArchiveUtils.HarPathHelper( - conf, archiveUri, originalUri); - - // we checked if partitions matching specification are marked as archived - // in the metadata; if they are and their levels are the same as we would - // set it later it means previous run failed and we have to do the recovery; - // if they are different, we throw an error - for(Partition p: partitions) { - if(ArchiveUtils.isArchived(p)) { - if(ArchiveUtils.getArchivingLevel(p) != partSpecInfo.values.size()) { - String name = ArchiveUtils.getPartialName(p, ArchiveUtils.getArchivingLevel(p)); - String m = String.format("Conflict with existing archive %s", name); - throw new HiveException(m); - } else { - throw new HiveException("Partition(s) already archived"); - } - } - } - - boolean recovery = false; - if (pathExists(intermediateArchivedDir) - || pathExists(intermediateOriginalDir)) { - recovery = true; - console.printInfo("Starting recovery after failed ARCHIVE"); - } - - // The following steps seem roundabout, but they are meant to aid in - // recovery if a failure occurs and to keep a consistent state in the FS - - // Steps: - // 1. Create the archive in a temporary folder - // 2. Move the archive dir to an intermediate dir that is in at the same - // dir as the original partition dir. Call the new dir - // intermediate-archive. - // 3. Rename the original partition dir to an intermediate dir. Call the - // renamed dir intermediate-original - // 4. Rename intermediate-archive to the original partition dir - // 5. Change the metadata - // 6. Delete the original partition files in intermediate-original - - // The original partition files are deleted after the metadata change - // because the presence of those files are used to indicate whether - // the original partition directory contains archived or unarchived files. - - // Create an archived version of the partition in a directory ending in - // ARCHIVE_INTERMEDIATE_DIR_SUFFIX that's the same level as the partition, - // if it does not already exist. If it does exist, we assume the dir is good - // to use as the move operation that created it is atomic. - if (!pathExists(intermediateArchivedDir) && - !pathExists(intermediateOriginalDir)) { - - // First create the archive in a tmp dir so that if the job fails, the - // bad files don't pollute the filesystem - Path tmpPath = new Path(driverContext.getCtx() - .getExternalTmpPath(originalDir), "partlevel"); - - console.printInfo("Creating " + archiveName + - " for " + originalDir.toString()); - console.printInfo("in " + tmpPath); - console.printInfo("Please wait... (this may take a while)"); - - // Create the Hadoop archive - int ret=0; - try { - int maxJobNameLen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH); - String jobname = String.format("Archiving %s@%s", - tbl.getTableName(), partSpecInfo.getName()); - jobname = Utilities.abbreviate(jobname, maxJobNameLen - 6); - conf.set(MRJobConfig.JOB_NAME, jobname); - HadoopArchives har = new HadoopArchives(conf); - List args = new ArrayList(); - - args.add("-archiveName"); - args.add(archiveName); - args.add("-p"); - args.add(originalDir.toString()); - args.add(tmpPath.toString()); - - ret = ToolRunner.run(har, args.toArray(new String[0])); - } catch (Exception e) { - throw new HiveException(e); - } - if (ret != 0) { - throw new HiveException("Error while creating HAR"); - } - - // Move from the tmp dir to an intermediate directory, in the same level as - // the partition directory. e.g. .../hr=12-intermediate-archived - try { - console.printInfo("Moving " + tmpPath + " to " + intermediateArchivedDir); - if (pathExists(intermediateArchivedDir)) { - throw new HiveException("The intermediate archive directory already exists."); - } - fs.rename(tmpPath, intermediateArchivedDir); - } catch (IOException e) { - throw new HiveException("Error while moving tmp directory"); - } - } else { - if (pathExists(intermediateArchivedDir)) { - console.printInfo("Intermediate archive directory " + intermediateArchivedDir + - " already exists. Assuming it contains an archived version of the partition"); - } - } - - // If we get to here, we know that we've archived the partition files, but - // they may be in the original partition location, or in the intermediate - // original dir. - - // Move the original parent directory to the intermediate original directory - // if the move hasn't been made already - if (!pathExists(intermediateOriginalDir)) { - console.printInfo("Moving " + originalDir + " to " + - intermediateOriginalDir); - moveDir(fs, originalDir, intermediateOriginalDir); - } else { - console.printInfo(intermediateOriginalDir + " already exists. " + - "Assuming it contains the original files in the partition"); - } - - // If there's a failure from here to when the metadata is updated, - // there will be no data in the partition, or an error while trying to read - // the partition (if the archive files have been moved to the original - // partition directory.) But re-running the archive command will allow - // recovery - - // Move the intermediate archived directory to the original parent directory - if (!pathExists(originalDir)) { - console.printInfo("Moving " + intermediateArchivedDir + " to " + - originalDir); - moveDir(fs, intermediateArchivedDir, originalDir); - } else { - console.printInfo(originalDir + " already exists. " + - "Assuming it contains the archived version of the partition"); - } - - // Record this change in the metastore - try { - for(Partition p: partitions) { - URI originalPartitionUri = ArchiveUtils.addSlash(p.getDataLocation().toUri()); - URI harPartitionDir = harHelper.getHarUri(originalPartitionUri); - StringBuilder authority = new StringBuilder(); - if(harPartitionDir.getUserInfo() != null) { - authority.append(harPartitionDir.getUserInfo()).append("@"); - } - authority.append(harPartitionDir.getHost()); - if(harPartitionDir.getPort() != -1) { - authority.append(":").append(harPartitionDir.getPort()); - } - Path harPath = new Path(harPartitionDir.getScheme(), - authority.toString(), - harPartitionDir.getPath()); // make in Path to ensure no slash at the end - setArchived(p, harPath, partSpecInfo.values.size()); - // TODO: catalog - db.alterPartition(simpleDesc.getTableName(), p, null, true); - } - } catch (Exception e) { - throw new HiveException("Unable to change the partition info for HAR", e); - } - - // If a failure occurs here, the directory containing the original files - // will not be deleted. The user will run ARCHIVE again to clear this up - if(pathExists(intermediateOriginalDir)) { - deleteDir(intermediateOriginalDir, db.getDatabase(tbl.getDbName())); - } - - if(recovery) { - console.printInfo("Recovery after ARCHIVE succeeded"); - } - - return 0; - } - - private int unarchive(Hive db, AlterTableSimpleDesc simpleDesc) - throws HiveException, URISyntaxException { - - Table tbl = db.getTable(simpleDesc.getTableName()); - - // Means user specified a table, not a partition - if (simpleDesc.getPartSpec() == null) { - throw new HiveException("UNARCHIVE is for partitions only"); - } - - if (tbl.getTableType() != TableType.MANAGED_TABLE) { - throw new HiveException("UNARCHIVE can only be performed on managed tables"); - } - - Map partSpec = simpleDesc.getPartSpec(); - PartSpecInfo partSpecInfo = PartSpecInfo.create(tbl, partSpec); - List partitions = db.getPartitions(tbl, partSpec); - - int partSpecLevel = partSpec.size(); - - Path originalDir = null; - - // when we have partial partitions specification we must assume partitions - // lie in standard place - if they were in custom locations putting - // them into one archive would involve mass amount of copying - // in full partition specification case we allow custom locations - // to keep backward compatibility - if (partitions.isEmpty()) { - throw new HiveException("No partition matches the specification"); - } else if(partSpecInfo.values.size() != tbl.getPartCols().size()) { - // for partial specifications we need partitions to follow the scheme - for(Partition p: partitions){ - if(partitionInCustomLocation(tbl, p)) { - String message = String.format("UNARCHIVE cannot run for partition " + - "groups with custom locations like %s", p.getLocation()); - throw new HiveException(message); - } - } - originalDir = partSpecInfo.createPath(tbl); - } else { - Partition p = partitions.get(0); - if(ArchiveUtils.isArchived(p)) { - originalDir = new Path(getOriginalLocation(p)); - } else { - originalDir = new Path(p.getLocation()); - } - } - - URI originalUri = ArchiveUtils.addSlash(originalDir.toUri()); - Path intermediateArchivedDir = new Path(originalDir.getParent(), - originalDir.getName() + INTERMEDIATE_ARCHIVED_DIR_SUFFIX); - Path intermediateExtractedDir = new Path(originalDir.getParent(), - originalDir.getName() + INTERMEDIATE_EXTRACTED_DIR_SUFFIX); - boolean recovery = false; - if(pathExists(intermediateArchivedDir) || pathExists(intermediateExtractedDir)) { - recovery = true; - console.printInfo("Starting recovery after failed UNARCHIVE"); - } - - for(Partition p: partitions) { - checkArchiveProperty(partSpecLevel, recovery, p); - } - - String archiveName = "data.har"; - FileSystem fs = null; - try { - fs = originalDir.getFileSystem(conf); - } catch (IOException e) { - throw new HiveException(e); - } - - // assume the archive is in the original dir, check if it exists - Path archivePath = new Path(originalDir, archiveName); - URI archiveUri = archivePath.toUri(); - ArchiveUtils.HarPathHelper harHelper = new ArchiveUtils.HarPathHelper(conf, - archiveUri, originalUri); - URI sourceUri = harHelper.getHarUri(originalUri); - Path sourceDir = new Path(sourceUri.getScheme(), sourceUri.getAuthority(), sourceUri.getPath()); - - if(!pathExists(intermediateArchivedDir) && !pathExists(archivePath)) { - throw new HiveException("Haven't found any archive where it should be"); - } - - Path tmpPath = driverContext.getCtx().getExternalTmpPath(originalDir); - - try { - fs = tmpPath.getFileSystem(conf); - } catch (IOException e) { - throw new HiveException(e); - } - - // Clarification of terms: - // - The originalDir directory represents the original directory of the - // partitions' files. They now contain an archived version of those files - // eg. hdfs:/warehouse/myTable/ds=1/ - // - The source directory is the directory containing all the files that - // should be in the partitions. e.g. har:/warehouse/myTable/ds=1/myTable.har/ - // Note the har:/ scheme - - // Steps: - // 1. Extract the archive in a temporary folder - // 2. Move the archive dir to an intermediate dir that is in at the same - // dir as originalLocation. Call the new dir intermediate-extracted. - // 3. Rename the original partitions dir to an intermediate dir. Call the - // renamed dir intermediate-archive - // 4. Rename intermediate-extracted to the original partitions dir - // 5. Change the metadata - // 6. Delete the archived partitions files in intermediate-archive - - if (!pathExists(intermediateExtractedDir) && - !pathExists(intermediateArchivedDir)) { - try { - - // Copy the files out of the archive into the temporary directory - String copySource = sourceDir.toString(); - String copyDest = tmpPath.toString(); - List args = new ArrayList(); - args.add("-cp"); - args.add(copySource); - args.add(copyDest); - - console.printInfo("Copying " + copySource + " to " + copyDest); - FileSystem srcFs = FileSystem.get(sourceDir.toUri(), conf); - srcFs.initialize(sourceDir.toUri(), conf); - - FsShell fss = new FsShell(conf); - int ret = 0; - try { - ret = ToolRunner.run(fss, args.toArray(new String[0])); - } catch (Exception e) { - throw new HiveException(e); - } - - if (ret != 0) { - throw new HiveException("Error while copying files from archive, return code=" + ret); - } else { - console.printInfo("Successfully Copied " + copySource + " to " + copyDest); - } - - console.printInfo("Moving " + tmpPath + " to " + intermediateExtractedDir); - if (fs.exists(intermediateExtractedDir)) { - throw new HiveException("Invalid state: the intermediate extracted " + - "directory already exists."); - } - fs.rename(tmpPath, intermediateExtractedDir); - } catch (Exception e) { - throw new HiveException(e); - } - } - - // At this point, we know that the extracted files are in the intermediate - // extracted dir, or in the the original directory. - - if (!pathExists(intermediateArchivedDir)) { - try { - console.printInfo("Moving " + originalDir + " to " + intermediateArchivedDir); - fs.rename(originalDir, intermediateArchivedDir); - } catch (IOException e) { - throw new HiveException(e); - } - } else { - console.printInfo(intermediateArchivedDir + " already exists. " + - "Assuming it contains the archived version of the partition"); - } - - // If there is a failure from here to until when the metadata is changed, - // the partition will be empty or throw errors on read. - - // If the original location exists here, then it must be the extracted files - // because in the previous step, we moved the previous original location - // (containing the archived version of the files) to intermediateArchiveDir - if (!pathExists(originalDir)) { - try { - console.printInfo("Moving " + intermediateExtractedDir + " to " + originalDir); - fs.rename(intermediateExtractedDir, originalDir); - } catch (IOException e) { - throw new HiveException(e); - } - } else { - console.printInfo(originalDir + " already exists. " + - "Assuming it contains the extracted files in the partition"); - } - - for(Partition p: partitions) { - setUnArchived(p); - try { - // TODO: catalog - db.alterPartition(simpleDesc.getTableName(), p, null, true); - } catch (InvalidOperationException e) { - throw new HiveException(e); - } - } - - // If a failure happens here, the intermediate archive files won't be - // deleted. The user will need to call unarchive again to clear those up. - if(pathExists(intermediateArchivedDir)) { - deleteDir(intermediateArchivedDir, db.getDatabase(tbl.getDbName())); - } - - if(recovery) { - console.printInfo("Recovery after UNARCHIVE succeeded"); - } - - return 0; - } - - private void checkArchiveProperty(int partSpecLevel, - boolean recovery, Partition p) throws HiveException { - if (!ArchiveUtils.isArchived(p) && !recovery) { - throw new HiveException("Partition " + p.getName() - + " is not archived."); - } - int archiveLevel = ArchiveUtils.getArchivingLevel(p); - if (partSpecLevel > archiveLevel) { - throw new HiveException("Partition " + p.getName() - + " is archived at level " + archiveLevel - + ", and given partspec only has " + partSpecLevel - + " specs."); - } - } - - private int compact(Hive db, AlterTableSimpleDesc desc) throws HiveException { - - Table tbl = db.getTable(desc.getTableName()); - if (!AcidUtils.isTransactionalTable(tbl)) { - throw new HiveException(ErrorMsg.NONACID_COMPACTION_NOT_SUPPORTED, tbl.getDbName(), - tbl.getTableName()); - } - - String partName = null; - if (desc.getPartSpec() == null) { - // Compaction can only be done on the whole table if the table is non-partitioned. - if (tbl.isPartitioned()) { - throw new HiveException(ErrorMsg.NO_COMPACTION_PARTITION); - } - } else { - Map partSpec = desc.getPartSpec(); - List partitions = db.getPartitions(tbl, partSpec); - if (partitions.size() > 1) { - throw new HiveException(ErrorMsg.TOO_MANY_COMPACTION_PARTITIONS); - } else if (partitions.size() == 0) { - throw new HiveException(ErrorMsg.INVALID_PARTITION_SPEC); - } - partName = partitions.get(0).getName(); - } - CompactionResponse resp = db.compact2(tbl.getDbName(), tbl.getTableName(), partName, - desc.getCompactionType(), desc.getProps()); - if(resp.isAccepted()) { - console.printInfo("Compaction enqueued with id " + resp.getId()); - } - else { - console.printInfo("Compaction already enqueued with id " + resp.getId() + - "; State is " + resp.getState()); - } - if(desc.isBlocking() && resp.isAccepted()) { - StringBuilder progressDots = new StringBuilder(); - long waitTimeMs = 1000; - wait: while (true) { - //double wait time until 5min - waitTimeMs = waitTimeMs*2; - waitTimeMs = waitTimeMs < 5*60*1000 ? waitTimeMs : 5*60*1000; - try { - Thread.sleep(waitTimeMs); - } - catch(InterruptedException ex) { - console.printInfo("Interrupted while waiting for compaction with id=" + resp.getId()); - break; - } - //this could be expensive when there are a lot of compactions.... - //todo: update to search by ID once HIVE-13353 is done - ShowCompactResponse allCompactions = db.showCompactions(); - for(ShowCompactResponseElement compaction : allCompactions.getCompacts()) { - if (resp.getId() != compaction.getId()) { - continue; - } - switch (compaction.getState()) { - case TxnStore.WORKING_RESPONSE: - case TxnStore.INITIATED_RESPONSE: - //still working - console.printInfo(progressDots.toString()); - progressDots.append("."); - continue wait; - default: - //done - console.printInfo("Compaction with id " + resp.getId() + " finished with status: " + compaction.getState()); - break wait; - } - } - } - } - return 0; - } - /** * MetastoreCheck, see if the data in the metastore matches what is on the * dfs. Current version checks for tables and partitions that are either @@ -1082,130 +216,6 @@ private int msck(Hive db, MsckDesc msckDesc) { } } - /** - * Alter a given table. - * - * @param db - * The database in question. - * @param alterTbl - * This is the table we're altering. - * @return Returns 0 when execution succeeds and above 0 if it fails. - * @throws HiveException - * Throws this exception if an unexpected error occurs. - */ - private int alterTable(Hive db, AlterTableDesc alterTbl) throws HiveException { - if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.RENAME) { - String names[] = Utilities.getDbTableName(alterTbl.getOldName()); - if (Utils.isBootstrapDumpInProgress(db, names[0])) { - LOG.error("DDLTask: Rename Table not allowed as bootstrap dump in progress"); - throw new HiveException("Rename Table: Not allowed as bootstrap dump in progress"); - } - } - - // alter the table - Table tbl = db.getTable(alterTbl.getOldName()); - - List allPartitions = null; - if (alterTbl.getPartSpec() != null) { - Map partSpec = alterTbl.getPartSpec(); - if (DDLSemanticAnalyzer.isFullSpec(tbl, partSpec)) { - allPartitions = new ArrayList(); - Partition part = db.getPartition(tbl, partSpec, false); - if (part == null) { - // User provided a fully specified partition spec but it doesn't exist, fail. - throw new HiveException(ErrorMsg.INVALID_PARTITION, - StringUtils.join(alterTbl.getPartSpec().keySet(), ',') + " for table " + alterTbl.getOldName()); - - } - allPartitions.add(part); - } else { - // DDLSemanticAnalyzer has already checked if partial partition specs are allowed, - // thus we should not need to check it here. - allPartitions = db.getPartitions(tbl, alterTbl.getPartSpec()); - } - } - - // Don't change the table object returned by the metastore, as we'll mess with it's caches. - Table oldTbl = tbl; - tbl = oldTbl.copy(); - // Handle child tasks here. We could add them directly whereever we need, - // but let's make it a little bit more explicit. - if (allPartitions != null) { - // Alter all partitions - for (Partition part : allPartitions) { - addChildTasks(alterTableOrSinglePartition(alterTbl, tbl, part)); - } - } else { - // Just alter the table - addChildTasks(alterTableOrSinglePartition(alterTbl, tbl, null)); - } - - if (allPartitions == null) { - updateModifiedParameters(tbl.getTTable().getParameters(), conf); - tbl.checkValidity(conf); - } else { - for (Partition tmpPart: allPartitions) { - updateModifiedParameters(tmpPart.getParameters(), conf); - } - } - - try { - EnvironmentContext environmentContext = alterTbl.getEnvironmentContext(); - if (environmentContext == null) { - environmentContext = new EnvironmentContext(); - } - environmentContext.putToProperties(HiveMetaHook.ALTER_TABLE_OPERATION_TYPE, alterTbl.getOp().name()); - if (allPartitions == null) { - long writeId = alterTbl.getWriteId() != null ? alterTbl.getWriteId() : 0; - if (alterTbl.getReplicationSpec() != null && - alterTbl.getReplicationSpec().isMigratingToTxnTable()) { - Long tmpWriteId = ReplUtils.getMigrationCurrentTblWriteId(conf); - if (tmpWriteId == null) { - throw new HiveException("DDLTask : Write id is not set in the config by open txn task for migration"); - } - writeId = tmpWriteId; - } - db.alterTable(alterTbl.getOldName(), tbl, alterTbl.getIsCascade(), environmentContext, - true, writeId); - } else { - // Note: this is necessary for UPDATE_STATISTICS command, that operates via ADDPROPS (why?). - // For any other updates, we don't want to do txn check on partitions when altering table. - boolean isTxn = false; - if (alterTbl.getPartSpec() != null && alterTbl.getOp() == AlterTableTypes.ADDPROPS) { - // ADDPROPS is used to add replication properties like repl.last.id, which isn't - // transactional change. In case of replication check for transactional properties - // explicitly. - Map props = alterTbl.getProps(); - if (alterTbl.getReplicationSpec() != null && alterTbl.getReplicationSpec().isInReplicationScope()) { - isTxn = (props.get(StatsSetupConst.COLUMN_STATS_ACCURATE) != null); - } else { - isTxn = true; - } - } - db.alterPartitions(Warehouse.getQualifiedName(tbl.getTTable()), allPartitions, environmentContext, isTxn); - } - } catch (InvalidOperationException e) { - LOG.error("alter table: ", e); - throw new HiveException(e, ErrorMsg.GENERIC_ERROR); - } - - // This is kind of hacky - the read entity contains the old table, whereas - // the write entity - // contains the new table. This is needed for rename - both the old and the - // new table names are - // passed - // Don't acquire locks for any of these, we have already asked for them in DDLSemanticAnalyzer. - if (allPartitions != null ) { - for (Partition tmpPart: allPartitions) { - work.getInputs().add(new ReadEntity(tmpPart)); - addIfAbsentByName(new WriteEntity(tmpPart, WriteEntity.WriteType.DDL_NO_LOCK)); - } - } else { - work.getInputs().add(new ReadEntity(oldTbl)); - addIfAbsentByName(new WriteEntity(tbl, WriteEntity.WriteType.DDL_NO_LOCK)); - } - return 0; - } /** * There are many places where "duplicate" Read/WriteEnity objects are added. The way this was * initially implemented, the duplicate just replaced the previous object. @@ -1235,221 +245,6 @@ static boolean addIfAbsentByName(WriteEntity newWriteEntity, Set ou outputs.add(newWriteEntity); return true; } - private boolean addIfAbsentByName(WriteEntity newWriteEntity) { - return addIfAbsentByName(newWriteEntity, work.getOutputs()); - } - - private void addChildTasks(List> extraTasks) { - if (extraTasks == null) { - return; - } - for (Task newTask : extraTasks) { - addDependentTask(newTask); - } - } - - private boolean isSchemaEvolutionEnabled(Table tbl) { - boolean isAcid = AcidUtils.isTablePropertyTransactional(tbl.getMetadata()); - if (isAcid || HiveConf.getBoolVar(conf, ConfVars.HIVE_SCHEMA_EVOLUTION)) { - return true; - } - return false; - } - - - private static StorageDescriptor retrieveStorageDescriptor(Table tbl, Partition part) { - return (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd()); - } - - private List> alterTableOrSinglePartition(AlterTableDesc alterTbl, Table tbl, - Partition part) throws HiveException { - EnvironmentContext environmentContext = alterTbl.getEnvironmentContext(); - if (environmentContext == null) { - environmentContext = new EnvironmentContext(); - alterTbl.setEnvironmentContext(environmentContext); - } - // do not need update stats in alter table/partition operations - if (environmentContext.getProperties() == null || - environmentContext.getProperties().get(StatsSetupConst.DO_NOT_UPDATE_STATS) == null) { - environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); - } - - if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.RENAME) { - tbl.setDbName(Utilities.getDatabaseName(alterTbl.getNewName())); - tbl.setTableName(Utilities.getTableName(alterTbl.getNewName())); - } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDPROPS) { - return alterTableAddProps(alterTbl, tbl, part, environmentContext); - } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.DROPPROPS) { - return alterTableDropProps(alterTbl, tbl, part, environmentContext); - } else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.OWNER) { - if (alterTbl.getOwnerPrincipal() != null) { - tbl.setOwner(alterTbl.getOwnerPrincipal().getName()); - tbl.setOwnerType(alterTbl.getOwnerPrincipal().getType()); - } - } else { - throw new HiveException(ErrorMsg.UNSUPPORTED_ALTER_TBL_OP, alterTbl.getOp().toString()); - } - - return null; - } - - private List> alterTableDropProps(AlterTableDesc alterTbl, Table tbl, - Partition part, EnvironmentContext environmentContext) throws HiveException { - if (StatsSetupConst.USER.equals(environmentContext.getProperties() - .get(StatsSetupConst.STATS_GENERATED))) { - // drop a stats parameter, which triggers recompute stats update automatically - environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS); - } - - List> result = null; - if (part == null) { - Set removedSet = alterTbl.getProps().keySet(); - boolean isFromMmTable = AcidUtils.isInsertOnlyTable(tbl.getParameters()), - isRemoved = AcidUtils.isRemovedInsertOnlyTable(removedSet); - if (isFromMmTable && isRemoved) { - throw new HiveException("Cannot convert an ACID table to non-ACID"); - } - - // Check if external table property being removed - if (removedSet.contains("EXTERNAL") && tbl.getTableType() == TableType.EXTERNAL_TABLE) { - tbl.setTableType(TableType.MANAGED_TABLE); - } - } - Iterator keyItr = alterTbl.getProps().keySet().iterator(); - while (keyItr.hasNext()) { - if (part != null) { - part.getTPartition().getParameters().remove(keyItr.next()); - } else { - tbl.getTTable().getParameters().remove(keyItr.next()); - } - } - return result; - } - - private void checkMmLb(Table tbl) throws HiveException { - if (!tbl.isStoredAsSubDirectories()) { - return; - } - // TODO [MM gap?]: by design; no-one seems to use LB tables. They will work, but not convert. - // It's possible to work around this by re-creating and re-inserting the table. - throw new HiveException("Converting list bucketed tables stored as subdirectories " - + " to MM is not supported. Please re-create a table in the desired format."); - } - - private void checkMmLb(Partition part) throws HiveException { - if (!part.isStoredAsSubDirectories()) { - return; - } - throw new HiveException("Converting list bucketed tables stored as subdirectories " - + " to MM is not supported. Please re-create a table in the desired format."); - } - - private List> generateAddMmTasks(Table tbl, Long writeId) throws HiveException { - // We will move all the files in the table/partition directories into the first MM - // directory, then commit the first write ID. - List srcs = new ArrayList<>(), tgts = new ArrayList<>(); - if (writeId == null) { - throw new HiveException("Internal error - write ID not set for MM conversion"); - } - - int stmtId = 0; - String mmDir = AcidUtils.deltaSubdir(writeId, writeId, stmtId); - - Hive db = getHive(); - if (tbl.getPartitionKeys().size() > 0) { - PartitionIterable parts = new PartitionIterable(db, tbl, null, - HiveConf.getIntVar(conf, ConfVars.METASTORE_BATCH_RETRIEVE_MAX)); - Iterator partIter = parts.iterator(); - while (partIter.hasNext()) { - Partition part = partIter.next(); - checkMmLb(part); - Path src = part.getDataLocation(), tgt = new Path(src, mmDir); - srcs.add(src); - tgts.add(tgt); - if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { - Utilities.FILE_OP_LOGGER.trace("Will move " + src + " to " + tgt); - } - } - } else { - checkMmLb(tbl); - Path src = tbl.getDataLocation(), tgt = new Path(src, mmDir); - srcs.add(src); - tgts.add(tgt); - if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { - Utilities.FILE_OP_LOGGER.trace("Will move " + src + " to " + tgt); - } - } - // Don't set inputs and outputs - the locks have already been taken so it's pointless. - MoveWork mw = new MoveWork(null, null, null, null, false); - mw.setMultiFilesDesc(new LoadMultiFilesDesc(srcs, tgts, true, null, null)); - return Lists.>newArrayList(TaskFactory.get(mw)); - } - - private List> alterTableAddProps(AlterTableDesc alterTbl, Table tbl, - Partition part, EnvironmentContext environmentContext) throws HiveException { - if (StatsSetupConst.USER.equals(environmentContext.getProperties() - .get(StatsSetupConst.STATS_GENERATED))) { - environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS); - } - List> result = null; - if (part != null) { - part.getTPartition().getParameters().putAll(alterTbl.getProps()); - } else { - boolean isFromMmTable = AcidUtils.isInsertOnlyTable(tbl.getParameters()); - Boolean isToMmTable = AcidUtils.isToInsertOnlyTable(tbl, alterTbl.getProps()); - if (isToMmTable != null) { - if (!isFromMmTable && isToMmTable) { - if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_MM_ALLOW_ORIGINALS)) { - result = generateAddMmTasks(tbl, alterTbl.getWriteId()); - } else { - if (tbl.getPartitionKeys().size() > 0) { - Hive db = getHive(); - PartitionIterable parts = new PartitionIterable(db, tbl, null, - HiveConf.getIntVar(conf, ConfVars.METASTORE_BATCH_RETRIEVE_MAX)); - Iterator partIter = parts.iterator(); - while (partIter.hasNext()) { - Partition part0 = partIter.next(); - checkMmLb(part0); - } - } else { - checkMmLb(tbl); - } - } - } else if (isFromMmTable && !isToMmTable) { - throw new HiveException("Cannot convert an ACID table to non-ACID"); - } - } - - // Converting to/from external table - String externalProp = alterTbl.getProps().get("EXTERNAL"); - if (externalProp != null) { - if (Boolean.parseBoolean(externalProp) && tbl.getTableType() == TableType.MANAGED_TABLE) { - tbl.setTableType(TableType.EXTERNAL_TABLE); - } else if (!Boolean.parseBoolean(externalProp) && tbl.getTableType() == TableType.EXTERNAL_TABLE) { - tbl.setTableType(TableType.MANAGED_TABLE); - } - } - - tbl.getTTable().getParameters().putAll(alterTbl.getProps()); - } - return result; - } - - /** - * Update last_modified_by and last_modified_time parameters in parameter map. - * - * @param params - * Parameters. - * @param conf - * HiveConf of session - */ - private boolean updateModifiedParameters(Map params, HiveConf conf) throws HiveException { - String user = null; - user = SessionState.getUserFromAuthenticator(); - params.put("last_modified_by", user); - params.put("last_modified_time", Long.toString(System.currentTimeMillis() / 1000)); - return true; - } /** * Check if the given serde is valid. @@ -1477,45 +272,6 @@ public String getName() { return "DDL"; } - /** - * Validate if the given table/partition is eligible for update - * - * @param db Database. - * @param tableName Table name of format db.table - * @param partSpec Partition spec for the partition - * @param replicationSpec Replications specification - * - * @return boolean true if allow the operation - * @throws HiveException - */ - private boolean allowOperationInReplicationScope(Hive db, String tableName, - Map partSpec, ReplicationSpec replicationSpec) throws HiveException { - if ((null == replicationSpec) || (!replicationSpec.isInReplicationScope())) { - // Always allow the operation if it is not in replication scope. - return true; - } - // If the table/partition exist and is older than the event, then just apply - // the event else noop. - Table existingTable = db.getTable(tableName, false); - if ((existingTable != null) - && replicationSpec.allowEventReplacementInto(existingTable.getParameters())) { - // Table exists and is older than the update. Now, need to ensure if update allowed on the - // partition. - if (partSpec != null) { - Partition existingPtn = db.getPartition(existingTable, partSpec, false); - return ((existingPtn != null) - && replicationSpec.allowEventReplacementInto(existingPtn.getParameters())); - } - - // Replacement is allowed as the existing table is older than event - return true; - } - - // The table is missing either due to drop/rename which follows the operation. - // Or the existing table is newer than our update. So, don't allow the update. - return false; - } - private int remFirstIncPendFlag(Hive hive, ReplRemoveFirstIncLoadPendFlagDesc desc) throws HiveException, TException { String dbNameOrPattern = desc.getDatabaseName(); String tableNameOrPattern = desc.getTableName(); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java index 078691cd06..ae9d040ef8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ExportTask.java @@ -44,7 +44,7 @@ public String getName() { } @Override - protected int execute(DriverContext driverContext) { + public int execute(DriverContext driverContext) { try { // Also creates the root directory TableExport.Paths exportPaths = new TableExport.Paths( diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java index dcf569feb8..6b3635d05a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java @@ -133,7 +133,7 @@ private void renameFileCopiedFromCmPath(Path toPath, FileSystem dstFs, List rootTask = work.getRootTask(); if (rootTask != null) { rootTask.setChildTasks(null); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java index 13de791fb3..441d1eae74 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/repl/incremental/IncrementalLoadTasksBuilder.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.ddl.DDLWork2; import org.apache.hadoop.hive.ql.ddl.database.AlterDatabaseDesc; +import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableSetPropertiesDesc; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork; @@ -47,7 +48,6 @@ import org.apache.hadoop.hive.ql.parse.repl.load.UpdatedMetaDataTracker; import org.apache.hadoop.hive.ql.parse.repl.load.log.IncrementalLoadLogger; import org.apache.hadoop.hive.ql.parse.repl.load.message.MessageHandler; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc; import org.apache.hadoop.hive.ql.plan.DDLWork; import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork; import org.apache.hadoop.hive.ql.plan.ReplTxnWork; @@ -280,13 +280,11 @@ private boolean shouldReplayEvent(FileStatus dir, DumpType dumpType, String dbNa HashMap mapProp = new HashMap<>(); mapProp.put(ReplicationSpec.KEY.CURR_STATE_ID.toString(), replState); - AlterTableDesc alterTblDesc = new AlterTableDesc( - AlterTableDesc.AlterTableTypes.ADDPROPS, new ReplicationSpec(replState, replState)); - alterTblDesc.setProps(mapProp); - alterTblDesc.setOldName(StatsUtils.getFullyQualifiedTableName(dbName, tableName)); - alterTblDesc.setPartSpec((HashMap) partSpec); + String fqTableName = StatsUtils.getFullyQualifiedTableName(dbName, tableName); + AlterTableSetPropertiesDesc alterTblDesc = new AlterTableSetPropertiesDesc(fqTableName, partSpec, + new ReplicationSpec(replState, replState), false, mapProp, false, false, null); - Task updateReplIdTask = TaskFactory.get(new DDLWork(inputs, outputs, alterTblDesc), conf); + Task updateReplIdTask = TaskFactory.get(new DDLWork2(inputs, outputs, alterTblDesc), conf); // Link the update repl state task with dependency collection task if (preCursor != null) { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java index f9f13e1a4c..8627ba7d08 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java @@ -26,6 +26,8 @@ import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.ddl.DDLWork2; +import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableSetPropertiesDesc; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork; @@ -34,9 +36,7 @@ import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.repl.ReplLogger; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc; import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork; -import org.apache.hadoop.hive.ql.plan.DDLWork; import org.apache.hadoop.hive.ql.plan.ReplTxnWork; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; @@ -59,7 +59,6 @@ import static org.apache.hadoop.hive.ql.util.HiveStrictManagedMigration.TableMigrationOption.MANAGED; - public class ReplUtils { public static final String LAST_REPL_ID_KEY = "hive.repl.last.repl.id"; @@ -144,14 +143,10 @@ HashMap mapProp = new HashMap<>(); mapProp.put(REPL_CHECKPOINT_KEY, dumpRoot); - AlterTableDesc alterTblDesc = new AlterTableDesc(AlterTableDesc.AlterTableTypes.ADDPROPS); - alterTblDesc.setProps(mapProp); - alterTblDesc.setOldName( - StatsUtils.getFullyQualifiedTableName(tableDesc.getDatabaseName(), tableDesc.getTableName())); - if (partSpec != null) { - alterTblDesc.setPartSpec(partSpec); - } - return TaskFactory.get(new DDLWork(new HashSet<>(), new HashSet<>(), alterTblDesc), conf); + String fqTableName = StatsUtils.getFullyQualifiedTableName(tableDesc.getDatabaseName(), tableDesc.getTableName()); + AlterTableSetPropertiesDesc alterTblDesc = new AlterTableSetPropertiesDesc(fqTableName, partSpec, null, false, + mapProp, false, false, null); + return TaskFactory.get(new DDLWork2(new HashSet<>(), new HashSet<>(), alterTblDesc), conf); } public static boolean replCkptStatus(String dbName, Map props, String dumpRoot) diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java index 7f8f9a7631..e412f9482b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java @@ -20,10 +20,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.metadata.DummyPartition; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc; import java.io.Serializable; @@ -195,7 +195,7 @@ public boolean isTempURI() { * @param op Operation type from the alter table description * @return the write type this should use. */ - public static WriteType determineAlterTableWriteType(AlterTableDesc.AlterTableTypes op) { + public static WriteType determineAlterTableWriteType(AlterTableTypes op) { switch (op) { case RENAME_COLUMN: case CLUSTERED_BY: @@ -203,7 +203,7 @@ public static WriteType determineAlterTableWriteType(AlterTableDesc.AlterTableTy case NOT_CLUSTERED: case SET_FILE_FORMAT: case SET_SERDE: - case DROPPROPS: + case UNSET_PROPERTIES: case REPLACE_COLUMNS: case ARCHIVE: case UNARCHIVE: @@ -223,7 +223,7 @@ public static WriteType determineAlterTableWriteType(AlterTableDesc.AlterTableTy case ADDPARTITION: case SET_SERDE_PROPS: - case ADDPROPS: + case SET_PROPERTIES: case UPDATESTATS: return WriteType.DDL_SHARED; diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java index 76415cf7e2..cd7b69dc8a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/AcidExportSemanticAnalyzer.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hive.ql.ddl.DDLWork2; import org.apache.hadoop.hive.ql.ddl.table.creation.CreateTableLikeDesc; import org.apache.hadoop.hive.ql.ddl.table.creation.DropTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableSetPropertiesDesc; import org.apache.hadoop.hive.ql.exec.StatsTask; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; @@ -49,8 +50,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc; -import org.apache.hadoop.hive.ql.plan.DDLWork; import org.apache.hadoop.hive.ql.plan.ExportWork; import org.apache.hadoop.hive.ql.session.SessionState; @@ -189,12 +188,11 @@ private void analyzeAcidExport(ASTNode ast) throws SemanticException { // do it after populating temp table so that it's written as non-transactional table but // update props before export so that export archive metadata has these props. This way when // IMPORT is done for this archive and target table doesn't exist, it will be created as Acid. - AlterTableDesc alterTblDesc = new AlterTableDesc(AlterTableDesc.AlterTableTypes.ADDPROPS); Map mapProps = new HashMap<>(); mapProps.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, Boolean.TRUE.toString()); - alterTblDesc.setProps(mapProps); - alterTblDesc.setOldName(newTableName); - addExportTask(rootTasks, exportTask, TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc))); + AlterTableSetPropertiesDesc alterTblDesc = new AlterTableSetPropertiesDesc(newTableName, null, null, false, + mapProps, false, false, null); + addExportTask(rootTasks, exportTask, TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), alterTblDesc))); // Now make a task to drop temp table // {@link DDLSemanticAnalyzer#analyzeDropTable(ASTNode ast, TableType expectedType) @@ -254,7 +252,7 @@ private StringBuilder generateExportQuery(List partCols, ASTNode to * Makes the exportTask run after all other tasks of the "insert into T ..." are done. */ private void addExportTask(List> rootTasks, - Task exportTask, Task alterTable) { + Task exportTask, Task alterTable) { for (Task t : rootTasks) { if (t.getNumChild() <= 0) { //todo: ConditionalTask#addDependentTask(Task) doesn't do the right thing: HIVE-18978 diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java index a58ac2ffe9..698eb043d5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java @@ -71,6 +71,7 @@ import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.DDLDesc; import org.apache.hadoop.hive.ql.ddl.DDLWork2; import org.apache.hadoop.hive.ql.ddl.database.AlterDatabaseDesc; import org.apache.hadoop.hive.ql.ddl.database.CreateDatabaseDesc; @@ -93,6 +94,7 @@ import org.apache.hadoop.hive.ql.ddl.process.ShowCompactionsDesc; import org.apache.hadoop.hive.ql.ddl.process.ShowTransactionsDesc; import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.AlterTableTypes; import org.apache.hadoop.hive.ql.ddl.table.column.AlterTableAddColumnsDesc; import org.apache.hadoop.hive.ql.ddl.table.column.AlterTableChangeColumnDesc; import org.apache.hadoop.hive.ql.ddl.table.column.AlterTableReplaceColumnsDesc; @@ -110,6 +112,11 @@ import org.apache.hadoop.hive.ql.ddl.table.lock.LockTableDesc; import org.apache.hadoop.hive.ql.ddl.table.lock.ShowLocksDesc; import org.apache.hadoop.hive.ql.ddl.table.lock.UnlockTableDesc; +import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableRenameDesc; +import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableSetOwnerDesc; +import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableSetPropertiesDesc; +import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableTouchDesc; +import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableUnsetPropertiesDesc; import org.apache.hadoop.hive.ql.ddl.table.misc.TruncateTableDesc; import org.apache.hadoop.hive.ql.ddl.table.partition.AlterTableAddPartitionDesc; import org.apache.hadoop.hive.ql.ddl.table.partition.AlterTableAlterPartitionDesc; @@ -117,17 +124,21 @@ import org.apache.hadoop.hive.ql.ddl.table.partition.AlterTableExchangePartitionsDesc; import org.apache.hadoop.hive.ql.ddl.table.partition.AlterTableRenamePartitionDesc; import org.apache.hadoop.hive.ql.ddl.table.partition.ShowPartitionsDesc; +import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableArchiveDesc; import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableClusteredByDesc; +import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableCompactDesc; import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableIntoBucketsDesc; import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableNotClusteredDesc; import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableNotSkewedDesc; import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableNotSortedDesc; +import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableConcatenateDesc; import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableSetFileFormatDesc; import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableSetLocationDesc; import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableSetSerdeDesc; import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableSetSerdePropsDesc; import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableSetSkewedLocationDesc; import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableSkewedByDesc; +import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableUnarchiveDesc; import org.apache.hadoop.hive.ql.ddl.table.partition.AlterTableAddPartitionDesc.PartitionDesc; import org.apache.hadoop.hive.ql.ddl.view.AlterMaterializedViewRewriteDesc; import org.apache.hadoop.hive.ql.ddl.workloadmanagement.AlterPoolAddTriggerDesc; @@ -173,13 +184,9 @@ import org.apache.hadoop.hive.ql.parse.authorization.AuthorizationParseUtils; import org.apache.hadoop.hive.ql.parse.authorization.HiveAuthorizationTaskFactory; import org.apache.hadoop.hive.ql.parse.authorization.HiveAuthorizationTaskFactoryImpl; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; -import org.apache.hadoop.hive.ql.plan.AlterTableSimpleDesc; import org.apache.hadoop.hive.ql.plan.BasicStatsWork; import org.apache.hadoop.hive.ql.plan.CacheMetadataDesc; import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork; -import org.apache.hadoop.hive.ql.plan.DDLDesc; import org.apache.hadoop.hive.ql.plan.DDLDesc.DDLDescWithWriteId; import org.apache.hadoop.hive.ql.plan.DDLWork; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -228,7 +235,7 @@ private final HiveAuthorizationTaskFactory hiveAuthorizationTaskFactory; private WriteEntity alterTableOutput; // Equivalent to acidSinks, but for DDL operations that change data. - private DDLDesc.DDLDescWithWriteId ddlDescWithWriteId; + private DDLDescWithWriteId ddlDescWithWriteId; static { TokenToTypeName.put(HiveParser.TOK_BOOLEAN, serdeConstants.BOOLEAN_TYPE_NAME); @@ -683,7 +690,7 @@ private void analyzeAlterTableUpdateStats(ASTNode ast, String tblName, Map tableParams = tab.getTTable().getParameters(); + for (String currKey : mapProp.keySet()) { + if (!tableParams.containsKey(currKey)) { + String errorMsg = "The following property " + currKey + " does not exist in " + tab.getTableName(); + throw new SemanticException( + ErrorMsg.ALTER_TBL_UNSET_NON_EXIST_PROPERTY.getMsg(errorMsg)); + } + } } + + alterTblDesc = new AlterTableUnsetPropertiesDesc(tableName, partSpec, null, expectView, mapProp, + isExplicitStatsUpdate, environmentContext); + addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, alterTblDesc.getType(), isToTxn); + ddlWork = new DDLWork2(getInputs(), getOutputs(), alterTblDesc); } else { addPropertyReadEntry(mapProp, inputs); - alterTblDesc = new AlterTableDesc(AlterTableTypes.ADDPROPS, partSpec, expectView); + boolean isAcidConversion = isToTxn && AcidUtils.isFullAcidTable(mapProp) + && !AcidUtils.isFullAcidTable(getTable(qualified, true)); + alterTblDesc = new AlterTableSetPropertiesDesc(tableName, partSpec, null, expectView, mapProp, + isExplicitStatsUpdate, isAcidConversion, environmentContext); + addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, alterTblDesc.getType(), isToTxn); + ddlWork = new DDLWork2(getInputs(), getOutputs(), alterTblDesc); } - alterTblDesc.setProps(mapProp); - alterTblDesc.setEnvironmentContext(environmentContext); - alterTblDesc.setOldName(tableName); - - - - boolean isToTxn = AcidUtils.isTablePropertyTransactional(mapProp) - || mapProp.containsKey(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES); - addInputsOutputsAlterTable(tableName, partSpec, alterTblDesc, isToTxn); - // This special handling is because we cannot generate write ID for full ACID conversion, - // it will break the weird 10000001-write-ID logic that is currently in use. However, we do - // want to generate a write ID for prop changes for existing txn tables, or MM conversion. - boolean isAcidConversion = isToTxn && AcidUtils.isFullAcidTable(mapProp) - && !AcidUtils.isFullAcidTable(getTable(qualified, true)); - - DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), alterTblDesc); if (isToTxn) { - alterTblDesc.setIsFullAcidConversion(isAcidConversion); - setAcidDdlDesc(alterTblDesc); ddlWork.setNeedLock(true); // Hmm... why don't many other operations here need locks? } - if (changeStatsSucceeded) { - Table table = getTable(qualified, true); - if (AcidUtils.isTransactionalTable(table)) { - alterTblDesc.setIsExplicitStatsUpdate(true); - setAcidDdlDesc(alterTblDesc); - } + if (isToTxn || isExplicitStatsUpdate) { + setAcidDdlDesc(alterTblDesc); } rootTasks.add(TaskFactory.get(ddlWork)); @@ -1864,19 +1873,6 @@ private void analyzeAlterTableFileFormat(ASTNode ast, String tableName, Map partSpec, - AlterTableTypes op) throws SemanticException { - addInputsOutputsAlterTable(tableName, partSpec, (AlterTableDesc)null, op, false); - } - - private void addInputsOutputsAlterTable(String tableName, Map partSpec, - AlterTableDesc desc, boolean doForceExclusive) throws SemanticException { - addInputsOutputsAlterTable(tableName, partSpec, desc, desc.getOp(), doForceExclusive); - } - - private void addInputsOutputsAlterTable(String tableName, Map partSpec, - AlterTableDesc desc) throws SemanticException { - addInputsOutputsAlterTable(tableName, partSpec, desc, desc.getOp(), false); - } - - private void addInputsOutputsAlterTable(String tableName, Map partSpec, - AlterTableDesc desc, AlterTableTypes op, boolean doForceExclusive) throws SemanticException { - boolean isCascade = desc != null && desc.getIsCascade(); - boolean alterPartitions = partSpec != null && !partSpec.isEmpty(); - //cascade only occurs at table level then cascade to partition level - if (isCascade && alterPartitions) { - throw new SemanticException( - ErrorMsg.ALTER_TABLE_PARTITION_CASCADE_NOT_SUPPORTED, op.getName()); - } - - Table tab = getTable(tableName, true); - // cascade only occurs with partitioned table - if (isCascade && !tab.isPartitioned()) { - throw new SemanticException( - ErrorMsg.ALTER_TABLE_NON_PARTITIONED_TABLE_CASCADE_NOT_SUPPORTED); - } - - // Determine the lock type to acquire - WriteEntity.WriteType writeType = doForceExclusive - ? WriteType.DDL_EXCLUSIVE : determineAlterTableWriteType(tab, desc, op); - - if (!alterPartitions) { - inputs.add(new ReadEntity(tab)); - alterTableOutput = new WriteEntity(tab, writeType); - outputs.add(alterTableOutput); - //do not need the lock for partitions since they are covered by the table lock - if (isCascade) { - for (Partition part : getPartitions(tab, partSpec, false)) { - outputs.add(new WriteEntity(part, WriteEntity.WriteType.DDL_NO_LOCK)); - } - } - } else { - ReadEntity re = new ReadEntity(tab); - // In the case of altering a table for its partitions we don't need to lock the table - // itself, just the partitions. But the table will have a ReadEntity. So mark that - // ReadEntity as no lock. - re.noLockNeeded(); - inputs.add(re); - - if (isFullSpec(tab, partSpec)) { - // Fully specified partition spec - Partition part = getPartition(tab, partSpec, true); - outputs.add(new WriteEntity(part, writeType)); - } else { - // Partial partition spec supplied. Make sure this is allowed. - if (!AlterTableDesc.doesAlterTableTypeSupportPartialPartitionSpec(op)) { - throw new SemanticException( - ErrorMsg.ALTER_TABLE_TYPE_PARTIAL_PARTITION_SPEC_NO_SUPPORTED, op.getName()); - } else if (!conf.getBoolVar(HiveConf.ConfVars.DYNAMICPARTITIONING)) { - throw new SemanticException(ErrorMsg.DYNAMIC_PARTITION_DISABLED); - } - - for (Partition part : getPartitions(tab, partSpec, true)) { - outputs.add(new WriteEntity(part, writeType)); - } - } - } - - if (desc != null) { - validateAlterTableType(tab, op, desc.getExpectView()); - - // validate Unset Non Existed Table Properties - if (op == AlterTableDesc.AlterTableTypes.DROPPROPS && !desc.getIsDropIfExists()) { - Map tableParams = tab.getTTable().getParameters(); - for (String currKey : desc.getProps().keySet()) { - if (!tableParams.containsKey(currKey)) { - String errorMsg = - "The following property " + currKey + - " does not exist in " + tab.getTableName(); - throw new SemanticException( - ErrorMsg.ALTER_TBL_UNSET_NON_EXIST_PROPERTY.getMsg(errorMsg)); - } - } - } - } - } - // For the time while all the alter table operations are getting migrated there is a duplication of this method here private void addInputsOutputsAlterTable(String tableName, Map partSpec, AbstractAlterTableDesc desc, AlterTableTypes op, boolean doForceExclusive) throws SemanticException { boolean isCascade = desc != null && desc.isCascade(); @@ -2029,7 +1933,7 @@ private void addInputsOutputsAlterTable(String tableName, Map pa outputs.add(new WriteEntity(part, writeType)); } else { // Partial partition spec supplied. Make sure this is allowed. - if (!AlterTableDesc.doesAlterTableTypeSupportPartialPartitionSpec(op)) { + if (!AlterTableTypes.SUPPORT_PARTIAL_PARTITION_SPEC.contains(op)) { throw new SemanticException( ErrorMsg.ALTER_TABLE_TYPE_PARTIAL_PARTITION_SPEC_NO_SUPPORTED, op.getName()); } else if (!conf.getBoolVar(HiveConf.ConfVars.DYNAMICPARTITIONING)) { @@ -2044,20 +1948,6 @@ private void addInputsOutputsAlterTable(String tableName, Map pa if (desc != null) { validateAlterTableType(tab, op, desc.expectView()); - - // validate Unset Non Existed Table Properties -/* if (op == AlterTableDesc.AlterTableTypes.DROPPROPS && !desc.getIsDropIfExists()) { - Map tableParams = tab.getTTable().getParameters(); - for (String currKey : desc.getProps().keySet()) { - if (!tableParams.containsKey(currKey)) { - String errorMsg = - "The following property " + currKey + - " does not exist in " + tab.getTableName(); - throw new SemanticException( - ErrorMsg.ALTER_TBL_UNSET_NON_EXIST_PROPERTY.getMsg(errorMsg)); - } - } - }*/ } } @@ -2072,8 +1962,8 @@ private void analyzeAlterTableOwner(ASTNode ast, String tableName) throws Semant throw new SemanticException("Owner name can't be null in alter table set owner command"); } - AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, ownerPrincipal); - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc), conf)); + AlterTableSetOwnerDesc alterTblDesc = new AlterTableSetOwnerDesc(tableName, ownerPrincipal); + rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), alterTblDesc), conf)); } private void analyzeAlterTableLocation(ASTNode ast, String tableName, Map partSpec) @@ -2103,7 +1993,7 @@ private void analyzeAlterTableLocation(ASTNode ast, String tableName, Map partSpec) throws SemanticException { - AlterTablePartMergeFilesDesc mergeDesc = new AlterTablePartMergeFilesDesc( + AlterTableConcatenateDesc mergeDesc = new AlterTableConcatenateDesc( tableName, partSpec); List inputDir = new ArrayList(); @@ -2122,10 +2012,9 @@ private void analyzeAlterTablePartMergeFiles(ASTNode ast, boolean isBlocking = !HiveConf.getBoolVar(conf, ConfVars.TRANSACTIONAL_CONCATENATE_NOBLOCK, false); - AlterTableSimpleDesc desc = new AlterTableSimpleDesc( - tableName, newPartSpec, "MAJOR", isBlocking); + AlterTableCompactDesc desc = new AlterTableCompactDesc(tableName, newPartSpec, "MAJOR", isBlocking, null); - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); + rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), desc))); return; } mergeDesc.setTableDesc(Utilities.getTableDesc(tblObj)); @@ -2205,8 +2094,8 @@ private void analyzeAlterTablePartMergeFiles(ASTNode ast, mergeDesc.setLbCtx(lbCtx); - addInputsOutputsAlterTable(tableName, partSpec, AlterTableTypes.MERGEFILES); - DDLWork ddlWork = new DDLWork(getInputs(), getOutputs(), mergeDesc); + addInputsOutputsAlterTable(tableName, partSpec, null, AlterTableTypes.MERGEFILES, false); + DDLWork2 ddlWork = new DDLWork2(getInputs(), getOutputs(), mergeDesc); ddlWork.setNeedLock(true); Task mergeTask = TaskFactory.get(ddlWork); TableDesc tblDesc = Utilities.getTableDesc(tblObj); @@ -2312,11 +2201,8 @@ private void analyzeAlterTableCompact(ASTNode ast, String tableName, break; } } - AlterTableSimpleDesc desc = new AlterTableSimpleDesc( - tableName, newPartSpec, type, isBlocking); - desc.setProps(mapProp); - - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); + AlterTableCompactDesc desc = new AlterTableCompactDesc(tableName, newPartSpec, type, isBlocking, mapProp); + rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), desc))); } private void analyzeAlterTableDropConstraint(ASTNode ast, String tableName) @@ -3251,14 +3137,13 @@ private void analyzeAlterTableRename(String[] source, ASTNode ast, boolean expec String sourceName = getDotName(source); String targetName = getDotName(target); - AlterTableDesc alterTblDesc = new AlterTableDesc(sourceName, targetName, expectView, null); + AlterTableRenameDesc alterTblDesc = new AlterTableRenameDesc(sourceName, null, expectView, targetName); Table table = getTable(sourceName, true); if (AcidUtils.isTransactionalTable(table)) { setAcidDdlDesc(alterTblDesc); } - addInputsOutputsAlterTable(sourceName, null, alterTblDesc); - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), - alterTblDesc))); + addInputsOutputsAlterTable(sourceName, null, alterTblDesc, alterTblDesc.getType(), false); + rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), alterTblDesc))); } private void analyzeAlterTableRenameCol(String catName, String[] qualified, ASTNode ast, @@ -3774,21 +3659,15 @@ private void analyzeAlterTableTouch(String[] qualified, CommonTree ast) // partition name to value List> partSpecs = getPartitionSpecs(tab, ast); - if (partSpecs.size() == 0) { - AlterTableSimpleDesc touchDesc = new AlterTableSimpleDesc( - getDotName(qualified), null, - AlterTableDesc.AlterTableTypes.TOUCH); + if (partSpecs.isEmpty()) { + AlterTableTouchDesc touchDesc = new AlterTableTouchDesc(getDotName(qualified), null); outputs.add(new WriteEntity(tab, WriteEntity.WriteType.DDL_NO_LOCK)); - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), - touchDesc))); + rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), touchDesc))); } else { addTablePartsOutputs(tab, partSpecs, WriteEntity.WriteType.DDL_NO_LOCK); for (Map partSpec : partSpecs) { - AlterTableSimpleDesc touchDesc = new AlterTableSimpleDesc( - getDotName(qualified), partSpec, - AlterTableDesc.AlterTableTypes.TOUCH); - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), - touchDesc))); + AlterTableTouchDesc touchDesc = new AlterTableTouchDesc(getDotName(qualified), partSpec); + rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), touchDesc))); } } } @@ -3823,12 +3702,13 @@ private void analyzeAlterTableArchive(String[] qualified, CommonTree ast, boolea } catch (HiveException e) { throw new SemanticException(e.getMessage(), e); } - AlterTableSimpleDesc archiveDesc = new AlterTableSimpleDesc( - getDotName(qualified), partSpec, - (isUnArchive ? AlterTableTypes.UNARCHIVE : AlterTableTypes.ARCHIVE)); - rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), - archiveDesc))); - + DDLDesc archiveDesc = null; + if (isUnArchive) { + archiveDesc = new AlterTableUnarchiveDesc(getDotName(qualified), partSpec); + } else { + archiveDesc = new AlterTableArchiveDesc(getDotName(qualified), partSpec); + } + rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), archiveDesc))); } /** diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 808c5c1350..51e78fab23 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -103,6 +103,7 @@ import org.apache.hadoop.hive.ql.ddl.DDLWork2; import org.apache.hadoop.hive.ql.ddl.table.creation.CreateTableDesc; import org.apache.hadoop.hive.ql.ddl.table.creation.CreateTableLikeDesc; +import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableUnsetPropertiesDesc; import org.apache.hadoop.hive.ql.ddl.table.misc.PreInsertTableDesc; import org.apache.hadoop.hive.ql.ddl.view.CreateViewDesc; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; @@ -195,8 +196,6 @@ import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowSpec; import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowType; import org.apache.hadoop.hive.ql.plan.AggregationDesc; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; import org.apache.hadoop.hive.ql.plan.DDLWork; import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; @@ -6947,13 +6946,11 @@ private void genPartnCols(String dest, Operator input, QB qb, private void setStatsForNonNativeTable(String dbName, String tableName) throws SemanticException { String qTableName = DDLSemanticAnalyzer.getDotName(new String[] { dbName, tableName }); - AlterTableDesc alterTblDesc = new AlterTableDesc(AlterTableTypes.DROPPROPS, null, false); HashMap mapProp = new HashMap<>(); mapProp.put(StatsSetupConst.COLUMN_STATS_ACCURATE, null); - alterTblDesc.setOldName(qTableName); - alterTblDesc.setProps(mapProp); - alterTblDesc.setDropIfExists(true); - this.rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), alterTblDesc))); + AlterTableUnsetPropertiesDesc alterTblDesc = new AlterTableUnsetPropertiesDesc(qTableName, null, null, false, + mapProp, false, null); + this.rootTasks.add(TaskFactory.get(new DDLWork2(getInputs(), getOutputs(), alterTblDesc))); } @@ -12576,7 +12573,7 @@ void analyzeInternal(ASTNode ast, PlannerContextFactory pcf) throws SemanticExce rootTasks.stream() .filter(task -> task.getWork() instanceof DDLWork2) .map(task -> (DDLWork2) task.getWork()) - .filter(ddlWork -> ddlWork.getDDLDesc() != null) + .filter(ddlWork -> ddlWork.getDDLDesc() instanceof PreInsertTableDesc) .map(ddlWork -> (PreInsertTableDesc)ddlWork.getDDLDesc()) .map(ddlPreInsertTask -> new InsertCommitHookDesc(ddlPreInsertTask.getTable(), ddlPreInsertTask.isOverwrite())) diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java index 53d998200c..ee6ee8b6a6 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java @@ -19,13 +19,13 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage; +import org.apache.hadoop.hive.ql.ddl.DDLWork2; +import org.apache.hadoop.hive.ql.ddl.table.misc.AlterTableRenameDesc; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.AlterTableDesc; -import org.apache.hadoop.hive.ql.plan.DDLWork; import org.apache.hadoop.hive.ql.stats.StatsUtils; import java.io.Serializable; @@ -66,11 +66,10 @@ if (ReplUtils.isTableMigratingToTransactional(context.hiveConf, tableObjAfter)) { replicationSpec.setMigratingToTxnTable(); } - AlterTableDesc renameTableDesc = new AlterTableDesc( - oldName, newName, false, replicationSpec); + AlterTableRenameDesc renameTableDesc = new AlterTableRenameDesc(oldName, replicationSpec, false, newName); renameTableDesc.setWriteId(msg.getWriteId()); - Task renameTableTask = TaskFactory.get( - new DDLWork(readEntitySet, writeEntitySet, renameTableDesc), context.hiveConf); + Task renameTableTask = TaskFactory.get( + new DDLWork2(readEntitySet, writeEntitySet, renameTableDesc), context.hiveConf); context.log.debug("Added rename table task : {}:{}->{}", renameTableTask.getId(), oldName, newName); diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java deleted file mode 100644 index 9aa7e73fa7..0000000000 --- ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -import org.apache.hadoop.hive.ql.io.AcidUtils; - -import org.apache.hadoop.hive.metastore.api.EnvironmentContext; -import org.apache.hadoop.hive.ql.ddl.privilege.PrincipalDesc; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.ql.parse.ReplicationSpec; -import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.plan.Explain.Level; -import com.google.common.collect.ImmutableList; -import java.io.Serializable; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * AlterTableDesc. - * - */ -@Explain(displayName = "Alter Table", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) -public class AlterTableDesc extends DDLDesc implements Serializable, DDLDesc.DDLDescWithWriteId { - private static final long serialVersionUID = 1L; - - /** - * alterTableTypes. - * - */ - public static enum AlterTableTypes { - RENAME("rename"), ADD_COLUMNS("add columns"), REPLACE_COLUMNS("replace columns"), - ADDPROPS("add props"), DROPPROPS("drop props"), SET_SERDE("set serde"), SET_SERDE_PROPS("set serde props"), - SET_FILE_FORMAT("add fileformat"), CLUSTERED_BY("clustered by"), NOT_SORTED("not sorted"), - NOT_CLUSTERED("not clustered"), - RENAME_COLUMN("rename column"), ADDPARTITION("add partition"), TOUCH("touch"), ARCHIVE("archieve"), - UNARCHIVE("unarchieve"), SET_LOCATION("set location"), - DROPPARTITION("drop partition"), - RENAMEPARTITION("rename partition"), // Note: used in RenamePartitionDesc, not here. - SKEWED_BY("skewed by"), NOT_SKEWED("not skewed"), - SET_SKEWED_LOCATION("alter skew location"), INTO_BUCKETS("alter bucket number"), - ALTERPARTITION("alter partition"), // Note: this is never used in AlterTableDesc. - COMPACT("compact"), - TRUNCATE("truncate"), MERGEFILES("merge files"), DROP_CONSTRAINT("drop constraint"), - ADD_CONSTRAINT("add constraint"), - UPDATE_COLUMNS("update columns"), OWNER("set owner"), - UPDATESTATS("update stats"); // Note: used in ColumnStatsUpdateWork, not here. - ; - - private final String name; - private AlterTableTypes(String name) { this.name = name; } - public String getName() { return name; } - - public static final List nonNativeTableAllowedTypes = - ImmutableList.of(ADDPROPS, DROPPROPS, ADD_COLUMNS); - } - - public static enum ProtectModeType { - NO_DROP, OFFLINE, READ_ONLY, NO_DROP_CASCADE - } - - public static final Set alterTableTypesWithPartialSpec = - new HashSet(); - - static { - alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.ADD_COLUMNS); - alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.REPLACE_COLUMNS); - alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.RENAME_COLUMN); - alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.ADDPROPS); - alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.DROPPROPS); - alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.SET_SERDE); - alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.SET_SERDE_PROPS); - alterTableTypesWithPartialSpec.add(AlterTableDesc.AlterTableTypes.SET_FILE_FORMAT); - } - - AlterTableTypes op; - String oldName; - String newName; - Map props; - - boolean expectView; - HashMap partSpec; - boolean protectModeEnable; - ProtectModeType protectModeType; - boolean isTurnOffSkewed = false; - boolean isDropIfExists = false; - boolean isCascade = false; - EnvironmentContext environmentContext; - ReplicationSpec replicationSpec; - private Long writeId = null; - PrincipalDesc ownerPrincipal; - private boolean isExplicitStatsUpdate, isFullAcidConversion; - - public AlterTableDesc() { - } - - /** - * @param oldName - * old name of the table - * @param newName - * new name of the table - * @param expectView - * Flag to denote if current table can be a view - * @param replicationSpec - * Replication specification with current event ID - * @throws SemanticException - */ - public AlterTableDesc(String oldName, String newName, boolean expectView, ReplicationSpec replicationSpec) throws SemanticException { - op = AlterTableTypes.RENAME; - setOldName(oldName); - this.newName = newName; - this.expectView = expectView; - this.replicationSpec = replicationSpec; - } - - /** - * @param alterType - * type of alter op - * @param replicationSpec - * Replication specification with current event ID - */ - public AlterTableDesc(AlterTableTypes alterType, ReplicationSpec replicationSpec) { - op = alterType; - this.replicationSpec = replicationSpec; - } - - /** - * @param alterType - * type of alter op - */ - public AlterTableDesc(AlterTableTypes alterType) { - this(alterType, null, false); - } - - /** - * @param alterType - * type of alter op - * @param expectView - * Flag to denote if current table can be a view - * @param partSpec - * Partition specifier with map of key and values. - */ - public AlterTableDesc(AlterTableTypes alterType, HashMap partSpec, boolean expectView) { - op = alterType; - this.partSpec = partSpec; - this.expectView = expectView; - } - - public AlterTableDesc(String tableName, PrincipalDesc ownerPrincipal) { - op = AlterTableTypes.OWNER; - this.oldName = tableName; - this.ownerPrincipal = ownerPrincipal; - } - - /** - * @param ownerPrincipal the owner principal of the table - */ - public void setOwnerPrincipal(PrincipalDesc ownerPrincipal) { - this.ownerPrincipal = ownerPrincipal; - } - - @Explain(displayName="owner") - public PrincipalDesc getOwnerPrincipal() { - return this.ownerPrincipal; - } - - @Explain(displayName = "type", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) - public String getAlterTableTypeString() { - return op.getName(); - } - - /** - * @return the old name of the table - */ - @Explain(displayName = "old name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) - public String getOldName() { - return oldName; - } - - /** - * @param oldName - * the oldName to set - */ - public void setOldName(String oldName) throws SemanticException { - // Make sure we qualify the name from the outset so there's no ambiguity. - this.oldName = String.join(".", Utilities.getDbTableName(oldName)); - } - - /** - * @return the newName - */ - @Explain(displayName = "new name", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) - public String getNewName() { - return newName; - } - - /** - * @param newName - * the newName to set - */ - public void setNewName(String newName) { - this.newName = newName; - } - - /** - * @return the op - */ - public AlterTableTypes getOp() { - return op; - } - - /** - * @param op - * the op to set - */ - public void setOp(AlterTableTypes op) { - this.op = op; - } - - /** - * @return the props - */ - @Explain(displayName = "properties") - public Map getProps() { - return props; - } - - /** - * @param props - * the props to set - */ - public void setProps(Map props) { - this.props = props; - } - - /** - * @return whether to expect a view being altered - */ - public boolean getExpectView() { - return expectView; - } - - /** - * @param expectView - * set whether to expect a view being altered - */ - public void setExpectView(boolean expectView) { - this.expectView = expectView; - } - - /** - * @return part specification - */ - public HashMap getPartSpec() { - return partSpec; - } - - /** - * @param partSpec - */ - public void setPartSpec(HashMap partSpec) { - this.partSpec = partSpec; - } - - public boolean isProtectModeEnable() { - return protectModeEnable; - } - - public void setProtectModeEnable(boolean protectModeEnable) { - this.protectModeEnable = protectModeEnable; - } - - public ProtectModeType getProtectModeType() { - return protectModeType; - } - - public void setProtectModeType(ProtectModeType protectModeType) { - this.protectModeType = protectModeType; - } - - /** - * @return the turnOffSkewed - */ - public boolean isTurnOffSkewed() { - return isTurnOffSkewed; - } - - /** - * @param turnOffSkewed the turnOffSkewed to set - */ - public void setTurnOffSkewed(boolean turnOffSkewed) { - this.isTurnOffSkewed = turnOffSkewed; - } - - /** - * @param isDropIfExists the isDropIfExists to set - */ - public void setDropIfExists(boolean isDropIfExists) { - this.isDropIfExists = isDropIfExists; - } - - /** - * @return isDropIfExists - */ - public boolean getIsDropIfExists() { - return isDropIfExists; - } - - /** - * @return isCascade - */ - public boolean getIsCascade() { - return isCascade; - } - - /** - * @param isCascade the isCascade to set - */ - public void setIsCascade(boolean isCascade) { - this.isCascade = isCascade; - } - - public static boolean doesAlterTableTypeSupportPartialPartitionSpec(AlterTableTypes type) { - return alterTableTypesWithPartialSpec.contains(type); - } - - public EnvironmentContext getEnvironmentContext() { - return environmentContext; - } - - public void setEnvironmentContext(EnvironmentContext environmentContext) { - this.environmentContext = environmentContext; - } - - /** - * @return what kind of replication scope this alter is running under. - * This can result in a "ALTER IF NEWER THAN" kind of semantic - */ - public ReplicationSpec getReplicationSpec(){ return this.replicationSpec; } - - @Override - public void setWriteId(long writeId) { - this.writeId = writeId; - } - - @Override - public String getFullTableName() { - return getOldName(); - } - - @Override - public boolean mayNeedWriteId() { - switch (getOp()) { - case ADDPROPS: { - return isExplicitStatsUpdate || AcidUtils.isToInsertOnlyTable(null, getProps()) - || (AcidUtils.isTransactionalTable(getProps()) && !isFullAcidConversion); - } - case DROPPROPS: return isExplicitStatsUpdate; - // The check for the following ones is performed before setting AlterTableDesc into the acid field. - // These need write ID and stuff because they invalidate column stats. - case RENAME_COLUMN: - case RENAME: - case REPLACE_COLUMNS: - case ADD_COLUMNS: - case SET_LOCATION: - case UPDATE_COLUMNS: return true; - // RENAMEPARTITION is handled in RenamePartitionDesc - default: return false; - } - } - - public Long getWriteId() { - return this.writeId; - } - - @Override - public String toString() { - return this.getClass().getSimpleName() + " for " + getFullTableName(); - } - - public void setIsExplicitStatsUpdate(boolean b) { - this.isExplicitStatsUpdate = b; - } - - public void setIsFullAcidConversion(boolean b) { - this.isFullAcidConversion = b; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableSimpleDesc.java ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableSimpleDesc.java deleted file mode 100644 index 759a14f95c..0000000000 --- ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableSimpleDesc.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.plan; - -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; - -import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes; - -/** - * Contains information needed to modify a partition or a table - */ -public class AlterTableSimpleDesc extends DDLDesc { - private String tableName; - private LinkedHashMap partSpec; - private String compactionType; - private boolean isBlocking = false; - - AlterTableTypes type; - private Map props; - - public AlterTableSimpleDesc() { - } - - /** - * @param tableName - * table containing the partition - * @param partSpec - */ - public AlterTableSimpleDesc(String tableName, - Map partSpec, AlterTableTypes type) { - this.tableName = tableName; - if(partSpec == null) { - this.partSpec = null; - } else { - this.partSpec = new LinkedHashMap(partSpec); - } - this.type = type; - } - - /** - * Constructor for ALTER TABLE ... COMPACT. - * @param tableName name of the table to compact - * @param partSpec partition to compact - * @param compactionType currently supported values: 'major' and 'minor' - */ - public AlterTableSimpleDesc(String tableName, - LinkedHashMap partSpec, String compactionType, boolean isBlocking) { - type = AlterTableTypes.COMPACT; - this.compactionType = compactionType; - this.tableName = tableName; - this.partSpec = partSpec; - this.isBlocking = isBlocking; - } - - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public AlterTableDesc.AlterTableTypes getType() { - return type; - } - - public void setType(AlterTableDesc.AlterTableTypes type) { - this.type = type; - } - - public LinkedHashMap getPartSpec() { - return partSpec; - } - - public void setPartSpec(LinkedHashMap partSpec) { - this.partSpec = partSpec; - } - - /** - * Get what type of compaction is being done by a ALTER TABLE ... COMPACT statement. - * @return Compaction type, currently supported values are 'major' and 'minor'. - */ - public String getCompactionType() { - return compactionType; - } - - /** - * if compaction request should block until completion - */ - public boolean isBlocking() { - return isBlocking; - } - - public Map getProps() { - return props; - } - - public void setProps(Map props) { - this.props = props; - } -} diff --git ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java index 07feae32e7..639aaba9e8 100644 --- ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java +++ ql/src/java/org/apache/hadoop/hive/ql/plan/DDLWork.java @@ -19,7 +19,6 @@ import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; -import org.apache.hadoop.hive.ql.parse.AlterTablePartMergeFilesDesc; import org.apache.hadoop.hive.ql.plan.Explain.Level; import java.io.Serializable; @@ -34,8 +33,6 @@ // TODO: this can probably be replaced with much less code via dynamic dispatch and/or templates. private InsertCommitHookDesc insertCommitHookDesc; - private AlterTableDesc alterTblDesc; - private AlterTableSimpleDesc alterTblSimpleDesc; private MsckDesc msckDesc; private ShowConfDesc showConfDesc; @@ -52,7 +49,6 @@ * List of WriteEntities that are passed to the hooks. */ protected HashSet outputs; - private AlterTablePartMergeFilesDesc mergeFilesDesc; private CacheMetadataDesc cacheMetadataDesc; public DDLWork() { @@ -69,28 +65,6 @@ public DDLWork(HashSet inputs, HashSet outputs, this.showConfDesc = showConfDesc; } - /** - * @param alterTblDesc - * alter table descriptor - */ - public DDLWork(HashSet inputs, HashSet outputs, - AlterTableDesc alterTblDesc) { - this(inputs, outputs); - this.alterTblDesc = alterTblDesc; - } - - /** - * @param inputs - * @param outputs - * @param simpleDesc - */ - public DDLWork(HashSet inputs, HashSet outputs, - AlterTableSimpleDesc simpleDesc) { - this(inputs, outputs); - - this.alterTblSimpleDesc = simpleDesc; - } - public DDLWork(HashSet inputs, HashSet outputs, MsckDesc checkDesc) { this(inputs, outputs); @@ -98,12 +72,6 @@ public DDLWork(HashSet inputs, HashSet outputs, msckDesc = checkDesc; } - public DDLWork(HashSet inputs, HashSet outputs, - AlterTablePartMergeFilesDesc mergeDesc) { - this(inputs, outputs); - this.mergeFilesDesc = mergeDesc; - } - public DDLWork(HashSet inputs, HashSet outputs, CacheMetadataDesc cacheMetadataDesc) { this(inputs, outputs); @@ -123,21 +91,6 @@ public DDLWork(HashSet inputs, HashSet outputs, this.replSetFirstIncLoadFlagDesc = replSetFirstIncLoadFlagDesc; } - /** - * @return the alterTblDesc - */ - @Explain(displayName = "Alter Table Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED }) - public AlterTableDesc getAlterTblDesc() { - return alterTblDesc; - } - - /** - * @return information about the table/partitions we want to alter. - */ - public AlterTableSimpleDesc getAlterTblSimpleDesc() { - return alterTblSimpleDesc; - } - /** * @return Metastore check description */ @@ -153,13 +106,6 @@ public MsckDesc getMsckDesc() { return outputs; } - /** - * @return descriptor for merging files - */ - public AlterTablePartMergeFilesDesc getMergeFilesDesc() { - return mergeFilesDesc; - } - public boolean getNeedLock() { return needLock; } diff --git ql/src/test/org/apache/hadoop/hive/ql/parse/TestQBCompact.java ql/src/test/org/apache/hadoop/hive/ql/parse/TestQBCompact.java index 61d74de443..19ab8e6474 100644 --- ql/src/test/org/apache/hadoop/hive/ql/parse/TestQBCompact.java +++ ql/src/test/org/apache/hadoop/hive/ql/parse/TestQBCompact.java @@ -26,13 +26,13 @@ import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryState; +import org.apache.hadoop.hive.ql.ddl.DDLWork2; +import org.apache.hadoop.hive.ql.ddl.table.storage.AlterTableCompactDesc; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.plan.AlterTableSimpleDesc; -import org.apache.hadoop.hive.ql.plan.DDLWork; import org.apache.hadoop.hive.ql.session.SessionState; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -84,19 +84,19 @@ private void parseAndAnalyze(String query) throws Exception { Assert.assertEquals(1, roots.size()); } - private AlterTableSimpleDesc parseAndAnalyzeAlterTable(String query) throws Exception { + private AlterTableCompactDesc parseAndAnalyzeAlterTable(String query) throws Exception { ParseDriver hd = new ParseDriver(); ASTNode head = (ASTNode)hd.parse(query).getChild(0); BaseSemanticAnalyzer a = SemanticAnalyzerFactory.get(queryState, head); a.analyze(head, new Context(conf)); List> roots = a.getRootTasks(); Assert.assertEquals(1, roots.size()); - return ((DDLWork)roots.get(0).getWork()).getAlterTblSimpleDesc(); + return (AlterTableCompactDesc)((DDLWork2)roots.get(0).getWork()).getDDLDesc(); } @Test public void testNonPartitionedTable() throws Exception { - AlterTableSimpleDesc desc = parseAndAnalyzeAlterTable("alter table foo compact 'major'"); + AlterTableCompactDesc desc = parseAndAnalyzeAlterTable("alter table foo compact 'major'"); Assert.assertEquals("major", desc.getCompactionType()); Assert.assertEquals("default.foo", desc.getTableName()); } @@ -115,22 +115,22 @@ public void testBogusLevel() throws Exception { @Test public void testMajor() throws Exception { - AlterTableSimpleDesc desc = + AlterTableCompactDesc desc = parseAndAnalyzeAlterTable("alter table foo partition(ds = 'today') compact 'major'"); Assert.assertEquals("major", desc.getCompactionType()); Assert.assertEquals("default.foo", desc.getTableName()); - HashMap parts = desc.getPartSpec(); + Map parts = desc.getPartitionSpec(); Assert.assertEquals(1, parts.size()); Assert.assertEquals("today", parts.get("ds")); } @Test public void testMinor() throws Exception { - AlterTableSimpleDesc desc = + AlterTableCompactDesc desc = parseAndAnalyzeAlterTable("alter table foo partition(ds = 'today') compact 'minor'"); Assert.assertEquals("minor", desc.getCompactionType()); Assert.assertEquals("default.foo", desc.getTableName()); - HashMap parts = desc.getPartSpec(); + Map parts = desc.getPartitionSpec(); Assert.assertEquals(1, parts.size()); Assert.assertEquals("today", parts.get("ds")); } diff --git ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q index 50fb3e9ad9..8e3179cea8 100644 --- ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q +++ ql/src/test/queries/clientpositive/acid_vectorization_original_tez.q @@ -106,6 +106,7 @@ explain select ROW__ID, count(*) from over10k_orc_bucketed_n0 group by ROW__ID h select ROW__ID, count(*) from over10k_orc_bucketed_n0 group by ROW__ID having count(*) > 1; -- schedule compactor +explain alter table over10k_orc_bucketed_n0 compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500'); alter table over10k_orc_bucketed_n0 compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500'); diff --git ql/src/test/queries/clientpositive/archive_multi.q ql/src/test/queries/clientpositive/archive_multi.q index b372ea2f01..9081dec920 100644 --- ql/src/test/queries/clientpositive/archive_multi.q +++ ql/src/test/queries/clientpositive/archive_multi.q @@ -27,6 +27,7 @@ select key, value from default.srcpart where ds='2008-04-09' and hr='12'; SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col FROM (SELECT * FROM ac_test.tstsrcpart WHERE ds='2008-04-08') subq1) subq2; +EXPLAIN ALTER TABLE ac_test.tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); ALTER TABLE ac_test.tstsrcpart ARCHIVE PARTITION (ds='2008-04-08'); SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col @@ -37,6 +38,7 @@ SELECT key, count(1) FROM ac_test.tstsrcpart WHERE ds='2008-04-08' AND hr='12' A SELECT * FROM ac_test.tstsrcpart a JOIN ac_test.tstsrc b ON a.key=b.key WHERE a.ds='2008-04-08' AND a.hr='12' AND a.key='0'; +EXPLAIN ALTER TABLE ac_test.tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08'); ALTER TABLE ac_test.tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08'); SELECT SUM(hash(col)) FROM (SELECT transform(*) using 'tr "\t" "_"' AS col diff --git ql/src/test/queries/clientpositive/dbtxnmgr_compact2.q ql/src/test/queries/clientpositive/dbtxnmgr_compact2.q index 6e560dd446..424255ce9f 100644 --- ql/src/test/queries/clientpositive/dbtxnmgr_compact2.q +++ ql/src/test/queries/clientpositive/dbtxnmgr_compact2.q @@ -8,6 +8,7 @@ alter table T1_n105 add partition (ds = 'yesterday'); alter table T1_n105 partition (ds = 'today') compact 'major'; +explain alter table T1_n105 partition (ds = 'yesterday') compact 'minor'; alter table T1_n105 partition (ds = 'yesterday') compact 'minor'; drop table T1_n105; diff --git ql/src/test/queries/clientpositive/set_tblproperties.q ql/src/test/queries/clientpositive/set_tblproperties.q new file mode 100644 index 0000000000..48cbe7ca02 --- /dev/null +++ ql/src/test/queries/clientpositive/set_tblproperties.q @@ -0,0 +1,8 @@ +CREATE TABLE t (a INT); + +EXPLAIN ALTER TABLE t SET TBLPROPERTIES('a'='x', 'b'='y'); +ALTER TABLE t SET TBLPROPERTIES('a'='x', 'b'='y'); +SHOW CREATE TABLE t; +EXPLAIN ALTER TABLE t UNSET TBLPROPERTIES('a', 'b'); +ALTER TABLE t UNSET TBLPROPERTIES('a', 'b'); +SHOW CREATE TABLE t; diff --git ql/src/test/queries/clientpositive/table_set_owner.q ql/src/test/queries/clientpositive/table_set_owner.q new file mode 100644 index 0000000000..73c9eabafe --- /dev/null +++ ql/src/test/queries/clientpositive/table_set_owner.q @@ -0,0 +1,4 @@ +SET hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider; +CREATE TABLE t (a INT); +EXPLAIN ALTER TABLE t SET OWNER USER user1; +ALTER TABLE t SET OWNER USER user1; diff --git ql/src/test/queries/clientpositive/touch.q ql/src/test/queries/clientpositive/touch.q index 8711b7775c..0fb34d01cc 100644 --- ql/src/test/queries/clientpositive/touch.q +++ ql/src/test/queries/clientpositive/touch.q @@ -9,8 +9,10 @@ create table tc.tstsrcpart like default.srcpart; insert overwrite table tc.tstsrcpart partition (ds='2008-04-08', hr='12') select key, value from default.srcpart where ds='2008-04-08' and hr='12'; +EXPLAIN ALTER TABLE tc.tstsrc TOUCH; ALTER TABLE tc.tstsrc TOUCH; ALTER TABLE tc.tstsrcpart TOUCH; +EXPLAIN ALTER TABLE tc.tstsrcpart TOUCH PARTITION (ds='2008-04-08', hr='12'); ALTER TABLE tc.tstsrcpart TOUCH PARTITION (ds='2008-04-08', hr='12'); drop table tc.tstsrc; diff --git ql/src/test/results/clientnegative/alter_external_acid.q.out ql/src/test/results/clientnegative/alter_external_acid.q.out index 69bba3b8bd..51e44b8340 100644 --- ql/src/test/results/clientnegative/alter_external_acid.q.out +++ ql/src/test/results/clientnegative/alter_external_acid.q.out @@ -10,4 +10,4 @@ PREHOOK: query: alter table acid_external set TBLPROPERTIES ('transactional'='tr PREHOOK: type: ALTERTABLE_PROPERTIES PREHOOK: Input: default@acid_external PREHOOK: Output: default@acid_external -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. default.acid_external cannot be declared transactional because it's an external table +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Unable to alter table. default.acid_external cannot be declared transactional because it's an external table diff --git ql/src/test/results/clientnegative/alter_non_native.q.out ql/src/test/results/clientnegative/alter_non_native.q.out index 110c9f01e2..f54546a0d7 100644 --- ql/src/test/results/clientnegative/alter_non_native.q.out +++ ql/src/test/results/clientnegative/alter_non_native.q.out @@ -8,4 +8,4 @@ STORED BY 'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler' POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@non_native1 -FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [ADDPROPS, DROPPROPS, ADD_COLUMNS] to a non-native table non_native1 +FAILED: SemanticException [Error 10134]: ALTER TABLE can only be used for [SET_PROPERTIES, UNSET_PROPERTIES, ADD_COLUMNS] to a non-native table non_native1 diff --git ql/src/test/results/clientnegative/alter_table_wrong_db.q.out ql/src/test/results/clientnegative/alter_table_wrong_db.q.out index 641e09f221..eca572b4dd 100644 --- ql/src/test/results/clientnegative/alter_table_wrong_db.q.out +++ ql/src/test/results/clientnegative/alter_table_wrong_db.q.out @@ -22,4 +22,4 @@ PREHOOK: query: alter table bad_rename1.rename1 rename to bad_db_notexists.renam PREHOOK: type: ALTERTABLE_RENAME PREHOOK: Input: bad_rename1@rename1 PREHOOK: Output: bad_rename1@rename1 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Unable to change partition or table. Object bad_db_notexists does not exist. Check metastore logs for detailed stack. +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Unable to alter table. Unable to change partition or table. Object bad_db_notexists does not exist. Check metastore logs for detailed stack. diff --git ql/src/test/results/clientnegative/archive1.q.out ql/src/test/results/clientnegative/archive1.q.out index b53085578b..d52ac0e1b1 100644 --- ql/src/test/results/clientnegative/archive1.q.out +++ ql/src/test/results/clientnegative/archive1.q.out @@ -32,4 +32,4 @@ PREHOOK: query: ALTER TABLE srcpart_archived ARCHIVE PARTITION (ds='2008-04-08', PREHOOK: type: ALTERTABLE_ARCHIVE PREHOOK: Input: default@srcpart_archived PREHOOK: Output: default@srcpart_archived@ds=2008-04-08/hr=12 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Partition(s) already archived +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Partition(s) already archived diff --git ql/src/test/results/clientnegative/archive2.q.out ql/src/test/results/clientnegative/archive2.q.out index 471df78e72..75d823eb37 100644 --- ql/src/test/results/clientnegative/archive2.q.out +++ ql/src/test/results/clientnegative/archive2.q.out @@ -28,4 +28,4 @@ PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr= PREHOOK: type: ALTERTABLE_UNARCHIVE PREHOOK: Input: default@tstsrcpart PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Partition ds=2008-04-08/hr=12 is not archived. +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Partition ds=2008-04-08/hr=12 is not archived. diff --git ql/src/test/results/clientnegative/archive_multi1.q.out ql/src/test/results/clientnegative/archive_multi1.q.out index 3fed11ed68..c5f85cc792 100644 --- ql/src/test/results/clientnegative/archive_multi1.q.out +++ ql/src/test/results/clientnegative/archive_multi1.q.out @@ -49,4 +49,4 @@ PREHOOK: type: ALTERTABLE_ARCHIVE PREHOOK: Input: default@tstsrcpart PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Partition(s) already archived +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Partition(s) already archived diff --git ql/src/test/results/clientnegative/archive_multi2.q.out ql/src/test/results/clientnegative/archive_multi2.q.out index 134a342e28..460b78108a 100644 --- ql/src/test/results/clientnegative/archive_multi2.q.out +++ ql/src/test/results/clientnegative/archive_multi2.q.out @@ -42,4 +42,4 @@ PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr= PREHOOK: type: ALTERTABLE_UNARCHIVE PREHOOK: Input: default@tstsrcpart PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Partition ds=2008-04-08/hr=12 is not archived. +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Partition ds=2008-04-08/hr=12 is not archived. diff --git ql/src/test/results/clientnegative/archive_multi3.q.out ql/src/test/results/clientnegative/archive_multi3.q.out index a4cd806450..0095bbef2b 100644 --- ql/src/test/results/clientnegative/archive_multi3.q.out +++ ql/src/test/results/clientnegative/archive_multi3.q.out @@ -47,4 +47,4 @@ PREHOOK: type: ALTERTABLE_ARCHIVE PREHOOK: Input: default@tstsrcpart PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Conflict with existing archive ds=2008-04-08/hr=12 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Conflict with existing archive ds=2008-04-08/hr=12 diff --git ql/src/test/results/clientnegative/archive_multi4.q.out ql/src/test/results/clientnegative/archive_multi4.q.out index 4221d2570e..8413e83a66 100644 --- ql/src/test/results/clientnegative/archive_multi4.q.out +++ ql/src/test/results/clientnegative/archive_multi4.q.out @@ -48,4 +48,4 @@ PREHOOK: query: ALTER TABLE tstsrcpart ARCHIVE PARTITION (ds='2008-04-08', hr='1 PREHOOK: type: ALTERTABLE_ARCHIVE PREHOOK: Input: default@tstsrcpart PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Conflict with existing archive ds=2008-04-08 +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Conflict with existing archive ds=2008-04-08 diff --git ql/src/test/results/clientnegative/archive_multi5.q.out ql/src/test/results/clientnegative/archive_multi5.q.out index 315fbe4884..36a78305ef 100644 --- ql/src/test/results/clientnegative/archive_multi5.q.out +++ ql/src/test/results/clientnegative/archive_multi5.q.out @@ -47,4 +47,4 @@ PREHOOK: type: ALTERTABLE_UNARCHIVE PREHOOK: Input: default@tstsrcpart PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=11 PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Partition ds=2008-04-08/hr=11 is not archived. +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Partition ds=2008-04-08/hr=11 is not archived. diff --git ql/src/test/results/clientnegative/archive_multi6.q.out ql/src/test/results/clientnegative/archive_multi6.q.out index e73ddd1449..2046db96b7 100644 --- ql/src/test/results/clientnegative/archive_multi6.q.out +++ ql/src/test/results/clientnegative/archive_multi6.q.out @@ -48,4 +48,4 @@ PREHOOK: query: ALTER TABLE tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08', hr= PREHOOK: type: ALTERTABLE_UNARCHIVE PREHOOK: Input: default@tstsrcpart PREHOOK: Output: default@tstsrcpart@ds=2008-04-08/hr=12 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Partition ds=2008-04-08/hr=12 is archived at level 1, and given partspec only has 2 specs. +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Partition ds=2008-04-08/hr=12 is archived at level 1, and given partspec only has 2 specs. diff --git ql/src/test/results/clientnegative/compact_non_acid_table.q.out ql/src/test/results/clientnegative/compact_non_acid_table.q.out index eab9e19ae6..2e1f1970bc 100644 --- ql/src/test/results/clientnegative/compact_non_acid_table.q.out +++ ql/src/test/results/clientnegative/compact_non_acid_table.q.out @@ -8,4 +8,4 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@not_an_acid_table PREHOOK: query: alter table not_an_acid_table compact 'major' PREHOOK: type: ALTERTABLE_COMPACT -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Compaction is not allowed on non-ACID table default.not_an_acid_table +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Compaction is not allowed on non-ACID table default.not_an_acid_table diff --git ql/src/test/results/clientnegative/mm_convert.q.out ql/src/test/results/clientnegative/mm_convert.q.out index 496f8b5f29..2463a56b86 100644 --- ql/src/test/results/clientnegative/mm_convert.q.out +++ ql/src/test/results/clientnegative/mm_convert.q.out @@ -14,4 +14,4 @@ PREHOOK: query: alter table convert_mm unset tblproperties('transactional_proper PREHOOK: type: ALTERTABLE_PROPERTIES PREHOOK: Input: default@convert_mm PREHOOK: Output: default@convert_mm -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Cannot convert an ACID table to non-ACID +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Cannot convert an ACID table to non-ACID diff --git ql/src/test/results/clientnegative/strict_managed_tables2.q.out ql/src/test/results/clientnegative/strict_managed_tables2.q.out index 85753a494c..286698d486 100644 --- ql/src/test/results/clientnegative/strict_managed_tables2.q.out +++ ql/src/test/results/clientnegative/strict_managed_tables2.q.out @@ -10,4 +10,4 @@ PREHOOK: query: alter table strict_managed_tables2_tab1 set tblproperties ('EXTE PREHOOK: type: ALTERTABLE_PROPERTIES PREHOOK: Input: default@strict_managed_tables2_tab1 PREHOOK: Output: default@strict_managed_tables2_tab1 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Table default.strict_managed_tables2_tab1 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional. +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Unable to alter table. Table default.strict_managed_tables2_tab1 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional. diff --git ql/src/test/results/clientnegative/strict_managed_tables3.q.out ql/src/test/results/clientnegative/strict_managed_tables3.q.out index 7de8a80779..a5a82e03f9 100644 --- ql/src/test/results/clientnegative/strict_managed_tables3.q.out +++ ql/src/test/results/clientnegative/strict_managed_tables3.q.out @@ -10,4 +10,4 @@ PREHOOK: query: alter table strict_managed_tables3_tab1 unset tblproperties ('EX PREHOOK: type: ALTERTABLE_PROPERTIES PREHOOK: Input: default@strict_managed_tables3_tab1 PREHOOK: Output: default@strict_managed_tables3_tab1 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Table default.strict_managed_tables3_tab1 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional. +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Unable to alter table. Table default.strict_managed_tables3_tab1 failed strict managed table checks due to the following reason: Table is marked as a managed table but is not transactional. diff --git ql/src/test/results/clientnegative/temp_table_rename.q.out ql/src/test/results/clientnegative/temp_table_rename.q.out index 0956ababf3..4c24efd17e 100644 --- ql/src/test/results/clientnegative/temp_table_rename.q.out +++ ql/src/test/results/clientnegative/temp_table_rename.q.out @@ -18,4 +18,4 @@ PREHOOK: query: alter table tmp2 rename to tmp1 PREHOOK: type: ALTERTABLE_RENAME PREHOOK: Input: default@tmp2 PREHOOK: Output: default@tmp2 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Cannot rename temporary table to tmp1 - temporary table already exists with the same name +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Unable to alter table. Cannot rename temporary table to tmp1 - temporary table already exists with the same name diff --git ql/src/test/results/clientnegative/touch1.q.out ql/src/test/results/clientnegative/touch1.q.out index cc1e52ec4f..342a5052c4 100644 --- ql/src/test/results/clientnegative/touch1.q.out +++ ql/src/test/results/clientnegative/touch1.q.out @@ -1,4 +1,4 @@ PREHOOK: query: ALTER TABLE srcpart TOUCH PARTITION (ds='2008-04-08', hr='13') PREHOOK: type: ALTERTABLE_TOUCH PREHOOK: Input: default@srcpart -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Specified partition does not exist +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Specified partition does not exist diff --git ql/src/test/results/clientpositive/alter_rename_table.q.out ql/src/test/results/clientpositive/alter_rename_table.q.out index 32919ea11f..dd656954a1 100644 --- ql/src/test/results/clientpositive/alter_rename_table.q.out +++ ql/src/test/results/clientpositive/alter_rename_table.q.out @@ -129,11 +129,9 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - Alter Table Operator: - Alter Table - type: rename - new name: target.src - old name: source.src + Rename Table + new table name: target.src + table name: source.src PREHOOK: query: ALTER TABLE source.src RENAME TO target.src PREHOOK: type: ALTERTABLE_RENAME @@ -177,11 +175,9 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - Alter Table Operator: - Alter Table - type: rename - new name: target.srcpart - old name: source.srcpart + Rename Table + new table name: target.srcpart + table name: source.srcpart PREHOOK: query: ALTER TABLE source.srcpart RENAME TO target.srcpart PREHOOK: type: ALTERTABLE_RENAME diff --git ql/src/test/results/clientpositive/archive_multi.q.out ql/src/test/results/clientpositive/archive_multi.q.out index 5222c3398f..eea3a6ed86 100644 --- ql/src/test/results/clientpositive/archive_multi.q.out +++ ql/src/test/results/clientpositive/archive_multi.q.out @@ -101,6 +101,26 @@ POSTHOOK: Input: ac_test@tstsrcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: ac_test@tstsrcpart@ds=2008-04-08/hr=12 #### A masked pattern was here #### 48479881068 +PREHOOK: query: EXPLAIN ALTER TABLE ac_test.tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_ARCHIVE +PREHOOK: Input: ac_test@tstsrcpart +PREHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: EXPLAIN ALTER TABLE ac_test.tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') +POSTHOOK: type: ALTERTABLE_ARCHIVE +POSTHOOK: Input: ac_test@tstsrcpart +POSTHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=12 +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Archive + partition spec: + ds 2008-04-08 + table name: ac_test.tstsrcpart + PREHOOK: query: ALTER TABLE ac_test.tstsrcpart ARCHIVE PARTITION (ds='2008-04-08') PREHOOK: type: ALTERTABLE_ARCHIVE PREHOOK: Input: ac_test@tstsrcpart @@ -161,6 +181,26 @@ POSTHOOK: Input: ac_test@tstsrcpart@ds=2008-04-08/hr=12 0 val_0 2008-04-08 12 0 val_0 0 val_0 2008-04-08 12 0 val_0 0 val_0 2008-04-08 12 0 val_0 +PREHOOK: query: EXPLAIN ALTER TABLE ac_test.tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08') +PREHOOK: type: ALTERTABLE_UNARCHIVE +PREHOOK: Input: ac_test@tstsrcpart +PREHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=11 +PREHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: EXPLAIN ALTER TABLE ac_test.tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08') +POSTHOOK: type: ALTERTABLE_UNARCHIVE +POSTHOOK: Input: ac_test@tstsrcpart +POSTHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=11 +POSTHOOK: Output: ac_test@tstsrcpart@ds=2008-04-08/hr=12 +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Unarchive + partition spec: + ds 2008-04-08 + table name: ac_test.tstsrcpart + PREHOOK: query: ALTER TABLE ac_test.tstsrcpart UNARCHIVE PARTITION (ds='2008-04-08') PREHOOK: type: ALTERTABLE_UNARCHIVE PREHOOK: Input: ac_test@tstsrcpart diff --git ql/src/test/results/clientpositive/dbtxnmgr_compact2.q.out ql/src/test/results/clientpositive/dbtxnmgr_compact2.q.out index b749846030..9d744e2c61 100644 --- ql/src/test/results/clientpositive/dbtxnmgr_compact2.q.out +++ ql/src/test/results/clientpositive/dbtxnmgr_compact2.q.out @@ -24,6 +24,21 @@ PREHOOK: query: alter table T1_n105 partition (ds = 'today') compact 'major' PREHOOK: type: ALTERTABLE_COMPACT POSTHOOK: query: alter table T1_n105 partition (ds = 'today') compact 'major' POSTHOOK: type: ALTERTABLE_COMPACT +PREHOOK: query: explain alter table T1_n105 partition (ds = 'yesterday') compact 'minor' +PREHOOK: type: ALTERTABLE_COMPACT +POSTHOOK: query: explain alter table T1_n105 partition (ds = 'yesterday') compact 'minor' +POSTHOOK: type: ALTERTABLE_COMPACT +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Compact + compaction type: minor + partition spec: + ds yesterday + table name: default.T1_n105 + PREHOOK: query: alter table T1_n105 partition (ds = 'yesterday') compact 'minor' PREHOOK: type: ALTERTABLE_COMPACT POSTHOOK: query: alter table T1_n105 partition (ds = 'yesterday') compact 'minor' diff --git ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out index 7506ca8bea..1401dbc375 100644 --- ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_dynamic_partition.q.out @@ -344,12 +344,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - Alter Table Operator: - Alter Table - type: drop props - old name: default.druid_partitioned_table - properties: - COLUMN_STATS_ACCURATE + Unset Properties + properties: + COLUMN_STATS_ACCURATE + table name: default.druid_partitioned_table Stage: Stage-1 Pre-Insert task @@ -488,12 +486,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - Alter Table Operator: - Alter Table - type: drop props - old name: default.druid_partitioned_table - properties: - COLUMN_STATS_ACCURATE + Unset Properties + properties: + COLUMN_STATS_ACCURATE + table name: default.druid_partitioned_table Stage: Stage-1 Pre-Insert task diff --git ql/src/test/results/clientpositive/druid/druidmini_mv.q.out ql/src/test/results/clientpositive/druid/druidmini_mv.q.out index 483c9c16d5..08a83da67a 100644 --- ql/src/test/results/clientpositive/druid/druidmini_mv.q.out +++ ql/src/test/results/clientpositive/druid/druidmini_mv.q.out @@ -471,12 +471,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - Alter Table Operator: - Alter Table - type: drop props - old name: default.cmv_mat_view2_n0 - properties: - COLUMN_STATS_ACCURATE + Unset Properties + properties: + COLUMN_STATS_ACCURATE + table name: default.cmv_mat_view2_n0 Stage: Stage-4 Materialized View Work diff --git ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out index 23dbced668..f2fcf7fd53 100644 --- ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out +++ ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out @@ -202,7 +202,7 @@ PREHOOK: query: ALTER TABLE encrypted_db_outloc.renamed_encrypted_table_n1 RENAM PREHOOK: type: ALTERTABLE_RENAME PREHOOK: Input: encrypted_db_outloc@renamed_encrypted_table_n1 PREHOOK: Output: encrypted_db_outloc@renamed_encrypted_table_n1 -FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Alter Table operation for encrypted_db_outloc.renamed_encrypted_table_n1 failed to move data due to: 'Got exception: org.apache.hadoop.ipc.RemoteException /build/ql/test/data/specified_db_location/renamed_encrypted_table_n1 can't be moved from an encryption zone.' See hive log file for details. +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask2. Unable to alter table. Alter Table operation for encrypted_db_outloc.renamed_encrypted_table_n1 failed to move data due to: 'Got exception: org.apache.hadoop.ipc.RemoteException /build/ql/test/data/specified_db_location/renamed_encrypted_table_n1 can't be moved from an encryption zone.' See hive log file for details. PREHOOK: query: SHOW TABLES PREHOOK: type: SHOWTABLES PREHOOK: Input: database:encrypted_db_outloc diff --git ql/src/test/results/clientpositive/input3.q.out ql/src/test/results/clientpositive/input3.q.out index c521ed6c78..3d82be7716 100644 --- ql/src/test/results/clientpositive/input3.q.out +++ ql/src/test/results/clientpositive/input3.q.out @@ -91,11 +91,9 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 - Alter Table Operator: - Alter Table - type: rename - new name: default.TEST3c - old name: default.TEST3b + Rename Table + new table name: default.TEST3c + table name: default.TEST3b PREHOOK: query: ALTER TABLE TEST3b RENAME TO TEST3c PREHOOK: type: ALTERTABLE_RENAME diff --git ql/src/test/results/clientpositive/llap/orc_merge10.q.out ql/src/test/results/clientpositive/llap/orc_merge10.q.out index 22b4371fda..3af8190e6a 100644 --- ql/src/test/results/clientpositive/llap/orc_merge10.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge10.q.out @@ -631,6 +631,11 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 + Concatenate + partition desc: + ds 1 + part 0 + table name: default.orcfile_merge1 Stage: Stage-1 Move Operator diff --git ql/src/test/results/clientpositive/llap/orc_merge5.q.out ql/src/test/results/clientpositive/llap/orc_merge5.q.out index 5b24776b1f..6790cd7fff 100644 --- ql/src/test/results/clientpositive/llap/orc_merge5.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge5.q.out @@ -373,6 +373,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 + Concatenate + table name: default.orc_merge5b_n0 Stage: Stage-1 Move Operator diff --git ql/src/test/results/clientpositive/llap/orc_merge6.q.out ql/src/test/results/clientpositive/llap/orc_merge6.q.out index 396834045b..95cbe1ec58 100644 --- ql/src/test/results/clientpositive/llap/orc_merge6.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge6.q.out @@ -521,6 +521,11 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 + Concatenate + partition desc: + hour 24 + year 2000 + table name: default.orc_merge5a_n1 Stage: Stage-1 Move Operator diff --git ql/src/test/results/clientpositive/llap/orc_merge7.q.out ql/src/test/results/clientpositive/llap/orc_merge7.q.out index 8472cfbd8a..4155dc6123 100644 --- ql/src/test/results/clientpositive/llap/orc_merge7.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge7.q.out @@ -652,6 +652,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 + Concatenate + partition desc: + st 80.0 + table name: default.orc_merge5a_n0 Stage: Stage-1 Move Operator diff --git ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out index 319bac7020..48217cd44d 100644 --- ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out +++ ql/src/test/results/clientpositive/llap/orc_merge_incompat2.q.out @@ -337,6 +337,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 + Concatenate + partition desc: + st 80.0 + table name: default.orc_merge5a Stage: Stage-1 Move Operator diff --git ql/src/test/results/clientpositive/orc_merge10.q.out ql/src/test/results/clientpositive/orc_merge10.q.out index b23438183d..e8ebd4e717 100644 --- ql/src/test/results/clientpositive/orc_merge10.q.out +++ ql/src/test/results/clientpositive/orc_merge10.q.out @@ -602,6 +602,11 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 + Concatenate + partition desc: + ds 1 + part 0 + table name: default.orcfile_merge1 Stage: Stage-1 Move Operator diff --git ql/src/test/results/clientpositive/orc_merge5.q.out ql/src/test/results/clientpositive/orc_merge5.q.out index 88d93deae9..df5651e284 100644 --- ql/src/test/results/clientpositive/orc_merge5.q.out +++ ql/src/test/results/clientpositive/orc_merge5.q.out @@ -337,6 +337,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 + Concatenate + table name: default.orc_merge5b_n0 Stage: Stage-1 Move Operator diff --git ql/src/test/results/clientpositive/orc_merge6.q.out ql/src/test/results/clientpositive/orc_merge6.q.out index bd016202d4..a07cfd3cb4 100644 --- ql/src/test/results/clientpositive/orc_merge6.q.out +++ ql/src/test/results/clientpositive/orc_merge6.q.out @@ -485,6 +485,11 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 + Concatenate + partition desc: + hour 24 + year 2000 + table name: default.orc_merge5a_n1 Stage: Stage-1 Move Operator diff --git ql/src/test/results/clientpositive/orc_merge_incompat2.q.out ql/src/test/results/clientpositive/orc_merge_incompat2.q.out index e5930b2b58..71ae287285 100644 --- ql/src/test/results/clientpositive/orc_merge_incompat2.q.out +++ ql/src/test/results/clientpositive/orc_merge_incompat2.q.out @@ -332,6 +332,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 + Concatenate + partition desc: + st 80.0 + table name: default.orc_merge5a Stage: Stage-1 Move Operator diff --git ql/src/test/results/clientpositive/set_tblproperties.q.out ql/src/test/results/clientpositive/set_tblproperties.q.out new file mode 100644 index 0000000000..3503ea5514 --- /dev/null +++ ql/src/test/results/clientpositive/set_tblproperties.q.out @@ -0,0 +1,102 @@ +PREHOOK: query: CREATE TABLE t (a INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: CREATE TABLE t (a INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: EXPLAIN ALTER TABLE t SET TBLPROPERTIES('a'='x', 'b'='y') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@t +PREHOOK: Output: default@t +POSTHOOK: query: EXPLAIN ALTER TABLE t SET TBLPROPERTIES('a'='x', 'b'='y') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@t +POSTHOOK: Output: default@t +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Set Properties + properties: + a x + b y + table name: default.t + +PREHOOK: query: ALTER TABLE t SET TBLPROPERTIES('a'='x', 'b'='y') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@t +PREHOOK: Output: default@t +POSTHOOK: query: ALTER TABLE t SET TBLPROPERTIES('a'='x', 'b'='y') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@t +POSTHOOK: Output: default@t +PREHOOK: query: SHOW CREATE TABLE t +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@t +POSTHOOK: query: SHOW CREATE TABLE t +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@t +CREATE TABLE `t`( + `a` int) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'a'='x', + 'b'='y', + 'bucketing_version'='2', +#### A masked pattern was here #### +PREHOOK: query: EXPLAIN ALTER TABLE t UNSET TBLPROPERTIES('a', 'b') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@t +PREHOOK: Output: default@t +POSTHOOK: query: EXPLAIN ALTER TABLE t UNSET TBLPROPERTIES('a', 'b') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@t +POSTHOOK: Output: default@t +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Unset Properties + properties: + a + b + table name: default.t + +PREHOOK: query: ALTER TABLE t UNSET TBLPROPERTIES('a', 'b') +PREHOOK: type: ALTERTABLE_PROPERTIES +PREHOOK: Input: default@t +PREHOOK: Output: default@t +POSTHOOK: query: ALTER TABLE t UNSET TBLPROPERTIES('a', 'b') +POSTHOOK: type: ALTERTABLE_PROPERTIES +POSTHOOK: Input: default@t +POSTHOOK: Output: default@t +PREHOOK: query: SHOW CREATE TABLE t +PREHOOK: type: SHOW_CREATETABLE +PREHOOK: Input: default@t +POSTHOOK: query: SHOW CREATE TABLE t +POSTHOOK: type: SHOW_CREATETABLE +POSTHOOK: Input: default@t +CREATE TABLE `t`( + `a` int) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION +#### A masked pattern was here #### +TBLPROPERTIES ( + 'bucketing_version'='2', +#### A masked pattern was here #### diff --git ql/src/test/results/clientpositive/spark/orc_merge5.q.out ql/src/test/results/clientpositive/spark/orc_merge5.q.out index fe5a71ef5f..e72d1d23ff 100644 --- ql/src/test/results/clientpositive/spark/orc_merge5.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge5.q.out @@ -289,6 +289,8 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 + Concatenate + table name: default.orc_merge5b_n0 Stage: Stage-1 Move Operator diff --git ql/src/test/results/clientpositive/spark/orc_merge6.q.out ql/src/test/results/clientpositive/spark/orc_merge6.q.out index ccf766dac7..982b614e34 100644 --- ql/src/test/results/clientpositive/spark/orc_merge6.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge6.q.out @@ -421,6 +421,11 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 + Concatenate + partition desc: + hour 24 + year 2000 + table name: default.orc_merge5a_n1 Stage: Stage-1 Move Operator diff --git ql/src/test/results/clientpositive/spark/orc_merge7.q.out ql/src/test/results/clientpositive/spark/orc_merge7.q.out index 192f8c46c8..a641ed73b7 100644 --- ql/src/test/results/clientpositive/spark/orc_merge7.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge7.q.out @@ -552,6 +552,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 + Concatenate + partition desc: + st 80.0 + table name: default.orc_merge5a_n0 Stage: Stage-1 Move Operator diff --git ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out index 2330d9e272..1bfbf4d8df 100644 --- ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out +++ ql/src/test/results/clientpositive/spark/orc_merge_incompat2.q.out @@ -293,6 +293,10 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 + Concatenate + partition desc: + st 80.0 + table name: default.orc_merge5a Stage: Stage-1 Move Operator diff --git ql/src/test/results/clientpositive/table_set_owner.q.out ql/src/test/results/clientpositive/table_set_owner.q.out new file mode 100644 index 0000000000..8528faf245 --- /dev/null +++ ql/src/test/results/clientpositive/table_set_owner.q.out @@ -0,0 +1,28 @@ +PREHOOK: query: CREATE TABLE t (a INT) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: CREATE TABLE t (a INT) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: EXPLAIN ALTER TABLE t SET OWNER USER user1 +PREHOOK: type: ALTERTABLE_OWNER +POSTHOOK: query: EXPLAIN ALTER TABLE t SET OWNER USER user1 +POSTHOOK: type: ALTERTABLE_OWNER +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 +#### A masked pattern was here #### + Principal + name: user1 + table name: default.t + +PREHOOK: query: ALTER TABLE t SET OWNER USER user1 +PREHOOK: type: ALTERTABLE_OWNER +POSTHOOK: query: ALTER TABLE t SET OWNER USER user1 +POSTHOOK: type: ALTERTABLE_OWNER +POSTHOOK: Input: default@t +POSTHOOK: Output: default@t diff --git ql/src/test/results/clientpositive/table_storage.q.out ql/src/test/results/clientpositive/table_storage.q.out index 4311350afb..7de66e6a41 100644 --- ql/src/test/results/clientpositive/table_storage.q.out +++ ql/src/test/results/clientpositive/table_storage.q.out @@ -509,7 +509,7 @@ STAGE DEPENDENCIES: STAGE PLANS: Stage: Stage-0 Set Serde Props - props: + properties: property1 value1 property2 value2 table name: default.t diff --git ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out index adde2486e5..82666add4b 100644 --- ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out +++ ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out @@ -747,6 +747,23 @@ POSTHOOK: query: select ROW__ID, count(*) from over10k_orc_bucketed_n0 group by POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k_orc_bucketed_n0 POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: explain alter table over10k_orc_bucketed_n0 compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500') +PREHOOK: type: ALTERTABLE_COMPACT +POSTHOOK: query: explain alter table over10k_orc_bucketed_n0 compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500') +POSTHOOK: type: ALTERTABLE_COMPACT +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Compact + compaction type: major + properties: + compactor.hive.tez.container.size 500 + compactor.mapreduce.map.memory.mb 500 + compactor.mapreduce.reduce.memory.mb 500 + table name: default.over10k_orc_bucketed_n0 + PREHOOK: query: alter table over10k_orc_bucketed_n0 compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500') PREHOOK: type: ALTERTABLE_COMPACT POSTHOOK: query: alter table over10k_orc_bucketed_n0 compact 'major' WITH OVERWRITE TBLPROPERTIES ('compactor.mapreduce.map.memory.mb'='500', 'compactor.mapreduce.reduce.memory.mb'='500','compactor.mapreduce.map.memory.mb'='500', 'compactor.hive.tez.container.size'='500') diff --git ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out index 1ad26b5f66..6925f58af1 100644 --- ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out +++ ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out @@ -174,8 +174,7 @@ POSTHOOK: type: ALTERTABLE_RENAME POSTHOOK: Input: newdb@tab_n2 POSTHOOK: Output: newdb@tab_n2 Stage-0 - Alter Table Operator: - new name:newDB.newName,old name:newDB.tab_n2,type:rename + Rename Table{"new table name:":"newDB.newName","table name:":"newDB.tab_n2"} PREHOOK: query: drop table tab_n2 PREHOOK: type: DROPTABLE diff --git ql/src/test/results/clientpositive/tez/explainuser_3.q.out ql/src/test/results/clientpositive/tez/explainuser_3.q.out index c07c6a3572..26eae7e8d0 100644 --- ql/src/test/results/clientpositive/tez/explainuser_3.q.out +++ ql/src/test/results/clientpositive/tez/explainuser_3.q.out @@ -178,8 +178,7 @@ POSTHOOK: type: ALTERTABLE_RENAME POSTHOOK: Input: newdb@tab_n1 POSTHOOK: Output: newdb@tab_n1 Stage-0 - Alter Table Operator: - new name:newDB.newName,old name:newDB.tab_n1,type:rename + Rename Table{"new table name:":"newDB.newName","table name:":"newDB.tab_n1"} PREHOOK: query: explain drop table tab_n1 PREHOOK: type: DROPTABLE diff --git ql/src/test/results/clientpositive/touch.q.out ql/src/test/results/clientpositive/touch.q.out index c239715299..c58f600508 100644 --- ql/src/test/results/clientpositive/touch.q.out +++ ql/src/test/results/clientpositive/touch.q.out @@ -44,6 +44,22 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Output: tc@tstsrcpart@ds=2008-04-08/hr=12 POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: EXPLAIN ALTER TABLE tc.tstsrc TOUCH +PREHOOK: type: ALTERTABLE_TOUCH +PREHOOK: Input: tc@tstsrc +PREHOOK: Output: tc@tstsrc +POSTHOOK: query: EXPLAIN ALTER TABLE tc.tstsrc TOUCH +POSTHOOK: type: ALTERTABLE_TOUCH +POSTHOOK: Input: tc@tstsrc +POSTHOOK: Output: tc@tstsrc +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Touch + table name: tc.tstsrc + PREHOOK: query: ALTER TABLE tc.tstsrc TOUCH PREHOOK: type: ALTERTABLE_TOUCH PREHOOK: Input: tc@tstsrc @@ -60,6 +76,25 @@ POSTHOOK: query: ALTER TABLE tc.tstsrcpart TOUCH POSTHOOK: type: ALTERTABLE_TOUCH POSTHOOK: Input: tc@tstsrcpart POSTHOOK: Output: tc@tstsrcpart +PREHOOK: query: EXPLAIN ALTER TABLE tc.tstsrcpart TOUCH PARTITION (ds='2008-04-08', hr='12') +PREHOOK: type: ALTERTABLE_TOUCH +PREHOOK: Input: tc@tstsrcpart +PREHOOK: Output: tc@tstsrcpart@ds=2008-04-08/hr=12 +POSTHOOK: query: EXPLAIN ALTER TABLE tc.tstsrcpart TOUCH PARTITION (ds='2008-04-08', hr='12') +POSTHOOK: type: ALTERTABLE_TOUCH +POSTHOOK: Input: tc@tstsrcpart +POSTHOOK: Output: tc@tstsrcpart@ds=2008-04-08/hr=12 +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Touch + partition spec: + ds 2008-04-08 + hr 12 + table name: tc.tstsrcpart + PREHOOK: query: ALTER TABLE tc.tstsrcpart TOUCH PARTITION (ds='2008-04-08', hr='12') PREHOOK: type: ALTERTABLE_TOUCH PREHOOK: Input: tc@tstsrcpart diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java index e33f5e43ec..646b7c1620 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java @@ -44,8 +44,8 @@ public String ALTER_TABLE_OPERATION_TYPE = "alterTableOpType"; - // These should remain in sync with AlterTableDesc::AlterTableType enum - public List allowedAlterTypes = ImmutableList.of("ADDPROPS", "DROPPROPS"); + // These should remain in sync with AlterTableType enum + public List allowedAlterTypes = ImmutableList.of("SET_PROPERTIES", "UNSET_PROPERTIES"); String SET_LOCATION = "SET_LOCATION"; /** @@ -114,7 +114,7 @@ public void commitDropTable(Table table, boolean deleteData) public default void preAlterTable(Table table, EnvironmentContext context) throws MetaException { String alterOpType = (context == null || context.getProperties() == null) ? null : context.getProperties().get(ALTER_TABLE_OPERATION_TYPE); - // By default allow only ADDPROPS and DROPPROPS. + // By default allow only SET_PROPERTIESS and UNSET_PROPERTIES. // alterOpType is null in case of stats update. if (alterOpType != null && !allowedAlterTypes.contains(alterOpType)){ throw new MetaException(