diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java index 7a309626cc..ea7e4617df 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java @@ -71,6 +71,7 @@ import java.util.List; import java.util.Map; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY; import static org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.LoadDatabase.AlterDatabase; import static org.apache.hadoop.hive.ql.exec.repl.ReplAck.LOAD_ACKNOWLEDGEMENT; import static org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils.RANGER_AUTHORIZER; @@ -291,7 +292,10 @@ a database ( directory ) // Populate the driver context with the scratch dir info from the repl context, so that the // temp dirs will be cleaned up later context.getFsScratchDirs().putAll(loadContext.pathInfo.getFsScratchDirs()); - createReplLoadCompleteAckTask(); + boolean isMetaDataOnly = HiveConf.getBoolVar(conf, REPL_DUMP_METADATA_ONLY); + if (!isMetaDataOnly) { + createReplLoadCompleteAckTask(); + } LOG.info("completed load task run : {}", work.executedLoadTask()); return 0; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java index b36c4a531f..20e123fc09 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.table; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Database; @@ -63,6 +64,7 @@ import java.util.List; import java.util.Map; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_ENABLE_MOVE_OPTIMIZATION; import static org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.ReplicationState.PartitionState; import static org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer.isPartitioned; @@ -202,10 +204,14 @@ private void addPartition(boolean hasMorePartitions, AlterTableAddPartitionDesc context.hiveConf ); - boolean isOnlyDDLOperation = event.replicationSpec().isMetadataOnly() + boolean isMetaDataOnly = HiveConf.getBoolVar(context.hiveConf, REPL_DUMP_METADATA_ONLY); + boolean isOnlyDDLOperation = isMetaDataOnly || event.replicationSpec().isMetadataOnly() || (TableType.EXTERNAL_TABLE.equals(table.getTableType()) && !event.replicationSpec().isMigratingToExternalTable() ); + if (isMetaDataOnly) { + LOG.info("Patch: skipping load due to isMetaDataOnly"); + } if (isOnlyDDLOperation) { // Set Checkpoint task as dependant to add partition tasks. So, if same dump is retried for diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java index 6cea22c01f..6900efe0da 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java @@ -20,6 +20,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.ValidReaderWriteIdList; import org.apache.hadoop.hive.common.ValidWriteIdList; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Database; @@ -62,6 +63,7 @@ import java.util.List; import java.util.TreeMap; +import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY; import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_ENABLE_MOVE_OPTIMIZATION; import static org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer.isPartitioned; @@ -215,8 +217,9 @@ private void newTableTasks(ImportTableDesc tblDesc, Task tblRootTask, TableLo parentTask.addDependentTask(replTxnTask); parentTask = replTxnTask; } - boolean shouldCreateLoadTableTask = ( - !isPartitioned(tblDesc) + boolean isMetaDataOnly = HiveConf.getBoolVar(context.hiveConf, REPL_DUMP_METADATA_ONLY); + boolean shouldCreateLoadTableTask = (!isMetaDataOnly) && + (!isPartitioned(tblDesc) && !TableType.EXTERNAL_TABLE.equals(table.getTableType()) ) || tuple.isConvertedFromManagedToExternal; if (shouldCreateLoadTableTask) {