diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index ab818349b5..2c65059817 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -331,7 +331,7 @@ private void loadAndVerify(String replDbName, String sourceDbNameOrPattern, Stri * appropriately. This tests bootstrap behaviour primarily. */ @Test - public void testBasic() throws IOException { + public void testBasic() throws IOException, SemanticException { String name = testName.getMethodName(); String dbName = createDB(name, driver); run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java index 9d5e8af30c..379e487784 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils; +import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; import org.apache.hadoop.security.UserGroupInformation; import org.junit.Assert; @@ -46,6 +47,7 @@ import static org.apache.hadoop.hive.ql.exec.repl.ReplExternalTables.FILE_NAME; import static org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils.INC_BOOTSTRAP_ROOT_DIR_NAME; +import static org.junit.Assert.assertEquals; /** * Tests Table level replication scenarios. @@ -163,6 +165,8 @@ private String replicateAndVerify(String replPolicy, String oldReplPolicy, Strin // If the policy contains '.'' means its table level replication. verifyTableListForPolicy(tuple.dumpLocation, replPolicy.contains(".'") ? expectedTables : null); + verifyDumpMetadata(replPolicy, new Path(tuple.dumpLocation, ReplUtils.REPL_HIVE_BASE_DIR)); + replica.load(replicatedDbName, replPolicy, loadWithClause) .run("use " + replicatedDbName) .run("show tables") @@ -180,6 +184,21 @@ private String replicateAndVerify(String replPolicy, String oldReplPolicy, Strin return tuple.lastReplicationId; } + private void verifyDumpMetadata(String replPolicy, Path dumpPath) throws SemanticException { + String[] parseReplPolicy = replPolicy.split("\\.'"); + assertEquals(parseReplPolicy[0], new DumpMetaData(dumpPath, conf).getReplScope().getDbName()); + if (parseReplPolicy.length > 1) { + parseReplPolicy[1] = parseReplPolicy[1].replaceAll("'", ""); + assertEquals(parseReplPolicy[1], + new DumpMetaData(dumpPath, conf).getReplScope().getIncludedTableNames()); + } + if (parseReplPolicy.length > 2) { + parseReplPolicy[2] = parseReplPolicy[2].replaceAll("'", ""); + assertEquals(parseReplPolicy[2], + new DumpMetaData(dumpPath, conf).getReplScope().getExcludedTableNames()); + } + } + private String replicateAndVerifyClearDump(String replPolicy, String oldReplPolicy, String lastReplId, List dumpWithClause, List loadWithClause, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java index 1fce791cbe..8252b8563c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java @@ -54,7 +54,6 @@ import org.apache.hadoop.hive.ql.exec.util.DAGTraversal; import org.apache.hadoop.hive.ql.exec.util.Retryable; import org.apache.hadoop.hive.ql.io.AcidUtils; -import org.apache.hadoop.hive.ql.io.orc.ExternalCache; import org.apache.hadoop.hive.ql.lockmgr.DbLockManager; import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager; import org.apache.hadoop.hive.ql.lockmgr.LockException; @@ -607,9 +606,7 @@ private Long incrementalDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot, Hive dmd.setDump(DumpType.INCREMENTAL, work.eventFrom, lastReplId, cmRoot, executionId); // If repl policy is changed (oldReplScope is set), then pass the current replication policy, // so that REPL LOAD would drop the tables which are not included in current policy. - if (work.oldReplScope != null) { - dmd.setReplScope(work.replScope); - } + dmd.setReplScope(work.replScope); dmd.write(true); int cacheSize = conf.getIntVar(HiveConf.ConfVars.REPL_FILE_LIST_CACHE_SIZE); try (FileList managedTblList = createTableFileList(dumpRoot, EximUtil.FILE_LIST, cacheSize); @@ -941,6 +938,7 @@ Long bootStrapDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot, Hive hiveDb) dumpRoot.toUri(), bootDumpBeginReplId, bootDumpEndReplId); long executorId = conf.getLong(Constants.SCHEDULED_QUERY_EXECUTIONID, 0L); dmd.setDump(DumpType.BOOTSTRAP, bootDumpBeginReplId, bootDumpEndReplId, cmRoot, executorId); + dmd.setReplScope(work.replScope); dmd.write(true); work.setFunctionCopyPathIterator(functionsBinaryCopyPaths.iterator()); setDataCopyIterators(extTableFileList, managedTblList);