diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java index 9d5e8af30c..931943a750 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java @@ -123,20 +123,20 @@ private String replicateAndVerify(String replPolicy, String lastReplId, List dumpWithClause, List loadWithClause, String[] expectedTables) throws Throwable { - return replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, loadWithClause, + return replicateAndVerify(replPolicy, lastReplId, dumpWithClause, loadWithClause, null, expectedTables); } - private String replicateAndVerify(String replPolicy, String oldReplPolicy, String lastReplId, + private String replicateAndVerify(String replPolicy, String lastReplId, List dumpWithClause, List loadWithClause, String[] bootstrappedTables, String[] expectedTables) throws Throwable { - return replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, dumpWithClause, loadWithClause, + return replicateAndVerify(replPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrappedTables, expectedTables, null); } - private String replicateAndVerify(String replPolicy, String oldReplPolicy, String lastReplId, + private String replicateAndVerify(String replPolicy, String lastReplId, List dumpWithClause, List loadWithClause, String[] bootstrappedTables, @@ -154,7 +154,7 @@ private String replicateAndVerify(String replPolicy, String oldReplPolicy, Strin replica.run("drop database if exists " + replicatedDbName + " cascade"); } - WarehouseInstance.Tuple tuple = primary.dump(replPolicy, oldReplPolicy, dumpWithClause); + WarehouseInstance.Tuple tuple = primary.dump(replPolicy, dumpWithClause); if (bootstrappedTables != null) { verifyBootstrapDirInIncrementalDump(tuple.dumpLocation, bootstrappedTables); @@ -198,7 +198,7 @@ private String replicateAndVerifyClearDump(String replPolicy, String oldReplPoli replica.run("drop database if exists " + replicatedDbName + " cascade"); } - WarehouseInstance.Tuple tuple = primary.dump(replPolicy, oldReplPolicy, dumpWithClause); + WarehouseInstance.Tuple tuple = primary.dump(replPolicy, dumpWithClause); if (bootstrappedTables != null) { verifyBootstrapDirInIncrementalDump(tuple.dumpLocation, bootstrappedTables); @@ -377,24 +377,21 @@ public void testIncorrectTablePolicyInReplDump() throws Throwable { .dump(primaryDbName); replica.load(replicatedDbName, primaryDbName); String lastReplId = tupleBootstrap.lastReplicationId; - for (String oldReplPolicy : invalidReplPolicies) { - failed = false; - try { - replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, null, replicatedTables); - } catch (Exception ex) { - LOG.info("Got exception: {}", ex.getMessage()); - Assert.assertTrue(ex instanceof ParseException); - failed = true; - } - Assert.assertTrue(failed); + failed = false; + try { + replicateAndVerify(replPolicy, lastReplId, null, null, null, replicatedTables); + } catch (Exception ex) { + LOG.info("Got exception: {}", ex.getMessage()); + Assert.assertTrue(ex instanceof ParseException); + failed = true; } + Assert.assertTrue(failed); // Replace with replication policy having different DB name. - String oldReplPolicy = replPolicy; replPolicy = primaryDbName + "_dupe.'t1+'.'t1'"; failed = false; try { - replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, null, replicatedTables); + replicateAndVerify(replPolicy, lastReplId, null, null, null, replicatedTables); } catch (Exception ex) { LOG.info("Got exception: {}", ex.getMessage()); Assert.assertTrue(ex instanceof SemanticException); @@ -452,11 +449,10 @@ public void testCaseInSensitiveNatureOfReplPolicy() throws Throwable { String lastReplId = replicateAndVerify(replPolicy, null, null, null, replicatedTables); // Test case insensitive nature in REPLACE clause as well. - String oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".'(.*a1+)|(cc3)|(B2)'.'AA1+'"; replicatedTables = new String[] {"a1", "b2", "cc3"}; String[] bootstrappedTables = new String[] {"b2"}; - replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); + replicateAndVerify(replPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); } @Test @@ -566,33 +562,29 @@ public void testBasicReplaceReplPolicy() throws Throwable { // Replicate and verify if only 2 tables are replicated to target. String replPolicy = primaryDbName + ".'(t1)|(t4)'"; - String oldReplPolicy = null; String[] replicatedTables = new String[] {"t1", "t4"}; String lastReplId = replicateAndVerify(replPolicy, null, null, null, replicatedTables); // Exclude t4 and include t3, t6 createTables(new String[] {"t6"}, CreateTableType.MM_ACID); - oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".'t1|t3|t6'"; replicatedTables = new String[] {"t1", "t3", "t6"}; String[] bootstrappedTables = new String[] {"t3", "t6"}; - lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, + lastReplId = replicateAndVerify(replPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); // Convert to Full Db repl policy. All tables should be included. - oldReplPolicy = replPolicy; replPolicy = primaryDbName; replicatedTables = new String[] {"t1", "t2", "t3", "t4", "t5", "t6"}; bootstrappedTables = new String[] {"t2", "t4", "t5"}; - replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, + replicateAndVerify(replPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); // Convert to regex that excludes t3, t4 and t5. - oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".'.*?'.'t[3-5]+'"; replicatedTables = new String[] {"t1", "t2", "t6"}; bootstrappedTables = new String[] {}; - replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, + replicateAndVerify(replPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); } @@ -615,14 +607,13 @@ public void testReplacePolicyOnBootstrapAcidTablesIncrementalPhase() throws Thro // Enable acid tables for replication. Also, replace, replication policy to exclude "b1" and "a3" // instead of "a1" alone. - String oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".'[a-z]+[0-9]+'.'(a3)|(b1)'"; List dumpWithAcidBootstrapClause = Arrays.asList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES + "'='true'"); String[] incrementalReplicatedTables = new String[] {"a1", "a2", "a4", "b2", "c1"}; String[] bootstrappedTables = new String[] {"a2", "a4", "b2", "c1"}; - replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, + replicateAndVerify(replPolicy, lastReplId, dumpWithAcidBootstrapClause, null, bootstrappedTables, incrementalReplicatedTables); } @@ -644,11 +635,10 @@ public void testReplacePolicyWhenAcidTablesDisabledForRepl() throws Throwable { // Continue to disable ACID tables for replication. Also, replace, replication policy to include // "a2" but exclude "a1" and "b1". Still ACID tables shouldn't be bootstrapped. Only non-ACID // table "b1" should be bootstrapped. - String oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'a2'"; String[] incrementalReplicatedTables = new String[] {"a1", "b1"}; String[] bootstrappedTables = new String[] {"b1"}; - lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, + lastReplId = replicateAndVerify(replPolicy, lastReplId, dumpWithoutAcidClause, null, bootstrappedTables, incrementalReplicatedTables); } @@ -669,23 +659,21 @@ public void testReplacePolicyOnBootstrapExternalTablesIncrementalPhase() throws // Continue to disable external tables for replication. Also, replace, replication policy to exclude // "b1" and include "a1". - String oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'(a2)|(b1)'"; String[] incrementalReplicatedTables = new String[] {"a1"}; String[] bootstrappedTables = new String[] {"a1"}; - lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, + lastReplId = replicateAndVerify(replPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrappedTables, incrementalReplicatedTables); // Enable external tables replication and bootstrap in incremental phase. Also, replace, // replication policy to exclude tables with prefix "b". - oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".'[a-z]+[0-9]+'.'b[0-9]+'"; incrementalReplicatedTables = new String[] {"a1", "a2", "c2"}; bootstrappedTables = new String[] {"a2", "c2"}; dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'"); WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName) - .dump(replPolicy, oldReplPolicy, dumpWithClause); + .dump(replPolicy, dumpWithClause); loadWithClause = ReplicationTestUtils.includeExternalTableClause(true); String hiveDumpDir = tuple.dumpLocation + File.separator + ReplUtils.REPL_HIVE_BASE_DIR; @@ -719,7 +707,7 @@ public void testRenameTableScenariosBasic() throws Throwable { // Replicate and verify if only 2 tables are replicated to target. String[] replicatedTables = new String[] {"in1", "in2"}; String[] bootstrapTables = new String[] {}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null, + lastReplId = replicateAndVerify(replPolicy, lastReplId, null, null, bootstrapTables, replicatedTables); // Rename tables to make them satisfy the filter. @@ -730,7 +718,7 @@ public void testRenameTableScenariosBasic() throws Throwable { replicatedTables = new String[] {"in1", "in2", "in3", "in4", "in5"}; bootstrapTables = new String[] {"in3", "in4", "in5"}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null, + lastReplId = replicateAndVerify(replPolicy, lastReplId, null, null, bootstrapTables, replicatedTables); primary.run("use " + primaryDbName) @@ -742,7 +730,7 @@ public void testRenameTableScenariosBasic() throws Throwable { replicatedTables = new String[] {"in1", "in2", "in8", "in11"}; bootstrapTables = new String[] {"in11"}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null, + lastReplId = replicateAndVerify(replPolicy, lastReplId, null, null, bootstrapTables, replicatedTables); primary.run("use " + primaryDbName) @@ -757,7 +745,7 @@ public void testRenameTableScenariosBasic() throws Throwable { replicatedTables = new String[] {"in200", "in12", "in11", "in14"}; bootstrapTables = new String[] {"in14", "in200"}; - replicateAndVerify(replPolicy, null, lastReplId, null, + replicateAndVerify(replPolicy, lastReplId, null, null, bootstrapTables, replicatedTables); } @@ -775,7 +763,7 @@ public void testRenameTableScenariosWithDmlOperations() throws Throwable { // Replicate and verify if only 2 tables are replicated to target. String[] replicatedTables = new String[] {"in1", "in100"}; String[] bootstrapTables = new String[] {}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null, + lastReplId = replicateAndVerify(replPolicy, lastReplId, null, null, bootstrapTables, replicatedTables); // Rename tables and do some operations. @@ -789,7 +777,7 @@ public void testRenameTableScenariosWithDmlOperations() throws Throwable { replicatedTables = new String[] {"in4", "in400"}; bootstrapTables = new String[] {"in4", "in400"}; - replicateAndVerify(replPolicy, null, lastReplId, null, + replicateAndVerify(replPolicy, lastReplId, null, null, bootstrapTables, replicatedTables, new String[] {"1", "2"}); } @@ -813,7 +801,7 @@ public void testRenameTableScenariosAcidTable() throws Throwable { // Replicate and verify if only 1 tables are replicated to target. Acid tables are not dumped. String[] replicatedTables = new String[] {"in1"}; String[] bootstrapTables = new String[] {}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, + lastReplId = replicateAndVerify(replPolicy, lastReplId, dumpWithClause, null, bootstrapTables, replicatedTables); // Rename tables to make them satisfy the filter and enable acid tables. @@ -826,7 +814,7 @@ public void testRenameTableScenariosAcidTable() throws Throwable { "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'"); replicatedTables = new String[] {"in1", "in2", "in3", "in4", "in5"}; bootstrapTables = new String[] {"in2", "in3", "in4", "in5"}; - replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, + replicateAndVerify(replPolicy, lastReplId, dumpWithClause, null, bootstrapTables, replicatedTables); } @@ -840,7 +828,7 @@ public void testRenameTableScenariosExternalTable() throws Throwable { "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='false'", "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'" ); - String lastReplId = replicateAndVerify(replPolicy, null, null, dumpWithClause, + String lastReplId = replicateAndVerify(replPolicy, null, dumpWithClause, loadWithClause, new String[] {}, new String[] {}); String[] originalNonAcidTables = new String[] {"in1", "out4"}; @@ -853,7 +841,7 @@ public void testRenameTableScenariosExternalTable() throws Throwable { // Replicate and verify if only 1 tables are replicated to target. Acid and external tables are not dumped. String[] replicatedTables = new String[] {"in1"}; String[] bootstrapTables = new String[] {}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, + lastReplId = replicateAndVerify(replPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrapTables, replicatedTables); // Rename tables to make them satisfy the filter and enable acid and external tables. @@ -871,7 +859,7 @@ public void testRenameTableScenariosExternalTable() throws Throwable { loadWithClause = ReplicationTestUtils.includeExternalTableClause(true); replicatedTables = new String[] {"in1", "in2", "in3", "in4", "in5"}; bootstrapTables = new String[] {"in2", "in3", "in4", "in5"}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, + lastReplId = replicateAndVerify(replPolicy, lastReplId, dumpWithClause, loadWithClause, null, replicatedTables); dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", @@ -886,7 +874,7 @@ public void testRenameTableScenariosExternalTable() throws Throwable { replicatedTables = new String[] {"in2", "in3", "in4", "in11", "in6", "in7"}; bootstrapTables = new String[] {"in11", "in6", "in7"}; - replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, + replicateAndVerify(replPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrapTables, replicatedTables); } @@ -895,7 +883,7 @@ public void testRenameTableScenariosWithReplaceExternalTable() throws Throwable List loadWithClause = ReplicationTestUtils.includeExternalTableClause(true); List dumpWithClause = ReplicationTestUtils.externalTableWithClause(loadWithClause, true, true); String replPolicy = primaryDbName + ".'(in[0-9]+)|(out4)|(out5)|(out1500)'"; - String lastReplId = replicateAndVerify(replPolicy, null, null, dumpWithClause, + String lastReplId = replicateAndVerify(replPolicy, null, dumpWithClause, loadWithClause, new String[] {}, new String[] {}); String[] originalExternalTables = new String[] {"in1", "in2", "out3", "out4", "out10", "out11", "out1500"}; @@ -923,7 +911,7 @@ public void testRenameTableScenariosWithReplaceExternalTable() throws Throwable // in2 should be dropped. String[] replicatedTables = new String[] {"in5", "in6", "in7", "in12"}; String[] bootstrapTables = new String[] {"in5", "in6", "in7", "in12"}; - replicateAndVerify(newPolicy, replPolicy, lastReplId, dumpWithClause, + replicateAndVerify(newPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrapTables, replicatedTables); } @@ -942,7 +930,7 @@ public void testRenameTableScenariosWithReplacePolicyDMLOperattion() throws Thro // Replicate and verify if only 4 tables are replicated to target. String[] replicatedTables = new String[] {"in1", "in2", "in300", "in400"}; String[] bootstrapTables = new String[] {}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null, + lastReplId = replicateAndVerify(replPolicy, lastReplId, null, null, bootstrapTables, replicatedTables); // Rename the tables to satisfy the condition also replace the policy. @@ -977,11 +965,10 @@ public void testRenameTableScenariosWithReplacePolicyDMLOperattion() throws Thro // in2 should be dropped. replicatedTables = new String[] {"in100", "in2000", "in8", "in305", "in500", "in3000", "in5000"}; bootstrapTables = new String[] {"in500", "in8", "in5000", "in305", "in3000", "in2000", "in100"}; - lastReplId = replicateAndVerify(newPolicy, replPolicy, lastReplId, null, + lastReplId = replicateAndVerify(newPolicy, lastReplId, null, null, bootstrapTables, replicatedTables, new String[] {"1", "2"}); // No table filter - replPolicy = newPolicy; newPolicy = primaryDbName; primary.run("use " + primaryDbName) .run("alter table in100 rename to in12") // should be just rename, but for replace its always bootstrap @@ -999,7 +986,7 @@ public void testRenameTableScenariosWithReplacePolicyDMLOperattion() throws Thro replicatedTables = new String[] {"in12", "in400", "in1", "in300", "out12", "in500", "in3000", "in2", "in5000", "out5002"}; bootstrapTables = new String[] {"out12", "in2", "in400", "in1", "in300", "in12", "out5002"}; - replicateAndVerify(newPolicy, replPolicy, lastReplId, null, + replicateAndVerify(newPolicy, lastReplId, null, null, bootstrapTables, replicatedTables, new String[] {"1", "2"}); } @@ -1049,7 +1036,7 @@ public void testRenameTableScenariosUpgrade() throws Throwable { replicatedTables = new String[] {"in1", "in2", "in3", "in4", "in5", "in6", "in7", "in9"}; String[] bootstrapTables = new String[] {"in2", "in3", "in4", "in5", "in6", "in9"}; - lastReplId = replicateAndVerify(newReplPolicy, replPolicy, lastReplId, dumpWithClause, + lastReplId = replicateAndVerify(newReplPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrapTables, replicatedTables); primary.run("use " + primaryDbName) @@ -1067,7 +1054,7 @@ public void testRenameTableScenariosUpgrade() throws Throwable { replicatedTables = new String[] {"in1", "in2", "in3", "out4", "out5", "out6", "in7", "in8", "in9", "out10", "out11", "out12"}; bootstrapTables = new String[] {"out4", "out5", "out6", "in8", "out10", "out11", "out12"}; - replicateAndVerify(replPolicy, newReplPolicy, lastReplId, dumpWithClause, + replicateAndVerify(replPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrapTables, replicatedTables); } } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java index ed12478bdd..89e535d42a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java @@ -268,27 +268,16 @@ Tuple dump(String dbName) return dump(dbName, Collections.emptyList()); } - Tuple dump(String dbName, List withClauseOptions) + Tuple dump(String dumpExpression, List withClauseOptions) throws Throwable { String dumpCommand = - "REPL DUMP " + dbName; + "REPL DUMP " + dumpExpression; if (withClauseOptions != null && !withClauseOptions.isEmpty()) { dumpCommand += " with (" + StringUtils.join(withClauseOptions, ",") + ")"; } return dumpWithCommand(dumpCommand); } - Tuple dump(String replPolicy, String oldReplPolicy, List withClauseOptions) - throws Throwable { - String dumpCommand = - "REPL DUMP " + replPolicy - + (oldReplPolicy == null ? "" : " REPLACE " + oldReplPolicy); - if (!withClauseOptions.isEmpty()) { - dumpCommand += " with (" + StringUtils.join(withClauseOptions, ",") + ")"; - } - return dumpWithCommand(dumpCommand); - } - Tuple dumpWithCommand(String dumpCommand) throws Throwable { advanceDumpDir(); run(dumpCommand); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java index 1fce791cbe..fc2f512cc5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java @@ -54,7 +54,6 @@ import org.apache.hadoop.hive.ql.exec.util.DAGTraversal; import org.apache.hadoop.hive.ql.exec.util.Retryable; import org.apache.hadoop.hive.ql.io.AcidUtils; -import org.apache.hadoop.hive.ql.io.orc.ExternalCache; import org.apache.hadoop.hive.ql.lockmgr.DbLockManager; import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager; import org.apache.hadoop.hive.ql.lockmgr.LockException; @@ -164,6 +163,9 @@ public int execute() { Path previousValidHiveDumpPath = getPreviousValidDumpMetadataPath(dumpRoot); boolean isBootstrap = (previousValidHiveDumpPath == null); work.setBootstrap(isBootstrap); + if (previousValidHiveDumpPath != null) { + work.setOldReplScope(new DumpMetaData(previousValidHiveDumpPath, conf).getReplScope()); + } //If no previous dump is present or previous dump is already loaded, proceed with the dump operation. if (shouldDump(previousValidHiveDumpPath)) { Path currentDumpPath = getCurrentDumpPath(dumpRoot, isBootstrap); @@ -411,12 +413,16 @@ private boolean shouldDump(Path previousDumpPath) throws IOException { * @return true if need to examine tables for dump and false if not. */ private boolean shouldExamineTablesToDump() { - return (work.oldReplScope != null) + return (previousReplScopeModified()) || !tablesForBootstrap.isEmpty() || conf.getBoolVar(HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES) || conf.getBoolVar(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES); } + private boolean previousReplScopeModified() { + return work.oldReplScope != null && !work.oldReplScope.equals(work.replScope); + } + /** * Decide whether to dump external tables data. If external tables are enabled for replication, * then need to dump it's data in all the incremental dumps. @@ -764,7 +770,7 @@ private boolean needBootstrapAcidTablesDuringIncrementalDump() { // may not satisfying the old policy but satisfying the new policy. For filter, it may happen that the table // is renamed and started satisfying the policy. return ((!work.replScope.includeAllTables()) - || (work.oldReplScope != null) + || (previousReplScopeModified()) || conf.getBoolVar(HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java index aecfe757a4..47051e41ee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hive.ql.exec.ReplCopyTask; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; -import org.apache.hadoop.hive.ql.exec.repl.util.FileList; import org.apache.hadoop.hive.ql.exec.repl.util.TaskTracker; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.parse.EximUtil; @@ -47,7 +46,7 @@ private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(ReplDumpWork.class); final ReplScope replScope; - final ReplScope oldReplScope; + ReplScope oldReplScope; final String dbNameOrPattern, astRepresentationForErrorMsg, resultTempPath; Long eventTo; Long eventFrom; @@ -88,16 +87,19 @@ public static void testDeletePreviousDumpMetaPath(boolean failDeleteDumpMeta) { testDeletePreviousDumpMetaPath = failDeleteDumpMeta; } - public ReplDumpWork(ReplScope replScope, ReplScope oldReplScope, + public ReplDumpWork(ReplScope replScope, String astRepresentationForErrorMsg, String resultTempPath) { this.replScope = replScope; - this.oldReplScope = oldReplScope; this.dbNameOrPattern = replScope.getDbName(); this.astRepresentationForErrorMsg = astRepresentationForErrorMsg; this.resultTempPath = resultTempPath; } + void setOldReplScope(ReplScope replScope) { + oldReplScope = replScope; + } + int maxEventLimit() throws Exception { if (eventTo < eventFrom) { throw new Exception("Invalid event ID input received in TO clause"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java index 4c10499e5e..313474c7f6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java @@ -65,7 +65,6 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { // Replication Scope private ReplScope replScope = new ReplScope(); - private ReplScope oldReplScope = null; // Source DB Name for REPL LOAD private String sourceDbNameOrPattern; @@ -143,34 +142,6 @@ private void setReplDumpTablesList(Tree replTablesNode, ReplScope replScope) thr } } - private void setOldReplPolicy(Tree oldReplPolicyTree) throws HiveException { - oldReplScope = new ReplScope(); - int childCount = oldReplPolicyTree.getChildCount(); - - // First child is DB name and optional second child is tables list. - assert(childCount <= 2); - - // First child is always the DB name. So set it. - oldReplScope.setDbName(oldReplPolicyTree.getChild(0).getText()); - LOG.info("Old ReplScope: Set DB Name: {}", oldReplScope.getDbName()); - if (!oldReplScope.getDbName().equalsIgnoreCase(replScope.getDbName())) { - LOG.error("DB name {} cannot be replaced to {} in the replication policy.", - oldReplScope.getDbName(), replScope.getDbName()); - throw new SemanticException("DB name cannot be replaced in the replication policy."); - } - - // If the old policy is just , then tables list won't be there. - if (childCount <= 1) { - return; - } - - // Traverse the children which can be either just include tables list or both include - // and exclude tables lists. - Tree oldPolicyTablesListNode = oldReplPolicyTree.getChild(1); - assert(oldPolicyTablesListNode.getType() == TOK_REPL_TABLES); - setReplDumpTablesList(oldPolicyTablesListNode, oldReplScope); - } - private void initReplDump(ASTNode ast) throws HiveException { int numChildren = ast.getChildCount(); boolean isMetaDataOnly = false; @@ -196,9 +167,6 @@ private void initReplDump(ASTNode ast) throws HiveException { case TOK_REPL_TABLES: setReplDumpTablesList(currNode, replScope); break; - case TOK_REPLACE: - setOldReplPolicy(currNode); - break; default: throw new SemanticException("Unrecognized token " + currNode.getType() + " in REPL DUMP statement."); } @@ -237,7 +205,6 @@ private void analyzeReplDump(ASTNode ast) throws SemanticException { Task replDumpWorkTask = TaskFactory .get(new ReplDumpWork( replScope, - oldReplScope, ASTErrorUtils.getMsg(ErrorMsg.INVALID_PATH.getMsg(), ast), ctx.getResFile().toUri().toString() ), conf); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestReplDumpTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestReplDumpTask.java index 3f47678988..aede902d4a 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestReplDumpTask.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestReplDumpTask.java @@ -147,8 +147,7 @@ void dumpTable(String dbName, String tblName, String validTxnList, }; task.initialize(queryState, null, null, null); - ReplDumpWork replDumpWork = new ReplDumpWork(replScope, - null, "", ""); + ReplDumpWork replDumpWork = new ReplDumpWork(replScope, "", ""); replDumpWork.setMetricCollector(metricCollector); task.setWork(replDumpWork); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java index 8b26f1ce40..b9dbad77a6 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.common.repl; import java.io.Serializable; +import java.util.Objects; import java.util.regex.Pattern; /** @@ -116,4 +117,25 @@ private boolean inTableExcludedList(final String tableName) { } return excludedTableNamePattern.matcher(tableName).matches(); } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (! (o instanceof ReplScope)) { + return false; + } + + ReplScope replScope = (ReplScope)o; + return (replScope.getExcludedTableNames().equals(this.excludedTableNames) + && replScope.getIncludedTableNames().equals(this.includedTableNames) + && replScope.dbName.equals(this.dbName) + && replScope.dbNamePattern.equals(this.dbNamePattern)); + } + + @Override + public int hashCode() { + return Objects.hash(dbName, dbNamePattern, includedTableNames, excludedTableNames); + } }