diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java index 4d47254e0c..4dfb3956ed 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java @@ -125,20 +125,20 @@ private String replicateAndVerify(String replPolicy, String lastReplId, List dumpWithClause, List loadWithClause, String[] expectedTables) throws Throwable { - return replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, loadWithClause, + return replicateAndVerify(replPolicy, lastReplId, dumpWithClause, loadWithClause, null, expectedTables); } - private String replicateAndVerify(String replPolicy, String oldReplPolicy, String lastReplId, + private String replicateAndVerify(String replPolicy, String lastReplId, List dumpWithClause, List loadWithClause, String[] bootstrappedTables, String[] expectedTables) throws Throwable { - return replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, dumpWithClause, loadWithClause, + return replicateAndVerify(replPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrappedTables, expectedTables, null); } - private String replicateAndVerify(String replPolicy, String oldReplPolicy, String lastReplId, + private String replicateAndVerify(String replPolicy, String lastReplId, List dumpWithClause, List loadWithClause, String[] bootstrappedTables, @@ -156,7 +156,7 @@ private String replicateAndVerify(String replPolicy, String oldReplPolicy, Strin replica.run("drop database if exists " + replicatedDbName + " cascade"); } - WarehouseInstance.Tuple tuple = primary.dump(replPolicy, oldReplPolicy, dumpWithClause); + WarehouseInstance.Tuple tuple = primary.dump(replPolicy, dumpWithClause); DumpMetaData dumpMetaData = new DumpMetaData(new Path(tuple.dumpLocation, ReplUtils.REPL_HIVE_BASE_DIR), conf); Assert.assertEquals(oldReplPolicy != null && !replPolicy.equals(oldReplPolicy), @@ -221,7 +221,7 @@ private String replicateAndVerifyClearDump(String replPolicy, String oldReplPoli replica.run("drop database if exists " + replicatedDbName + " cascade"); } - WarehouseInstance.Tuple tuple = primary.dump(replPolicy, oldReplPolicy, dumpWithClause); + WarehouseInstance.Tuple tuple = primary.dump(replPolicy, dumpWithClause); if (bootstrappedTables != null) { verifyBootstrapDirInIncrementalDump(tuple.dumpLocation, bootstrappedTables); @@ -400,24 +400,21 @@ public void testIncorrectTablePolicyInReplDump() throws Throwable { .dump(primaryDbName); replica.load(replicatedDbName, primaryDbName); String lastReplId = tupleBootstrap.lastReplicationId; - for (String oldReplPolicy : invalidReplPolicies) { - failed = false; - try { - replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, null, replicatedTables); - } catch (Exception ex) { - LOG.info("Got exception: {}", ex.getMessage()); - Assert.assertTrue(ex instanceof ParseException); - failed = true; - } - Assert.assertTrue(failed); + failed = false; + try { + replicateAndVerify(replPolicy, lastReplId, null, null, null, replicatedTables); + } catch (Exception ex) { + LOG.info("Got exception: {}", ex.getMessage()); + Assert.assertTrue(ex instanceof ParseException); + failed = true; } + Assert.assertTrue(failed); // Replace with replication policy having different DB name. - String oldReplPolicy = replPolicy; replPolicy = primaryDbName + "_dupe.'t1+'.'t1'"; failed = false; try { - replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, null, replicatedTables); + replicateAndVerify(replPolicy, lastReplId, null, null, null, replicatedTables); } catch (Exception ex) { LOG.info("Got exception: {}", ex.getMessage()); Assert.assertTrue(ex instanceof SemanticException); @@ -475,11 +472,10 @@ public void testCaseInSensitiveNatureOfReplPolicy() throws Throwable { String lastReplId = replicateAndVerify(replPolicy, null, null, null, replicatedTables); // Test case insensitive nature in REPLACE clause as well. - String oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".'(.*a1+)|(cc3)|(B2)'.'AA1+'"; replicatedTables = new String[] {"a1", "b2", "cc3"}; String[] bootstrappedTables = new String[] {"b2"}; - replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); + replicateAndVerify(replPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); } @Test @@ -589,33 +585,29 @@ public void testBasicReplaceReplPolicy() throws Throwable { // Replicate and verify if only 2 tables are replicated to target. String replPolicy = primaryDbName + ".'(t1)|(t4)'"; - String oldReplPolicy = null; String[] replicatedTables = new String[] {"t1", "t4"}; String lastReplId = replicateAndVerify(replPolicy, null, null, null, replicatedTables); // Exclude t4 and include t3, t6 createTables(new String[] {"t6"}, CreateTableType.MM_ACID); - oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".'t1|t3|t6'"; replicatedTables = new String[] {"t1", "t3", "t6"}; String[] bootstrappedTables = new String[] {"t3", "t6"}; - lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, + lastReplId = replicateAndVerify(replPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); // Convert to Full Db repl policy. All tables should be included. - oldReplPolicy = replPolicy; replPolicy = primaryDbName; replicatedTables = new String[] {"t1", "t2", "t3", "t4", "t5", "t6"}; bootstrappedTables = new String[] {"t2", "t4", "t5"}; - replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, + replicateAndVerify(replPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); // Convert to regex that excludes t3, t4 and t5. - oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".'.*?'.'t[3-5]+'"; replicatedTables = new String[] {"t1", "t2", "t6"}; bootstrappedTables = new String[] {}; - replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, + replicateAndVerify(replPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); } @@ -638,14 +630,13 @@ public void testReplacePolicyOnBootstrapAcidTablesIncrementalPhase() throws Thro // Enable acid tables for replication. Also, replace, replication policy to exclude "b1" and "a3" // instead of "a1" alone. - String oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".'[a-z]+[0-9]+'.'(a3)|(b1)'"; List dumpWithAcidBootstrapClause = Arrays.asList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES + "'='true'"); String[] incrementalReplicatedTables = new String[] {"a1", "a2", "a4", "b2", "c1"}; String[] bootstrappedTables = new String[] {"a2", "a4", "b2", "c1"}; - replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, + replicateAndVerify(replPolicy, lastReplId, dumpWithAcidBootstrapClause, null, bootstrappedTables, incrementalReplicatedTables); } @@ -667,11 +658,10 @@ public void testReplacePolicyWhenAcidTablesDisabledForRepl() throws Throwable { // Continue to disable ACID tables for replication. Also, replace, replication policy to include // "a2" but exclude "a1" and "b1". Still ACID tables shouldn't be bootstrapped. Only non-ACID // table "b1" should be bootstrapped. - String oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'a2'"; String[] incrementalReplicatedTables = new String[] {"a1", "b1"}; String[] bootstrappedTables = new String[] {"b1"}; - lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, + lastReplId = replicateAndVerify(replPolicy, lastReplId, dumpWithoutAcidClause, null, bootstrappedTables, incrementalReplicatedTables); } @@ -692,23 +682,21 @@ public void testReplacePolicyOnBootstrapExternalTablesIncrementalPhase() throws // Continue to disable external tables for replication. Also, replace, replication policy to exclude // "b1" and include "a1". - String oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".'(a[0-9]+)|(b[0-9]+)'.'(a2)|(b1)'"; String[] incrementalReplicatedTables = new String[] {"a1"}; String[] bootstrappedTables = new String[] {"a1"}; - lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, + lastReplId = replicateAndVerify(replPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrappedTables, incrementalReplicatedTables); // Enable external tables replication and bootstrap in incremental phase. Also, replace, // replication policy to exclude tables with prefix "b". - oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".'[a-z]+[0-9]+'.'b[0-9]+'"; incrementalReplicatedTables = new String[] {"a1", "a2", "c2"}; bootstrappedTables = new String[] {"a2", "c2"}; dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'"); WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName) - .dump(replPolicy, oldReplPolicy, dumpWithClause); + .dump(replPolicy, dumpWithClause); loadWithClause = ReplicationTestUtils.includeExternalTableClause(true); String hiveDumpDir = tuple.dumpLocation + File.separator + ReplUtils.REPL_HIVE_BASE_DIR; @@ -742,7 +730,7 @@ public void testRenameTableScenariosBasic() throws Throwable { // Replicate and verify if only 2 tables are replicated to target. String[] replicatedTables = new String[] {"in1", "in2"}; String[] bootstrapTables = new String[] {}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null, + lastReplId = replicateAndVerify(replPolicy, lastReplId, null, null, bootstrapTables, replicatedTables); // Rename tables to make them satisfy the filter. @@ -753,7 +741,7 @@ public void testRenameTableScenariosBasic() throws Throwable { replicatedTables = new String[] {"in1", "in2", "in3", "in4", "in5"}; bootstrapTables = new String[] {"in3", "in4", "in5"}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null, + lastReplId = replicateAndVerify(replPolicy, lastReplId, null, null, bootstrapTables, replicatedTables); primary.run("use " + primaryDbName) @@ -765,7 +753,7 @@ public void testRenameTableScenariosBasic() throws Throwable { replicatedTables = new String[] {"in1", "in2", "in8", "in11"}; bootstrapTables = new String[] {"in11"}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null, + lastReplId = replicateAndVerify(replPolicy, lastReplId, null, null, bootstrapTables, replicatedTables); primary.run("use " + primaryDbName) @@ -780,7 +768,7 @@ public void testRenameTableScenariosBasic() throws Throwable { replicatedTables = new String[] {"in200", "in12", "in11", "in14"}; bootstrapTables = new String[] {"in14", "in200"}; - replicateAndVerify(replPolicy, null, lastReplId, null, + replicateAndVerify(replPolicy, lastReplId, null, null, bootstrapTables, replicatedTables); } @@ -798,7 +786,7 @@ public void testRenameTableScenariosWithDmlOperations() throws Throwable { // Replicate and verify if only 2 tables are replicated to target. String[] replicatedTables = new String[] {"in1", "in100"}; String[] bootstrapTables = new String[] {}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null, + lastReplId = replicateAndVerify(replPolicy, lastReplId, null, null, bootstrapTables, replicatedTables); // Rename tables and do some operations. @@ -812,7 +800,7 @@ public void testRenameTableScenariosWithDmlOperations() throws Throwable { replicatedTables = new String[] {"in4", "in400"}; bootstrapTables = new String[] {"in4", "in400"}; - replicateAndVerify(replPolicy, null, lastReplId, null, + replicateAndVerify(replPolicy, lastReplId, null, null, bootstrapTables, replicatedTables, new String[] {"1", "2"}); } @@ -836,7 +824,7 @@ public void testRenameTableScenariosAcidTable() throws Throwable { // Replicate and verify if only 1 tables are replicated to target. Acid tables are not dumped. String[] replicatedTables = new String[] {"in1"}; String[] bootstrapTables = new String[] {}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, + lastReplId = replicateAndVerify(replPolicy, lastReplId, dumpWithClause, null, bootstrapTables, replicatedTables); // Rename tables to make them satisfy the filter and enable acid tables. @@ -849,7 +837,7 @@ public void testRenameTableScenariosAcidTable() throws Throwable { "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'"); replicatedTables = new String[] {"in1", "in2", "in3", "in4", "in5"}; bootstrapTables = new String[] {"in2", "in3", "in4", "in5"}; - replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, + replicateAndVerify(replPolicy, lastReplId, dumpWithClause, null, bootstrapTables, replicatedTables); } @@ -863,7 +851,7 @@ public void testRenameTableScenariosExternalTable() throws Throwable { "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='false'", "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'" ); - String lastReplId = replicateAndVerify(replPolicy, null, null, dumpWithClause, + String lastReplId = replicateAndVerify(replPolicy, null, dumpWithClause, loadWithClause, new String[] {}, new String[] {}); String[] originalNonAcidTables = new String[] {"in1", "out4"}; @@ -876,7 +864,7 @@ public void testRenameTableScenariosExternalTable() throws Throwable { // Replicate and verify if only 1 tables are replicated to target. Acid and external tables are not dumped. String[] replicatedTables = new String[] {"in1"}; String[] bootstrapTables = new String[] {}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, + lastReplId = replicateAndVerify(replPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrapTables, replicatedTables); // Rename tables to make them satisfy the filter and enable acid and external tables. @@ -894,7 +882,7 @@ public void testRenameTableScenariosExternalTable() throws Throwable { loadWithClause = ReplicationTestUtils.includeExternalTableClause(true); replicatedTables = new String[] {"in1", "in2", "in3", "in4", "in5"}; bootstrapTables = new String[] {"in2", "in3", "in4", "in5"}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, + lastReplId = replicateAndVerify(replPolicy, lastReplId, dumpWithClause, loadWithClause, null, replicatedTables); dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", @@ -909,7 +897,7 @@ public void testRenameTableScenariosExternalTable() throws Throwable { replicatedTables = new String[] {"in2", "in3", "in4", "in11", "in6", "in7"}; bootstrapTables = new String[] {"in11", "in6", "in7"}; - replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, + replicateAndVerify(replPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrapTables, replicatedTables); } @@ -918,7 +906,7 @@ public void testRenameTableScenariosWithReplaceExternalTable() throws Throwable List loadWithClause = ReplicationTestUtils.includeExternalTableClause(true); List dumpWithClause = ReplicationTestUtils.externalTableWithClause(loadWithClause, true, true); String replPolicy = primaryDbName + ".'(in[0-9]+)|(out4)|(out5)|(out1500)'"; - String lastReplId = replicateAndVerify(replPolicy, null, null, dumpWithClause, + String lastReplId = replicateAndVerify(replPolicy, null, dumpWithClause, loadWithClause, new String[] {}, new String[] {}); String[] originalExternalTables = new String[] {"in1", "in2", "out3", "out4", "out10", "out11", "out1500"}; @@ -946,7 +934,7 @@ public void testRenameTableScenariosWithReplaceExternalTable() throws Throwable // in2 should be dropped. String[] replicatedTables = new String[] {"in5", "in6", "in7", "in12"}; String[] bootstrapTables = new String[] {"in5", "in6", "in7", "in12"}; - replicateAndVerify(newPolicy, replPolicy, lastReplId, dumpWithClause, + replicateAndVerify(newPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrapTables, replicatedTables); } @@ -965,7 +953,7 @@ public void testRenameTableScenariosWithReplacePolicyDMLOperattion() throws Thro // Replicate and verify if only 4 tables are replicated to target. String[] replicatedTables = new String[] {"in1", "in2", "in300", "in400"}; String[] bootstrapTables = new String[] {}; - lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null, + lastReplId = replicateAndVerify(replPolicy, lastReplId, null, null, bootstrapTables, replicatedTables); // Rename the tables to satisfy the condition also replace the policy. @@ -1000,11 +988,10 @@ public void testRenameTableScenariosWithReplacePolicyDMLOperattion() throws Thro // in2 should be dropped. replicatedTables = new String[] {"in100", "in2000", "in8", "in305", "in500", "in3000", "in5000"}; bootstrapTables = new String[] {"in500", "in8", "in5000", "in305", "in3000", "in2000", "in100"}; - lastReplId = replicateAndVerify(newPolicy, replPolicy, lastReplId, null, + lastReplId = replicateAndVerify(newPolicy, lastReplId, null, null, bootstrapTables, replicatedTables, new String[] {"1", "2"}); // No table filter - replPolicy = newPolicy; newPolicy = primaryDbName; primary.run("use " + primaryDbName) .run("alter table in100 rename to in12") // should be just rename, but for replace its always bootstrap @@ -1022,7 +1009,7 @@ public void testRenameTableScenariosWithReplacePolicyDMLOperattion() throws Thro replicatedTables = new String[] {"in12", "in400", "in1", "in300", "out12", "in500", "in3000", "in2", "in5000", "out5002"}; bootstrapTables = new String[] {"out12", "in2", "in400", "in1", "in300", "in12", "out5002"}; - replicateAndVerify(newPolicy, replPolicy, lastReplId, null, + replicateAndVerify(newPolicy, lastReplId, null, null, bootstrapTables, replicatedTables, new String[] {"1", "2"}); } @@ -1072,7 +1059,7 @@ public void testRenameTableScenariosUpgrade() throws Throwable { replicatedTables = new String[] {"in1", "in2", "in3", "in4", "in5", "in6", "in7", "in9"}; String[] bootstrapTables = new String[] {"in2", "in3", "in4", "in5", "in6", "in9"}; - lastReplId = replicateAndVerify(newReplPolicy, replPolicy, lastReplId, dumpWithClause, + lastReplId = replicateAndVerify(newReplPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrapTables, replicatedTables); primary.run("use " + primaryDbName) @@ -1090,7 +1077,7 @@ public void testRenameTableScenariosUpgrade() throws Throwable { replicatedTables = new String[] {"in1", "in2", "in3", "out4", "out5", "out6", "in7", "in8", "in9", "out10", "out11", "out12"}; bootstrapTables = new String[] {"out4", "out5", "out6", "in8", "out10", "out11", "out12"}; - replicateAndVerify(replPolicy, newReplPolicy, lastReplId, dumpWithClause, + replicateAndVerify(replPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrapTables, replicatedTables); } } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java index ed12478bdd..89e535d42a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java @@ -268,27 +268,16 @@ Tuple dump(String dbName) return dump(dbName, Collections.emptyList()); } - Tuple dump(String dbName, List withClauseOptions) + Tuple dump(String dumpExpression, List withClauseOptions) throws Throwable { String dumpCommand = - "REPL DUMP " + dbName; + "REPL DUMP " + dumpExpression; if (withClauseOptions != null && !withClauseOptions.isEmpty()) { dumpCommand += " with (" + StringUtils.join(withClauseOptions, ",") + ")"; } return dumpWithCommand(dumpCommand); } - Tuple dump(String replPolicy, String oldReplPolicy, List withClauseOptions) - throws Throwable { - String dumpCommand = - "REPL DUMP " + replPolicy - + (oldReplPolicy == null ? "" : " REPLACE " + oldReplPolicy); - if (!withClauseOptions.isEmpty()) { - dumpCommand += " with (" + StringUtils.join(withClauseOptions, ",") + ")"; - } - return dumpWithCommand(dumpCommand); - } - Tuple dumpWithCommand(String dumpCommand) throws Throwable { advanceDumpDir(); run(dumpCommand); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java index 7e690fce35..e5d459240c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java @@ -163,6 +163,9 @@ public int execute() { Path previousValidHiveDumpPath = getPreviousValidDumpMetadataPath(dumpRoot); boolean isBootstrap = (previousValidHiveDumpPath == null); work.setBootstrap(isBootstrap); + if (previousValidHiveDumpPath != null) { + work.setOldReplScope(new DumpMetaData(previousValidHiveDumpPath, conf).getReplScope()); + } //If no previous dump is present or previous dump is already loaded, proceed with the dump operation. if (shouldDump(previousValidHiveDumpPath)) { Path currentDumpPath = getCurrentDumpPath(dumpRoot, isBootstrap); @@ -410,12 +413,16 @@ private boolean shouldDump(Path previousDumpPath) throws IOException { * @return true if need to examine tables for dump and false if not. */ private boolean shouldExamineTablesToDump() { - return (work.oldReplScope != null) + return (previousReplScopeModified()) || !tablesForBootstrap.isEmpty() || conf.getBoolVar(HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES) || conf.getBoolVar(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES); } + private boolean previousReplScopeModified() { + return work.oldReplScope != null && !work.oldReplScope.equals(work.replScope); + } + /** * Decide whether to dump external tables data. If external tables are enabled for replication, * then need to dump it's data in all the incremental dumps. @@ -762,7 +769,7 @@ private boolean needBootstrapAcidTablesDuringIncrementalDump() { // may not satisfying the old policy but satisfying the new policy. For filter, it may happen that the table // is renamed and started satisfying the policy. return ((!work.replScope.includeAllTables()) - || (work.oldReplScope != null) + || (previousReplScopeModified()) || conf.getBoolVar(HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES)); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java index aecfe757a4..47051e41ee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java @@ -24,7 +24,6 @@ import org.apache.hadoop.hive.ql.exec.ReplCopyTask; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; -import org.apache.hadoop.hive.ql.exec.repl.util.FileList; import org.apache.hadoop.hive.ql.exec.repl.util.TaskTracker; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.parse.EximUtil; @@ -47,7 +46,7 @@ private static final long serialVersionUID = 1L; private static final Logger LOG = LoggerFactory.getLogger(ReplDumpWork.class); final ReplScope replScope; - final ReplScope oldReplScope; + ReplScope oldReplScope; final String dbNameOrPattern, astRepresentationForErrorMsg, resultTempPath; Long eventTo; Long eventFrom; @@ -88,16 +87,19 @@ public static void testDeletePreviousDumpMetaPath(boolean failDeleteDumpMeta) { testDeletePreviousDumpMetaPath = failDeleteDumpMeta; } - public ReplDumpWork(ReplScope replScope, ReplScope oldReplScope, + public ReplDumpWork(ReplScope replScope, String astRepresentationForErrorMsg, String resultTempPath) { this.replScope = replScope; - this.oldReplScope = oldReplScope; this.dbNameOrPattern = replScope.getDbName(); this.astRepresentationForErrorMsg = astRepresentationForErrorMsg; this.resultTempPath = resultTempPath; } + void setOldReplScope(ReplScope replScope) { + oldReplScope = replScope; + } + int maxEventLimit() throws Exception { if (eventTo < eventFrom) { throw new Exception("Invalid event ID input received in TO clause"); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java index ed408b0c1c..60ada73ae9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java @@ -65,7 +65,6 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { // Replication Scope private ReplScope replScope = new ReplScope(); - private ReplScope oldReplScope = null; // Source DB Name for REPL LOAD private String sourceDbNameOrPattern; @@ -143,34 +142,6 @@ private void setReplDumpTablesList(Tree replTablesNode, ReplScope replScope) thr } } - private void setOldReplPolicy(Tree oldReplPolicyTree) throws HiveException { - oldReplScope = new ReplScope(); - int childCount = oldReplPolicyTree.getChildCount(); - - // First child is DB name and optional second child is tables list. - assert(childCount <= 2); - - // First child is always the DB name. So set it. - oldReplScope.setDbName(oldReplPolicyTree.getChild(0).getText()); - LOG.info("Old ReplScope: Set DB Name: {}", oldReplScope.getDbName()); - if (!oldReplScope.getDbName().equalsIgnoreCase(replScope.getDbName())) { - LOG.error("DB name {} cannot be replaced to {} in the replication policy.", - oldReplScope.getDbName(), replScope.getDbName()); - throw new SemanticException("DB name cannot be replaced in the replication policy."); - } - - // If the old policy is just , then tables list won't be there. - if (childCount <= 1) { - return; - } - - // Traverse the children which can be either just include tables list or both include - // and exclude tables lists. - Tree oldPolicyTablesListNode = oldReplPolicyTree.getChild(1); - assert(oldPolicyTablesListNode.getType() == TOK_REPL_TABLES); - setReplDumpTablesList(oldPolicyTablesListNode, oldReplScope); - } - private void initReplDump(ASTNode ast) throws HiveException { int numChildren = ast.getChildCount(); boolean isMetaDataOnly = false; @@ -196,9 +167,6 @@ private void initReplDump(ASTNode ast) throws HiveException { case TOK_REPL_TABLES: setReplDumpTablesList(currNode, replScope); break; - case TOK_REPLACE: - setOldReplPolicy(currNode); - break; default: throw new SemanticException("Unrecognized token " + currNode.getType() + " in REPL DUMP statement."); } @@ -237,7 +205,6 @@ private void analyzeReplDump(ASTNode ast) throws SemanticException { Task replDumpWorkTask = TaskFactory .get(new ReplDumpWork( replScope, - oldReplScope, ASTErrorUtils.getMsg(ErrorMsg.INVALID_PATH.getMsg(), ast), ctx.getResFile().toUri().toString() ), conf); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestReplDumpTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestReplDumpTask.java index 3f47678988..aede902d4a 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestReplDumpTask.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestReplDumpTask.java @@ -147,8 +147,7 @@ void dumpTable(String dbName, String tblName, String validTxnList, }; task.initialize(queryState, null, null, null); - ReplDumpWork replDumpWork = new ReplDumpWork(replScope, - null, "", ""); + ReplDumpWork replDumpWork = new ReplDumpWork(replScope, "", ""); replDumpWork.setMetricCollector(metricCollector); task.setWork(replDumpWork); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java index 8b26f1ce40..a9d65aec05 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/repl/ReplScope.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.common.repl; import java.io.Serializable; +import java.util.Objects; import java.util.regex.Pattern; /** @@ -116,4 +117,27 @@ private boolean inTableExcludedList(final String tableName) { } return excludedTableNamePattern.matcher(tableName).matches(); } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (! (o instanceof ReplScope)) { + return false; + } + + ReplScope replScope = (ReplScope)o; + return ((replScope.getExcludedTableNames() == null ? (this.excludedTableNames == null) : + replScope.getExcludedTableNames().equals(this.excludedTableNames)) + && (replScope.getIncludedTableNames() == null ? (this.includedTableNames == null) : + replScope.getIncludedTableNames().equals(this.includedTableNames)) + && replScope.dbName == null ? (this.dbName == null) : replScope.dbName.equals(this.dbName) + && replScope.dbNamePattern == null ? (this.dbNamePattern == null) : replScope.dbNamePattern.equals(this.dbNamePattern)); + } + + @Override + public int hashCode() { + return Objects.hash(dbName, dbNamePattern, includedTableNames, excludedTableNames); + } }