diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java index 546678beb5..ea061e8c53 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java @@ -114,9 +114,9 @@ private void createTables(String[] tableNames, CreateTableType type) throws Thro } private String replicateAndVerify(String replPolicy, String lastReplId, - List dumpWithClause, - List loadWithClause, - String[] expectedTables) throws Throwable { + List dumpWithClause, + List loadWithClause, + String[] expectedTables) throws Throwable { return replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, loadWithClause, null, expectedTables); } @@ -159,7 +159,7 @@ private String replicateAndVerify(String replPolicy, String oldReplPolicy, Strin .verifyResults(expectedTables); if (records == null) { - records = new String[] {"1"}; + records = new String[]{"1"}; } for (String table : expectedTables) { replica.run("use " + replicatedDbName) @@ -196,27 +196,27 @@ private void verifyBootstrapDirInIncrementalDump(String dumpLocation, String[] b @Test public void testBasicBootstrapWithIncludeList() throws Throwable { - String[] originalNonAcidTables = new String[] {"t1", "t2" }; - String[] originalFullAcidTables = new String[] {"t3", "t4" }; - String[] originalMMAcidTables = new String[] {"t5" }; + String[] originalNonAcidTables = new String[]{"t1", "t2"}; + String[] originalFullAcidTables = new String[]{"t3", "t4"}; + String[] originalMMAcidTables = new String[]{"t5"}; createTables(originalNonAcidTables, CreateTableType.NON_ACID); createTables(originalFullAcidTables, CreateTableType.FULL_ACID); createTables(originalMMAcidTables, CreateTableType.MM_ACID); // Replicate and verify if only 2 tables are replicated to target. String replPolicy = primaryDbName + ".['t1', 't4', 't5']"; - String[] replicatedTables = new String[] {"t1", "t4", "t5" }; + String[] replicatedTables = new String[]{"t1", "t4", "t5"}; replicateAndVerify(replPolicy, null, null, null, replicatedTables); } @Test public void testBasicBootstrapWithIncludeAndExcludeList() throws Throwable { - String[] originalTables = new String[] {"t1", "t11", "t2", "t3", "t100" }; + String[] originalTables = new String[]{"t1", "t11", "t2", "t3", "t100"}; createTables(originalTables, CreateTableType.NON_ACID); // Replicate and verify if only 3 tables are replicated to target. String replPolicy = primaryDbName + ".['t1*', 't3'].['t100']"; - String[] replicatedTables = new String[] {"t1", "t11", "t3" }; + String[] replicatedTables = new String[]{"t1", "t11", "t3"}; replicateAndVerify(replPolicy, null, null, null, replicatedTables); } @@ -226,16 +226,16 @@ public void testBasicIncrementalWithIncludeList() throws Throwable { .dump(primaryDbName, null); replica.load(replicatedDbName, tupleBootstrap.dumpLocation); - String[] originalNonAcidTables = new String[] {"t1", "t2" }; - String[] originalFullAcidTables = new String[] {"t3", "t4" }; - String[] originalMMAcidTables = new String[] {"t5" }; + String[] originalNonAcidTables = new String[]{"t1", "t2"}; + String[] originalFullAcidTables = new String[]{"t3", "t4"}; + String[] originalMMAcidTables = new String[]{"t5"}; createTables(originalNonAcidTables, CreateTableType.NON_ACID); createTables(originalFullAcidTables, CreateTableType.FULL_ACID); createTables(originalMMAcidTables, CreateTableType.MM_ACID); // Replicate and verify if only 2 tables are replicated to target. String replPolicy = primaryDbName + ".['t1', 't5']"; - String[] replicatedTables = new String[] {"t1", "t5" }; + String[] replicatedTables = new String[]{"t1", "t5"}; replicateAndVerify(replPolicy, tupleBootstrap.lastReplicationId, null, null, replicatedTables); } @@ -245,30 +245,30 @@ public void testBasicIncrementalWithIncludeAndExcludeList() throws Throwable { .dump(primaryDbName, null); replica.load(replicatedDbName, tupleBootstrap.dumpLocation); - String[] originalTables = new String[] {"t1", "t11", "t2", "t3", "t111" }; + String[] originalTables = new String[]{"t1", "t11", "t2", "t3", "t111"}; createTables(originalTables, CreateTableType.NON_ACID); // Replicate and verify if only 3 tables are replicated to target. String replPolicy = primaryDbName + ".['t1+', 't2'].['t11', 't3']"; - String[] replicatedTables = new String[] {"t1", "t111", "t2" }; + String[] replicatedTables = new String[]{"t1", "t111", "t2"}; replicateAndVerify(replPolicy, tupleBootstrap.lastReplicationId, null, null, replicatedTables); } @Test public void testIncorrectTablePolicyInReplDump() throws Throwable { - String[] originalTables = new String[] {"t1", "t11", "t2", "t3", "t111" }; + String[] originalTables = new String[]{"t1", "t11", "t2", "t3", "t111"}; createTables(originalTables, CreateTableType.NON_ACID); // Invalid repl policy where abrubtly placed DOT which causes ParseException during REPL dump. String[] replicatedTables = new String[]{}; boolean failed; - String[] invalidReplPolicies = new String[] { - primaryDbName + ".t1.t2", // Two explicit table names not allowed. - primaryDbName + ".['t1'].t2", // Table name and include list not allowed. - primaryDbName + ".t1.['t2']", // Table name and exclude list not allowed. - primaryDbName + ".[t1].t2", // Table name and include list not allowed. - primaryDbName + ".['t1+'].", // Abrubtly ended dot. - primaryDbName + "..[]" // Multiple dots + String[] invalidReplPolicies = new String[]{ + primaryDbName + ".t1.t2", // Two explicit table names not allowed. + primaryDbName + ".['t1'].t2", // Table name and include list not allowed. + primaryDbName + ".t1.['t2']", // Table name and exclude list not allowed. + primaryDbName + ".[t1].t2", // Table name and include list not allowed. + primaryDbName + ".['t1+'].", // Abrubtly ended dot. + primaryDbName + "..[]" // Multiple dots }; for (String replPolicy : invalidReplPolicies) { failed = false; @@ -329,13 +329,13 @@ public void testIncorrectTablePolicyInReplDump() throws Throwable { @Test public void testFullDbBootstrapReplicationWithDifferentReplPolicyFormats() throws Throwable { - String[] originalTables = new String[] {"t1", "t200", "t3333" }; + String[] originalTables = new String[]{"t1", "t200", "t3333"}; createTables(originalTables, CreateTableType.NON_ACID); // List of repl policy formats that leads to Full DB replication. - String[] fullDbReplPolicies = new String[] { - primaryDbName + ".['.*?']", - primaryDbName + ".['.*?'].[]" + String[] fullDbReplPolicies = new String[]{ + primaryDbName + ".['.*?']", + primaryDbName + ".['.*?'].[]" }; // Replicate and verify if all 3 tables are replicated to target. @@ -346,27 +346,27 @@ public void testFullDbBootstrapReplicationWithDifferentReplPolicyFormats() throw @Test public void testCaseInSensitiveNatureOfReplPolicy() throws Throwable { - String[] originalTables = new String[] {"a1", "aA11", "B2", "Cc3" }; + String[] originalTables = new String[]{"a1", "aA11", "B2", "Cc3"}; createTables(originalTables, CreateTableType.NON_ACID); // Replicate and verify if 2 tables are replicated as per policy. String replPolicy = primaryDbName.toUpperCase() + ".['.*a1+', 'cc3', 'B2'].['AA1+', 'b2']"; - String[] replicatedTables = new String[] {"a1", "cc3" }; + String[] replicatedTables = new String[]{"a1", "cc3"}; String lastReplId = replicateAndVerify(replPolicy, null, null, null, replicatedTables); // Test case insensitive nature in REPLACE clause as well. String oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".['.*a1+', 'cc3', 'B2'].['AA1+']"; - replicatedTables = new String[] {"a1", "b2", "cc3" }; - String[] bootstrappedTables = new String[] {"b2" }; + replicatedTables = new String[]{"a1", "b2", "cc3"}; + String[] bootstrappedTables = new String[]{"b2"}; replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); } @Test public void testBootstrapAcidTablesIncrementalPhaseWithIncludeAndExcludeList() throws Throwable { - String[] originalNonAcidTables = new String[] {"a1", "b2" }; - String[] originalFullAcidTables = new String[] {"a2", "b1" }; - String[] originalMMAcidTables = new String[] {"a3", "a4" }; + String[] originalNonAcidTables = new String[]{"a1", "b2"}; + String[] originalFullAcidTables = new String[]{"a2", "b1"}; + String[] originalMMAcidTables = new String[]{"a3", "a4"}; createTables(originalNonAcidTables, CreateTableType.NON_ACID); createTables(originalFullAcidTables, CreateTableType.FULL_ACID); createTables(originalMMAcidTables, CreateTableType.MM_ACID); @@ -375,21 +375,21 @@ public void testBootstrapAcidTablesIncrementalPhaseWithIncludeAndExcludeList() t List dumpWithoutAcidClause = Collections.singletonList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'"); String replPolicy = primaryDbName + ".['a[0-9]+', 'b1'].['a4']"; - String[] bootstrapReplicatedTables = new String[] {"a1" }; + String[] bootstrapReplicatedTables = new String[]{"a1"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithoutAcidClause, null, bootstrapReplicatedTables); // Enable acid tables for replication. List dumpWithAcidBootstrapClause = Arrays.asList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES + "'='true'"); - String[] incrementalReplicatedTables = new String[] {"a1", "a2", "a3", "b1" }; + String[] incrementalReplicatedTables = new String[]{"a1", "a2", "a3", "b1"}; replicateAndVerify(replPolicy, lastReplId, dumpWithAcidBootstrapClause, null, incrementalReplicatedTables); } @Test public void testBootstrapExternalTablesWithIncludeAndExcludeList() throws Throwable { - String[] originalNonAcidTables = new String[] {"a1", "b2" }; - String[] originalExternalTables = new String[] {"a2", "b1" }; + String[] originalNonAcidTables = new String[]{"a1", "b2"}; + String[] originalExternalTables = new String[]{"a2", "b1"}; createTables(originalNonAcidTables, CreateTableType.NON_ACID); createTables(originalExternalTables, CreateTableType.EXTERNAL); @@ -399,7 +399,7 @@ public void testBootstrapExternalTablesWithIncludeAndExcludeList() throws Throwa "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'" ); String replPolicy = primaryDbName + ".['a[0-9]+', 'b2'].['a1']"; - String[] replicatedTables = new String[] {"a2", "b2" }; + String[] replicatedTables = new String[]{"a2", "b2"}; WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName) .dump(replPolicy, null, dumpWithClause); @@ -419,8 +419,8 @@ public void testBootstrapExternalTablesWithIncludeAndExcludeList() throws Throwa @Test public void testBootstrapExternalTablesIncrementalPhaseWithIncludeAndExcludeList() throws Throwable { - String[] originalNonAcidTables = new String[] {"a1", "b2" }; - String[] originalExternalTables = new String[] {"a2", "b1" }; + String[] originalNonAcidTables = new String[]{"a1", "b2"}; + String[] originalExternalTables = new String[]{"a2", "b1"}; createTables(originalNonAcidTables, CreateTableType.NON_ACID); createTables(originalExternalTables, CreateTableType.EXTERNAL); @@ -430,12 +430,12 @@ public void testBootstrapExternalTablesIncrementalPhaseWithIncludeAndExcludeList "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'" ); String replPolicy = primaryDbName + ".['a[0-9]+', 'b2'].['a1']"; - String[] bootstrapReplicatedTables = new String[] {"b2" }; + String[] bootstrapReplicatedTables = new String[]{"b2"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithClause, loadWithClause, bootstrapReplicatedTables); // Enable external tables replication and bootstrap in incremental phase. - String[] incrementalReplicatedTables = new String[] {"a2", "b2" }; + String[] incrementalReplicatedTables = new String[]{"a2", "b2"}; dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'"); WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName) @@ -457,9 +457,9 @@ public void testBootstrapExternalTablesIncrementalPhaseWithIncludeAndExcludeList @Test public void testBasicReplaceReplPolicy() throws Throwable { - String[] originalNonAcidTables = new String[] {"t1", "t2" }; - String[] originalFullAcidTables = new String[] {"t3", "t4" }; - String[] originalMMAcidTables = new String[] {"t5" }; + String[] originalNonAcidTables = new String[]{"t1", "t2"}; + String[] originalFullAcidTables = new String[]{"t3", "t4"}; + String[] originalMMAcidTables = new String[]{"t5"}; createTables(originalNonAcidTables, CreateTableType.NON_ACID); createTables(originalFullAcidTables, CreateTableType.FULL_ACID); createTables(originalMMAcidTables, CreateTableType.MM_ACID); @@ -467,30 +467,30 @@ public void testBasicReplaceReplPolicy() throws Throwable { // Replicate and verify if only 2 tables are replicated to target. String replPolicy = primaryDbName + ".['t1', 't4']"; String oldReplPolicy = null; - String[] replicatedTables = new String[] {"t1", "t4" }; + String[] replicatedTables = new String[]{"t1", "t4"}; String lastReplId = replicateAndVerify(replPolicy, null, null, null, replicatedTables); // Exclude t4 and include t3, t6 - createTables(new String[] {"t6" }, CreateTableType.MM_ACID); + createTables(new String[]{"t6"}, CreateTableType.MM_ACID); oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".['t1', 't3', 't6']"; - replicatedTables = new String[] {"t1", "t3", "t6" }; - String[] bootstrappedTables = new String[] {"t3", "t6" }; + replicatedTables = new String[]{"t1", "t3", "t6"}; + String[] bootstrappedTables = new String[]{"t3", "t6"}; lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); // Convert to Full Db repl policy. All tables should be included. oldReplPolicy = replPolicy; replPolicy = primaryDbName; - replicatedTables = new String[] {"t1", "t2", "t3", "t4", "t5", "t6" }; - bootstrappedTables = new String[] {"t2", "t4", "t5" }; + replicatedTables = new String[]{"t1", "t2", "t3", "t4", "t5", "t6"}; + bootstrappedTables = new String[]{"t2", "t4", "t5"}; replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); // Convert to regex that excludes t3, t4 and t5. oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".['.*?'].['t[3-5]+']"; - replicatedTables = new String[] {"t1", "t2", "t6" }; + replicatedTables = new String[]{"t1", "t2", "t6"}; bootstrappedTables = new String[]{}; replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, null, null, bootstrappedTables, replicatedTables); @@ -498,9 +498,9 @@ public void testBasicReplaceReplPolicy() throws Throwable { @Test public void testReplacePolicyOnBootstrapAcidTablesIncrementalPhase() throws Throwable { - String[] originalNonAcidTables = new String[] {"a1", "b1", "c1" }; - String[] originalFullAcidTables = new String[] {"a2", "b2" }; - String[] originalMMAcidTables = new String[] {"a3", "a4" }; + String[] originalNonAcidTables = new String[]{"a1", "b1", "c1"}; + String[] originalFullAcidTables = new String[]{"a2", "b2"}; + String[] originalMMAcidTables = new String[]{"a3", "a4"}; createTables(originalNonAcidTables, CreateTableType.NON_ACID); createTables(originalFullAcidTables, CreateTableType.FULL_ACID); createTables(originalMMAcidTables, CreateTableType.MM_ACID); @@ -509,7 +509,7 @@ public void testReplacePolicyOnBootstrapAcidTablesIncrementalPhase() throws Thro List dumpWithoutAcidClause = Collections.singletonList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'"); String replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['b1']"; - String[] bootstrapReplicatedTables = new String[] {"a1" }; + String[] bootstrapReplicatedTables = new String[]{"a1"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithoutAcidClause, null, bootstrapReplicatedTables); @@ -520,16 +520,16 @@ public void testReplacePolicyOnBootstrapAcidTablesIncrementalPhase() throws Thro List dumpWithAcidBootstrapClause = Arrays.asList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES + "'='true'"); - String[] incrementalReplicatedTables = new String[] {"a1", "a2", "a4", "b2", "c1" }; - String[] bootstrappedTables = new String[] {"a2", "a4", "b2", "c1" }; + String[] incrementalReplicatedTables = new String[]{"a1", "a2", "a4", "b2", "c1"}; + String[] bootstrappedTables = new String[]{"a2", "a4", "b2", "c1"}; replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, dumpWithAcidBootstrapClause, null, bootstrappedTables, incrementalReplicatedTables); } @Test public void testReplacePolicyWhenAcidTablesDisabledForRepl() throws Throwable { - String[] originalNonAcidTables = new String[] {"a1", "b1", "c1" }; - String[] originalFullAcidTables = new String[] {"a2" }; + String[] originalNonAcidTables = new String[]{"a1", "b1", "c1"}; + String[] originalFullAcidTables = new String[]{"a2"}; createTables(originalNonAcidTables, CreateTableType.NON_ACID); createTables(originalFullAcidTables, CreateTableType.FULL_ACID); @@ -537,7 +537,7 @@ public void testReplacePolicyWhenAcidTablesDisabledForRepl() throws Throwable { List dumpWithoutAcidClause = Collections.singletonList( "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'"); String replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['b1']"; - String[] bootstrapReplicatedTables = new String[] {"a1" }; + String[] bootstrapReplicatedTables = new String[]{"a1"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithoutAcidClause, null, bootstrapReplicatedTables); @@ -546,16 +546,16 @@ public void testReplacePolicyWhenAcidTablesDisabledForRepl() throws Throwable { // table "b1" should be bootstrapped. String oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['a2']"; - String[] incrementalReplicatedTables = new String[] {"a1", "b1" }; - String[] bootstrappedTables = new String[] {"b1" }; + String[] incrementalReplicatedTables = new String[]{"a1", "b1"}; + String[] bootstrappedTables = new String[]{"b1"}; lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, dumpWithoutAcidClause, null, bootstrappedTables, incrementalReplicatedTables); } @Test public void testReplacePolicyOnBootstrapExternalTablesIncrementalPhase() throws Throwable { - String[] originalAcidTables = new String[] {"a1", "b1" }; - String[] originalExternalTables = new String[] {"a2", "b2", "c2" }; + String[] originalAcidTables = new String[]{"a1", "b1"}; + String[] originalExternalTables = new String[]{"a2", "b2", "c2"}; createTables(originalAcidTables, CreateTableType.FULL_ACID); createTables(originalExternalTables, CreateTableType.EXTERNAL); @@ -565,7 +565,7 @@ public void testReplacePolicyOnBootstrapExternalTablesIncrementalPhase() throws "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'" ); String replPolicy = primaryDbName + ".['a[0-9]+', 'b1'].['a1']"; - String[] bootstrapReplicatedTables = new String[] {"b1" }; + String[] bootstrapReplicatedTables = new String[]{"b1"}; String lastReplId = replicateAndVerify(replPolicy, null, dumpWithClause, loadWithClause, bootstrapReplicatedTables); @@ -573,8 +573,8 @@ public void testReplacePolicyOnBootstrapExternalTablesIncrementalPhase() throws // "b1" and include "a1". String oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".['a[0-9]+', 'b[0-9]+'].['a2', 'b1']"; - String[] incrementalReplicatedTables = new String[] {"a1" }; - String[] bootstrappedTables = new String[] {"a1" }; + String[] incrementalReplicatedTables = new String[]{"a1"}; + String[] bootstrappedTables = new String[]{"a1"}; lastReplId = replicateAndVerify(replPolicy, oldReplPolicy, lastReplId, dumpWithClause, loadWithClause, bootstrappedTables, incrementalReplicatedTables); @@ -582,8 +582,8 @@ public void testReplacePolicyOnBootstrapExternalTablesIncrementalPhase() throws // replication policy to exclude tables with prefix "b". oldReplPolicy = replPolicy; replPolicy = primaryDbName + ".['[a-z]+[0-9]+'].['b[0-9]+']"; - incrementalReplicatedTables = new String[] {"a1", "a2", "c2" }; - bootstrappedTables = new String[] {"a2", "c2" }; + incrementalReplicatedTables = new String[]{"a1", "a2", "c2"}; + bootstrappedTables = new String[]{"a2", "c2"}; dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'"); WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName) @@ -689,7 +689,7 @@ public void testRenameTableScenariosWithDmlOperations() throws Throwable { replicatedTables = new String[]{"in4", "in400"}; bootstrapTables = new String[]{"in4", "in400"}; replicateAndVerify(replPolicy, null, lastReplId, null, - null, bootstrapTables, replicatedTables, new String[] {"1", "2"}); + null, bootstrapTables, replicatedTables, new String[]{"1", "2"}); } @Test @@ -735,7 +735,7 @@ public void testRenameTableScenariosExternalTable() throws Throwable { List loadWithClause = ReplicationTestUtils.externalTableBasePathWithClause(REPLICA_EXTERNAL_BASE, replica); List dumpWithClause = Arrays.asList( "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'", - "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='false'", + "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='false'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='false'", "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'" ); @@ -762,7 +762,7 @@ public void testRenameTableScenariosExternalTable() throws Throwable { dumpWithClause = Arrays.asList( "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", - "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'", + "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='true'", "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'" ); @@ -786,4 +786,273 @@ public void testRenameTableScenariosExternalTable() throws Throwable { replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, loadWithClause, bootstrapTables, replicatedTables); } + + @Test + public void testRenameTableScenariosWithReplacePolicy() throws Throwable { + String replPolicy = primaryDbName + ".['in[0-9]+'].['in100', 'in200', 'in305']"; + String lastReplId = replicateAndVerify(replPolicy, null, null, null, + null, new String[]{}, new String[]{}); + + String[] originalFullAcidTables = new String[]{"in1", "in2", "out3", "out4", "out5", "in100", "in200", "in300"}; + String[] originalNonAcidTables = new String[]{"in400", "out500"}; + createTables(originalFullAcidTables, CreateTableType.FULL_ACID); + createTables(originalNonAcidTables, CreateTableType.NON_ACID); + + // Replicate and verify if only 4 tables are replicated to target. + String[] replicatedTables = new String[]{"in1", "in2", "in300", "in400"}; + String[] bootstrapTables = new String[]{}; + lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null, + null, bootstrapTables, replicatedTables); + + // Rename the tables to satisfy the condition also replace the policy. + String newPolicy = primaryDbName + ".['in[0-9]+'].['in2']"; // in100, in200 and in305 are removed from exclude list. + primary.run("use " + primaryDbName) + .run("alter table in400 rename to out400") + .run("alter table out500 rename to in500") + .run("alter table in1 rename to out7") + .run("alter table in300 rename to in301") // for rename its all matching to matching. + .run("alter table in301 rename to in305") // ideally in305 bootstrap should not happen. + .run("alter table out3 rename to in8") + .run("alter table out4 rename to in9") + .run("drop table in9") + .run("alter table out5 rename to in10") + .run("alter table in10 rename to out11") + .run("drop table out11"); + + // in2 should be dropped. + replicatedTables = new String[]{"in100", "in200", "in8", "in305", "in500"}; + bootstrapTables = new String[]{"in500", "in8", "in100", "in200", "in305"}; + lastReplId = replicateAndVerify(newPolicy, replPolicy, lastReplId, null, + null, bootstrapTables, replicatedTables); + + // No table filter + replPolicy = newPolicy; + newPolicy = primaryDbName; + primary.run("use " + primaryDbName) + .run("alter table in100 rename to in12") // just rename no bootstrap + .run("alter table in200 rename to out12") // bootstrap by replace policy handler + .run("alter table out400 rename to in400") // just rename + .run("alter table in500 rename to out500") // bootstrap by replace policy handler + .run("alter table out7 rename to in1") // just rename + .run("alter table in305 rename to in301") // just rename + .run("alter table in301 rename to in300") // just rename + .run("alter table in8 rename to out3") // just rename + .run("drop table out3"); // table not present in get all table list + + replicatedTables = new String[]{"in12", "in400", "in1", "in300", "out500", "out12", "in2"}; + bootstrapTables = new String[]{"out12", "out500", "in2", "in400", "in1"}; + replicateAndVerify(newPolicy, replPolicy, lastReplId, null, + null, bootstrapTables, replicatedTables); + } + + @Test + public void testRenameTableScenariosWithReplaceExternalTable() throws Throwable { + List loadWithClause = ReplicationTestUtils.externalTableBasePathWithClause(REPLICA_EXTERNAL_BASE, replica); + List dumpWithClause = Arrays.asList( + "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='false'", + "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='false'", + "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='false'", + "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='false'" + ); + String replPolicy = primaryDbName + ".['in[0-9]+'].['in100', 'in200', 'in1400']"; + String lastReplId = replicateAndVerify(replPolicy, null, null, dumpWithClause, + loadWithClause, new String[]{}, new String[]{}); + + String[] originalFullAcidTables = new String[]{"in1", "in2", "out3", "out4", "out5", "in100", "in200", "in300"}; + String[] originalExternalTables = new String[]{"in400", "out500"}; + String[] originalNonAcidTables = new String[]{"in1400", "out1500", "out1600"}; + createTables(originalNonAcidTables, CreateTableType.NON_ACID); + createTables(originalFullAcidTables, CreateTableType.FULL_ACID); + createTables(originalExternalTables, CreateTableType.EXTERNAL); + + // Replicate and verify, no table should be there. + String[] replicatedTables = new String[]{}; + String[] bootstrapTables = new String[]{}; + lastReplId = replicateAndVerify(replPolicy, null, lastReplId, dumpWithClause, + loadWithClause, bootstrapTables, replicatedTables); + + // Rename the tables to satisfy the condition also replace the policy. + primary.run("use " + primaryDbName) + .run("alter table out1600 rename to in1600") + .run("alter table in1400 rename to out1400") + .run("alter table in1 rename to out7") + .run("alter table out3 rename to in8") + .run("alter table out4 rename to in9") + .run("drop table in9") + .run("alter table out5 rename to in10") + .run("alter table in10 rename to out11") + .run("alter table out500 rename to in500") + .run("alter table in400 rename to out400"); + + String newPolicy = primaryDbName + ".['in[0-9]+', 'out1500'].['in2']"; + dumpWithClause = Arrays.asList( + "'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", + "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'", + "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_ACID_TABLES.varname + "'='true'", + "'" + ReplUtils.REPL_DUMP_INCLUDE_ACID_TABLES + "'='true'" + ); + + // in2 should be dropped. + replicatedTables = new String[]{"in8", "in100", "in200", "in300", "in500", "in1600", "out1500"}; + bootstrapTables = new String[]{"in8", "in100", "in200", "in300", "in500", "in1600", "out1500"}; + replicateAndVerify(newPolicy, replPolicy, lastReplId, dumpWithClause, + loadWithClause, bootstrapTables, replicatedTables); + } + + @Test + public void testRenameTableScenariosWithReplacePolicyDMLOperattion() throws Throwable { + String replPolicy = primaryDbName + ".['in[0-9]+'].['in100', 'in200', 'in305']"; + String lastReplId = replicateAndVerify(replPolicy, null, null, null, + null, new String[]{}, new String[]{}); + + String[] originalFullAcidTables = new String[]{"in1", "in2", "out3", "out4", "out5", + "in100", "in200", "in300", "out3000"}; + String[] originalNonAcidTables = new String[]{"in400", "out500"}; + createTables(originalFullAcidTables, CreateTableType.FULL_ACID); + createTables(originalNonAcidTables, CreateTableType.NON_ACID); + + // Replicate and verify if only 4 tables are replicated to target. + String[] replicatedTables = new String[]{"in1", "in2", "in300", "in400"}; + String[] bootstrapTables = new String[]{}; + lastReplId = replicateAndVerify(replPolicy, null, lastReplId, null, + null, bootstrapTables, replicatedTables); + + // Rename the tables to satisfy the condition also replace the policy. + String newPolicy = primaryDbName + ".['in[0-9]+', 'out3000'].['in2']"; + primary.run("use " + primaryDbName) + .run("alter table in200 rename to in2000") // Old name not matching old, old and new matching new policy. + .run("alter table in400 rename to out400") // Old name matching new and old policy, new matching none. + .run("alter table out500 rename to in500") + .run("alter table out3000 rename to in3000") // Old name not matching old policy and both name matching new + .run("alter table in1 rename to out7") + .run("alter table in300 rename to in301") // for rename its all matching to matching. + .run("alter table in301 rename to in305") // ideally in305 bootstrap should not happen. + .run("alter table out3 rename to in8") + .run("alter table out4 rename to in9") + .run("drop table in9") + .run("alter table out5 rename to in10") + .run("alter table in10 rename to out11") + .run("drop table out11") + .run("insert into in100 values(2, 100)") + .run("insert into in8 values(2, 100)") + .run("insert into in305 values(2, 100)") + .run("insert into in3000 values (2, 100)") + .run("insert into in2000 values (2, 100)") + .run("insert into in500 values(2, 100)"); + + // in2 should be dropped. + replicatedTables = new String[]{"in100", "in2000", "in8", "in305", "in500", "in3000"}; + bootstrapTables = new String[]{"in500", "in8", "in100", "in2000", "in305", "in3000"}; + replicateAndVerify(newPolicy, replPolicy, lastReplId, null, + null, bootstrapTables, replicatedTables, new String[] {"1", "2"}); + } + + @Test + public void testRenameTableScenariosWithReplacePolicyAll() throws Throwable { + String replPolicy = primaryDbName + ".['inNew[0-9]+'].['outNew[0-9]+']"; + String replPolicyOld = primaryDbName + ".['inOld[0-9]+'].['outOld[0-9]+']"; + String lastReplId = replicateAndVerify(replPolicyOld, null, null, null, + null, new String[]{}, new String[]{}); + + int count = 0; + List commands = new ArrayList<>(); + String[] names = new String[] {"inOld", "outOld", "inNew", "outNew"}; + List tables = new ArrayList<>(); + List replicatedTables = new ArrayList<>(); + + for (String oldTable : names) { + for (String newTable : names) { + String oldName = oldTable + (count++); + String newName = newTable + (count++); + commands.add("alter table " + oldName + " rename to " + newName); + tables.add(oldName.toLowerCase()); + if (newName.startsWith("inNew")) { + replicatedTables.add(newName.toLowerCase()); + } + } + } + + String[] dummy = new String[]{}; + String[] originalFullAcidTables = tables.toArray(dummy); + createTables(originalFullAcidTables, CreateTableType.FULL_ACID); + primary.run("use " + primaryDbName); + for (String command : commands) { + primary.run(command); + } + + replicateAndVerify(replPolicy, replPolicyOld, lastReplId, null, + null, replicatedTables.toArray(dummy), replicatedTables.toArray(dummy)); + } + + private void addDoubleRename(List commands, List tables, List replicatedTables, + boolean isDrpo) throws Throwable { + int count = 0; + int idx = 0; + String[] names = new String[] {"inOld", "outOld", "inNew", "outNew"}; + for (String oldTable : names) { + for (String newTable : names) { + String oldName = oldTable + (count++); + String newName = newTable + (count++); + commands.add("alter table " + oldName + " rename to " + newName); + tables.add(oldName.toLowerCase()); + String newNameTemp = "junk"; + switch (idx) { + case 0: + newNameTemp = "inNew" + (count++); + if (!isDrpo) { + replicatedTables.add(newNameTemp.toLowerCase()); + } + break; + case 1: + newNameTemp = "outNew" + (count++); + break; + case 2: + newNameTemp = "inOld" + (count++); + break; + case 3: + newNameTemp = "outOld" + (count++); + break; + } + commands.add("alter table " + newName + " rename to " + newNameTemp); + commands.add("insert into " + newNameTemp + " values(2, 100)"); + if (isDrpo) { + commands.add("drop table " + newNameTemp); + } + idx = idx++ % 4; + } + } + } + + private void testDoubleRename(boolean isDrop) throws Throwable { + String replPolicy = primaryDbName + ".['inNew[0-9]+'].['outNew[0-9]+']"; + String replPolicyOld = primaryDbName + ".['inOld[0-9]+'].['outOld[0-9]+']"; + String lastReplId = replicateAndVerify(replPolicyOld, null, null, null, + null, new String[]{}, new String[]{}); + + List commands = new ArrayList<>(); + List tables = new ArrayList<>(); + List replicatedTables = new ArrayList<>(); + addDoubleRename(commands, tables, replicatedTables, isDrop); + + String[] dummy = new String[]{}; + String[] originalFullAcidTables = tables.toArray(dummy); + createTables(originalFullAcidTables, CreateTableType.FULL_ACID); + primary.run("use " + primaryDbName); + for (String command : commands) { + primary.run(command); + } + replicateAndVerify(replPolicy, replPolicyOld, lastReplId, null, null, + replicatedTables.toArray(dummy), replicatedTables.toArray(dummy), new String[]{"1", "2"}); + } + + @Test + public void testRenameTableTwiceScenariosWithReplacePolicyAll() throws Throwable { + testDoubleRename(false); + } + + @Test + public void testRenameTableTwiceScenariosWithReplacePolicyAllAndDrop() throws Throwable { + testDoubleRename(true); + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java index b06279dff8..bc9f06dfa9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java @@ -234,7 +234,7 @@ public static boolean shouldReplicate(ReplicationSpec replicationSpec, Table tab } // Tables in the list of tables to be bootstrapped should be skipped. - return (bootstrapTableList == null || !bootstrapTableList.contains(tableHandle.getTableName())); + return (bootstrapTableList == null || !bootstrapTableList.contains(tableHandle.getTableName().toLowerCase())); } } return true; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java index 3dd0d3b5c2..c7f6b7d2b3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.repl.ReplScope; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage; @@ -92,16 +93,38 @@ private Scenario scenarioType(org.apache.hadoop.hive.metastore.api.Table before, } } - // return true, if event needs to be dumped, else return false. - private boolean handleForTableLevelReplication(Context withinContext) { - String oldName = before.getTableName(); - String newName = after.getTableName(); + private boolean isSetForBootstrapByReplaceHandler(Context withinContext, String tblName) { + return (withinContext.oldReplScope != null) + && !(ReplUtils.tableIncludedInReplScope(withinContext.oldReplScope, tblName)) + && (ReplUtils.tableIncludedInReplScope(withinContext.replScope, tblName)); + } + + // Return true, if event needs to be dumped, else return false. + private boolean handleForTableLevelReplication(Context withinContext, String tblName) { + // For alter, if the table does not satisfy the new policy then ignore the event. In case of replace + // policy, if the table does not satisfy the old policy, then ignore the event. As, if the table satisfy the new + // policy, then the table will be bootstrapped by replace handler anb if the table does not satisfy the new policy, + // then anyways the table should be ignored. + if (!ReplUtils.tableIncludedInReplScope(withinContext.replScope, tblName)) { + // In case of replace, it will be dropped during load. In normal case just ignore the alter event. + return false; + } else if (withinContext.oldReplScope != null + && !ReplUtils.tableIncludedInReplScope(withinContext.oldReplScope, tblName)) { + return false; + } else { + // Table satisfies both old (if its there) and current policy, dump the alter event. + return true; + } + } - if (ReplUtils.tableIncludedInReplScope(withinContext.replScope, oldName)) { - // If the table is renamed after being added to the list of tables to be bootstrapped, then remove it from the - // list of tables to be bootstrapped. - boolean oldTableIsPresent = withinContext.removeFromListOfTablesForBootstrap(before.getTableName()); + // Return true, if event needs to be dumped, else return false. + private boolean handleForTableLevelReplicationForRename(Context withinContext, String oldName, String newName) { + // If the table is renamed after being added to the list of tables to be bootstrapped, then remove it from the + // list of tables to be bootstrapped. + boolean oldTableIsPresent = withinContext.removeFromListOfTablesForBootstrap(oldName); + ReplScope oldPolicy = withinContext.oldReplScope == null ? withinContext.replScope : withinContext.oldReplScope; + if (ReplUtils.tableIncludedInReplScope(oldPolicy, oldName)) { // If old table satisfies the filter, but the new table does not, then the old table should be dropped. // This should be done, only if the old table is not in the list of tables to be bootstrapped which is a multi // rename case. In case of multi rename, only the first rename should do the drop. @@ -123,6 +146,19 @@ private boolean handleForTableLevelReplication(Context withinContext) { return false; } + // All the subsequent events on this table newName are going to be skipped, so the rename is also skipped. + if (isSetForBootstrapByReplaceHandler(withinContext, newName)) { + // If the old table satisfies the new policy and is not in the list df tables to be bootstrapped, then + // drop it. + if (ReplUtils.tableIncludedInReplScope(withinContext.replScope, oldName)) { + scenario = Scenario.DROP; + LOG.info("Table " + oldName + " will be dropped as the table " + newName + " will be bootstrapped."); + return true; + } + LOG.info("Table " + newName + " is set to be bootstrapped by replace policy handler."); + return false; + } + // If both old and new table satisfies the filter and old table is present at target, then dump the rename event. LOG.info("both old and new table satisfies the filter"); return true; @@ -147,16 +183,19 @@ public void handle(Context withinContext) throws Exception { Table qlMdTableBefore = new Table(before); Set bootstrapTableList; + ReplScope oldReplScope; if (Scenario.RENAME == scenario) { // Handling for table level replication is done in handleForTableLevelReplication method. bootstrapTableList = null; + oldReplScope = null; } else { bootstrapTableList = withinContext.getTablesForBootstrap(); + oldReplScope = withinContext.oldReplScope; } if (!Utils .shouldReplicate(withinContext.replicationSpec, qlMdTableBefore, - true, bootstrapTableList, withinContext.oldReplScope, withinContext.hiveConf)) { + true, bootstrapTableList, oldReplScope, withinContext.hiveConf)) { return; } @@ -168,10 +207,18 @@ public void handle(Context withinContext) throws Exception { } } - // If the tables are filtered based on name, then needs to handle the rename scenarios. - if (!withinContext.replScope.includeAllTables()) { - if (!handleForTableLevelReplication(withinContext)) { - LOG.info("Alter event for table " + before.getTableName() + " is skipped from dumping"); + // If the tables are filtered based on name or policy is replaced, then needs to handle differently. + if (!withinContext.replScope.includeAllTables() || withinContext.oldReplScope != null) { + String oldName = before.getTableName(); + String newName = after.getTableName(); + boolean needDump; + if (Scenario.RENAME == scenario) { + needDump = handleForTableLevelReplicationForRename(withinContext, oldName, newName); + } else { + needDump = handleForTableLevelReplication(withinContext, oldName); + } + if (!needDump) { + LOG.info("Alter event for table " + oldName + " is skipped from dumping"); return; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CommitTxnHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CommitTxnHandler.java index 2391b9b50d..7d7dc26a25 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CommitTxnHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CommitTxnHandler.java @@ -117,9 +117,11 @@ private void createDumpFileForTable(Context withinContext, org.apache.hadoop.hiv // corresponding to tables which are included in both old and new policies should be dumped. // If table is included in new policy but not in old policy, then it should be skipped. // Those tables would be bootstrapped along with the current incremental - // replication dump. + // replication dump. If the table is in the list of tables to be bootstrapped, then + // it should be skipped. return (ReplUtils.tableIncludedInReplScope(withinContext.replScope, writeEventInfo.getTable()) - && ReplUtils.tableIncludedInReplScope(withinContext.oldReplScope, writeEventInfo.getTable())); + && ReplUtils.tableIncludedInReplScope(withinContext.oldReplScope, writeEventInfo.getTable()) + && !withinContext.getTablesForBootstrap().contains(writeEventInfo.getTable().toLowerCase())); }))); }