diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java index 8593480724..cb6bd2d70a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java @@ -153,12 +153,12 @@ private void prepareNonAcidData(String primaryDbName) throws Throwable { nonAcidTableNames.add("t4"); } - WarehouseInstance.Tuple prepareDataAndDump(String primaryDbName, String fromReplId, + WarehouseInstance.Tuple prepareDataAndDump(String primaryDbName, List withClause) throws Throwable { prepareAcidData(primaryDbName); prepareNonAcidData(primaryDbName); return primary.run("use " + primaryDbName) - .dump(primaryDbName, fromReplId, withClause != null ? + .dump(primaryDbName, withClause != null ? withClause : Collections.emptyList()); } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplicationTestUtils.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplicationTestUtils.java index c552bae56d..86c07da3d0 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplicationTestUtils.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/ReplicationTestUtils.java @@ -20,7 +20,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.parse.WarehouseInstance; import org.apache.hadoop.hive.ql.parse.repl.PathBuilder; import org.junit.Assert; @@ -28,10 +27,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.util.Arrays; -import java.util.HashSet; import java.util.List; +import java.util.Collections; +import java.util.Arrays; import java.util.Set; +import java.util.HashSet; /** * ReplicationTestUtils - static helper functions for replication test @@ -265,7 +265,7 @@ public static void verifyResultsInReplica(WarehouseInstance replica ,String repl String primaryDbName, String replicatedDbName, List selectStmtList, List expectedValues, String lastReplId) throws Throwable { - WarehouseInstance.Tuple incrementalDump = primary.dump(primaryDbName, lastReplId); + WarehouseInstance.Tuple incrementalDump = primary.dump(primaryDbName, Collections.emptyList()); replica.loadWithoutExplain(replicatedDbName, incrementalDump.dumpLocation) .run("REPL STATUS " + replicatedDbName).verifyResult(incrementalDump.lastReplicationId); verifyResultsInReplica(replica, replicatedDbName, selectStmtList, expectedValues); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestMetaStoreEventListenerInRepl.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestMetaStoreEventListenerInRepl.java index 76832a4526..08e470522d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestMetaStoreEventListenerInRepl.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestMetaStoreEventListenerInRepl.java @@ -181,7 +181,7 @@ public void testReplEvents() throws Throwable { eventsMap = prepareIncData(primaryDbName); LOG.info(testName.getMethodName() + ": first incremental dump and load."); WarehouseInstance.Tuple incDump = primary.run("use " + primaryDbName) - .dump(primaryDbName, bootstrapDump.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, incDump.dumpLocation); ReplMetaStoreEventListenerTestImpl.checkEventSanity(eventsMap, replicatedDbName); ReplMetaStoreEventListenerTestImpl.clearSanityData(); @@ -190,7 +190,7 @@ public void testReplEvents() throws Throwable { eventsMap = prepareInc2Data(primaryDbName); LOG.info(testName.getMethodName() + ": second incremental dump and load."); WarehouseInstance.Tuple inc2Dump = primary.run("use " + primaryDbName) - .dump(primaryDbName, incDump.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, inc2Dump.dumpLocation); ReplMetaStoreEventListenerTestImpl.checkEventSanity(eventsMap, replicatedDbName); ReplMetaStoreEventListenerTestImpl.clearSanityData(); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOfHiveStreaming.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOfHiveStreaming.java index 72f6644040..a38b8681d4 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOfHiveStreaming.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOfHiveStreaming.java @@ -148,7 +148,7 @@ public void testHiveStreamingUnpartitionedWithTxnBatchSizeAsOne() throws Throwab connection.commitTransaction(); // Replicate the committed data which should be visible. - WarehouseInstance.Tuple incrDump = primary.dump(primaryDbName, bootstrapDump.lastReplicationId); + WarehouseInstance.Tuple incrDump = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, incrDump.dumpLocation) .run("use " + replicatedDbName) .run("select msg from " + tblName + " order by msg") @@ -160,7 +160,7 @@ public void testHiveStreamingUnpartitionedWithTxnBatchSizeAsOne() throws Throwab connection.write("4,val4".getBytes()); // Replicate events before committing txn. The uncommitted data shouldn't be seen. - incrDump = primary.dump(primaryDbName, bootstrapDump.lastReplicationId); + incrDump = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, incrDump.dumpLocation) .run("use " + replicatedDbName) .run("select msg from " + tblName + " order by msg") @@ -169,7 +169,7 @@ public void testHiveStreamingUnpartitionedWithTxnBatchSizeAsOne() throws Throwab connection.commitTransaction(); // After commit, the data should be replicated and visible. - incrDump = primary.dump(primaryDbName, incrDump.lastReplicationId); + incrDump = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, incrDump.dumpLocation) .run("use " + replicatedDbName) .run("select msg from " + tblName + " order by msg") @@ -182,7 +182,7 @@ public void testHiveStreamingUnpartitionedWithTxnBatchSizeAsOne() throws Throwab connection.abortTransaction(); // Aborted data shouldn't be visible. - incrDump = primary.dump(primaryDbName, incrDump.lastReplicationId); + incrDump = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, incrDump.dumpLocation) .run("use " + replicatedDbName) .run("select msg from " + tblName + " order by msg") @@ -233,7 +233,7 @@ public void testHiveStreamingStaticPartitionWithTxnBatchSizeAsOne() throws Throw connection.commitTransaction(); // Replicate the committed data which should be visible. - WarehouseInstance.Tuple incrDump = primary.dump(primaryDbName, bootstrapDump.lastReplicationId); + WarehouseInstance.Tuple incrDump = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, incrDump.dumpLocation) .run("use " + replicatedDbName) .run("select msg from " + tblName + " where continent='Asia' and country='India' order by msg") @@ -245,7 +245,7 @@ public void testHiveStreamingStaticPartitionWithTxnBatchSizeAsOne() throws Throw connection.write("4,val4".getBytes()); // Replicate events before committing txn. The uncommitted data shouldn't be seen. - incrDump = primary.dump(primaryDbName, bootstrapDump.lastReplicationId); + incrDump = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, incrDump.dumpLocation) .run("use " + replicatedDbName) .run("select msg from " + tblName + " where continent='Asia' and country='India' order by msg") @@ -254,7 +254,7 @@ public void testHiveStreamingStaticPartitionWithTxnBatchSizeAsOne() throws Throw connection.commitTransaction(); // After commit, the data should be replicated and visible. - incrDump = primary.dump(primaryDbName, incrDump.lastReplicationId); + incrDump = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, incrDump.dumpLocation) .run("use " + replicatedDbName) .run("select msg from " + tblName + " where continent='Asia' and country='India' order by msg") @@ -267,7 +267,7 @@ public void testHiveStreamingStaticPartitionWithTxnBatchSizeAsOne() throws Throw connection.abortTransaction(); // Aborted data shouldn't be visible. - incrDump = primary.dump(primaryDbName, incrDump.lastReplicationId); + incrDump = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, incrDump.dumpLocation) .run("use " + replicatedDbName) .run("select msg from " + tblName + " where continent='Asia' and country='India' order by msg") @@ -315,7 +315,7 @@ public void testHiveStreamingDynamicPartitionWithTxnBatchSizeAsOne() throws Thro connection.commitTransaction(); // Replicate the committed data which should be visible. - WarehouseInstance.Tuple incrDump = primary.dump(primaryDbName, bootstrapDump.lastReplicationId); + WarehouseInstance.Tuple incrDump = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, incrDump.dumpLocation) .run("use " + replicatedDbName) .run("select msg from " + tblName + " where continent='Asia' and country='China' order by msg") @@ -329,7 +329,7 @@ public void testHiveStreamingDynamicPartitionWithTxnBatchSizeAsOne() throws Thro connection.write("14,val14,Asia,India".getBytes()); // Replicate events before committing txn. The uncommitted data shouldn't be seen. - incrDump = primary.dump(primaryDbName, bootstrapDump.lastReplicationId); + incrDump = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, incrDump.dumpLocation) .run("use " + replicatedDbName) .run("select msg from " + tblName + " where continent='Asia' and country='India' order by msg") @@ -338,7 +338,7 @@ public void testHiveStreamingDynamicPartitionWithTxnBatchSizeAsOne() throws Thro connection.commitTransaction(); // After committing the txn, the data should be visible. - incrDump = primary.dump(primaryDbName, bootstrapDump.lastReplicationId); + incrDump = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, incrDump.dumpLocation) .run("use " + replicatedDbName) .run("select msg from " + tblName + " where continent='Asia' and country='India' order by msg") @@ -353,7 +353,7 @@ public void testHiveStreamingDynamicPartitionWithTxnBatchSizeAsOne() throws Thro connection.abortTransaction(); // Aborted data should not be visible. - incrDump = primary.dump(primaryDbName, bootstrapDump.lastReplicationId); + incrDump = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, incrDump.dumpLocation) .run("use " + replicatedDbName) .run("select msg from " + tblName + " where continent='Asia' and country='India' order by msg") diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index 39d876802a..9273a3b33b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -248,45 +248,18 @@ private synchronized void advanceDumpDir() { } private Tuple bootstrapLoadAndVerify(String dbName, String replDbName) throws IOException { - return incrementalLoadAndVerify(dbName, null, replDbName); + return incrementalLoadAndVerify(dbName, replDbName); } - private Tuple incrementalLoadAndVerify(String dbName, String fromReplId, String replDbName) throws IOException { - Tuple dump = replDumpDb(dbName, fromReplId, null, null); + private Tuple incrementalLoadAndVerify(String dbName, String replDbName) throws IOException { + Tuple dump = replDumpDb(dbName); loadAndVerify(replDbName, dump.dumpLocation, dump.lastReplId); return dump; } - private Tuple incrementalLoadAndVerify(String dbName, String fromReplId, String toReplId, String replDbName) - throws IOException { - Tuple dump = replDumpDb(dbName, fromReplId, toReplId, null); - loadAndVerify(replDbName, dump.dumpLocation, dump.lastReplId); - return dump; - } - - private Tuple incrementalLoadAndVerify(String dbName, String fromReplId, String toReplId, String limit, - String replDbName) throws IOException { - Tuple dump = replDumpDb(dbName, fromReplId, toReplId, limit); - loadAndVerify(replDbName, dump.dumpLocation, dump.lastReplId); - return dump; - } - - private Tuple dumpDbFromLastDump(String dbName, Tuple lastDump) throws IOException { - return replDumpDb(dbName, lastDump.lastReplId, null, null); - } - - private Tuple replDumpDb(String dbName, String fromReplID, String toReplID, String limit) throws IOException { + private Tuple replDumpDb(String dbName) throws IOException { advanceDumpDir(); String dumpCmd = "REPL DUMP " + dbName; - if (null != fromReplID) { - dumpCmd = dumpCmd + " FROM " + fromReplID; - } - if (null != toReplID) { - dumpCmd = dumpCmd + " TO " + toReplID; - } - if (null != limit) { - dumpCmd = dumpCmd + " LIMIT " + limit; - } run(dumpCmd, driver); String dumpLocation = getResult(0, 0, driver); String lastReplId = getResult(0, 1, true, driver); @@ -413,7 +386,7 @@ public void testTaskCreationOptimization() throws Throwable { run("create table " + dbName + ".t2 (place string) partitioned by (country string)", driver); run("insert into table " + dbName + ".t2 partition(country='india') values ('bangalore')", driver); - Tuple dump = replDumpDb(dbName, null, null, null); + Tuple dump = replDumpDb(dbName); //bootstrap load should not have move task Task task = getReplLoadRootTask(dbNameReplica, false, dump); @@ -423,7 +396,7 @@ public void testTaskCreationOptimization() throws Throwable { loadAndVerify(dbNameReplica, dump.dumpLocation, dump.lastReplId); run("insert into table " + dbName + ".t2 partition(country='india') values ('delhi')", driver); - dump = replDumpDb(dbName, dump.lastReplId, null, null); + dump = replDumpDb(dbName); // Partition level statistics gets updated as part of the INSERT above. So we see a partition // task corresponding to an ALTER_PARTITION event. @@ -434,7 +407,7 @@ public void testTaskCreationOptimization() throws Throwable { loadAndVerify(dbNameReplica, dump.dumpLocation, dump.lastReplId); run("insert into table " + dbName + ".t2 partition(country='us') values ('sf')", driver); - dump = replDumpDb(dbName, dump.lastReplId, null, null); + dump = replDumpDb(dbName); //no move task should be added as the operation is adding a dynamic partition task = getReplLoadRootTask(dbNameReplica, true, dump); @@ -810,7 +783,7 @@ public void run() { InjectableBehaviourObjectStore.resetGetTableBehaviour(); // reset the behaviour } - incrementalLoadAndVerify(dbName, bootstrap.lastReplId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyIfTableNotExist(replDbName, "ptned", metaStoreClientMirror); } @@ -863,7 +836,7 @@ public void testIncrementalAdds() throws IOException { verifySetup("SELECT a from " + dbName + ".ptned_late WHERE b=2", ptn_data_2, driver); // Perform REPL-DUMP/LOAD - incrementalLoadAndVerify(dbName, bootstrapDump.lastReplId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); // VERIFY tables and partitions on destination for equivalence. verifyRun("SELECT * from " + replDbName + ".unptned_empty", empty, driverMirror); @@ -897,7 +870,7 @@ public void testIncrementalLoadWithVariableLengthEventId() throws IOException, T run("TRUNCATE TABLE " + dbName + ".unptned", driver); run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')", driver); - Tuple incrementalDump = replDumpDb(dbName, replDumpId, null, null); + Tuple incrementalDump = replDumpDb(dbName); String incrementalDumpLocn = incrementalDump.dumpLocation; replDumpId = incrementalDump.lastReplId; @@ -1057,7 +1030,7 @@ public void testDrops() throws IOException { verifySetup("SELECT a from " + dbName + ".ptned3", ptn_data_2, driver); // replicate the incremental drops - incrementalLoadAndVerify(dbName, bootstrapDump.lastReplId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); // verify that drops were replicated. This can either be from tables or ptns // not existing, and thus, throwing a NoSuchObjectException, or returning nulls @@ -1261,7 +1234,7 @@ public void testTableAlters() throws IOException { verifySetup("SELECT a from " + dbName + ".ptned2_rn WHERE b=2", ptn_data_2, driver); // All alters done, now we replicate them over. - incrementalLoadAndVerify(dbName, bootstrapDump.lastReplId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); // Replication done, we now do the following verifications: @@ -1332,7 +1305,7 @@ public void testDatabaseAlters() throws IOException { run("ALTER DATABASE " + dbName + " SET DBPROPERTIES ('" + testKey + "' = '" + testVal + "')", driver); // All alters done, now we replicate them over. - Tuple incremental = incrementalLoadAndVerify(dbName, bootstrap.lastReplId, replDbName); + Tuple incremental = incrementalLoadAndVerify(dbName, replDbName); // Replication done, we need to check if the new property is added try { @@ -1348,7 +1321,7 @@ public void testDatabaseAlters() throws IOException { run("ALTER DATABASE " + dbName + " SET DBPROPERTIES ('" + testKey + "' = '" + newValue + "')", driver); run("ALTER DATABASE " + dbName + " SET OWNER ROLE " + newOwnerName, driver); - incremental = incrementalLoadAndVerify(dbName, incremental.lastReplId, replDbName); + incremental = incrementalLoadAndVerify(dbName, replDbName); // Replication done, we need to check if new value is set for existing property try { @@ -1375,7 +1348,6 @@ public void testIncrementalLoad() throws IOException { + ".ptned_empty(a string) partitioned by (b int) STORED AS TEXTFILE", driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String replDumpId = bootstrapDump.lastReplId; String[] unptn_data = new String[] { "eleven", "twelve" }; String[] ptn_data_1 = new String[] { "thirteen", "fourteen", "fifteen" }; @@ -1399,8 +1371,7 @@ public void testIncrementalLoad() throws IOException { run("INSERT INTO TABLE " + dbName + ".unptned_late SELECT * FROM " + dbName + ".unptned", driver); verifySetup("SELECT * from " + dbName + ".unptned_late", unptn_data, driver); - Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, replDbName); - replDumpId = incrementalDump.lastReplId; + Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT * from " + replDbName + ".unptned_late", unptn_data, driverMirror); @@ -1422,7 +1393,7 @@ public void testIncrementalLoad() throws IOException { + ".ptned WHERE b=2", driver); verifySetup("SELECT a from " + dbName + ".ptned_late WHERE b=2", ptn_data_2, driver); - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".ptned_late WHERE b=1", ptn_data_1, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned_late WHERE b=2", ptn_data_2, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned WHERE b=1", ptn_data_1, driverMirror); @@ -1437,7 +1408,6 @@ public void testIncrementalInserts() throws IOException { run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String replDumpId = bootstrapDump.lastReplId; String[] unptn_data = new String[] { "eleven", "twelve" }; @@ -1449,8 +1419,7 @@ public void testIncrementalInserts() throws IOException { run("INSERT INTO TABLE " + dbName + ".unptned_late SELECT * FROM " + dbName + ".unptned", driver); verifySetup("SELECT * from " + dbName + ".unptned_late ORDER BY a", unptn_data, driver); - Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, replDbName); - replDumpId = incrementalDump.lastReplId; + Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".unptned ORDER BY a", unptn_data, driverMirror); verifyRun("SELECT a from " + replDbName + ".unptned_late ORDER BY a", unptn_data, driverMirror); @@ -1462,7 +1431,7 @@ public void testIncrementalInserts() throws IOException { run("INSERT OVERWRITE TABLE " + dbName + ".unptned values('" + data_after_ovwrite[0] + "')", driver); verifySetup("SELECT a from " + dbName + ".unptned", data_after_ovwrite, driver); - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".unptned_late ORDER BY a", unptn_data_after_ins, driverMirror); verifyRun("SELECT a from " + replDbName + ".unptned", data_after_ovwrite, driverMirror); } @@ -1508,7 +1477,7 @@ public NotificationEventResponse apply(@Nullable NotificationEventResponse event }; InjectableBehaviourObjectStore.setGetNextNotificationBehaviour(eventTypeValidator); try { - incrementalLoadAndVerify(dbName, bootstrap.lastReplId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); eventTypeValidator.assertInjectionsPerformed(true,false); } finally { InjectableBehaviourObjectStore.resetGetNextNotificationBehaviour(); // reset the behaviour @@ -1568,7 +1537,7 @@ public NotificationEventResponse apply(@Nullable NotificationEventResponse event }; InjectableBehaviourObjectStore.setGetNextNotificationBehaviour(insertEventRepeater); try { - incrementalLoadAndVerify(dbName, bootstrap.lastReplId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); insertEventRepeater.assertInjectionsPerformed(true,false); } finally { InjectableBehaviourObjectStore.resetGetNextNotificationBehaviour(); // reset the behaviour @@ -1593,7 +1562,6 @@ public void testIncrementalInsertToPartition() throws IOException { run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String replDumpId = bootstrapDump.lastReplId; String[] ptn_data_1 = new String[] { "fifteen", "fourteen", "thirteen" }; String[] ptn_data_2 = new String[] { "fifteen", "seventeen", "sixteen" }; @@ -1609,8 +1577,7 @@ public void testIncrementalInsertToPartition() throws IOException { verifySetup("SELECT a from " + dbName + ".ptned where (b=1) ORDER BY a", ptn_data_1, driver); verifySetup("SELECT a from " + dbName + ".ptned where (b=2) ORDER BY a", ptn_data_2, driver); - Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, replDbName); - replDumpId = incrementalDump.lastReplId; + Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".ptned where (b=1) ORDER BY a", ptn_data_1, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned where (b=2) ORDER BY a", ptn_data_2, driverMirror); @@ -1623,7 +1590,7 @@ public void testIncrementalInsertToPartition() throws IOException { run("INSERT OVERWRITE TABLE " + dbName + ".ptned partition(b=3) values('" + data_after_ovwrite[0] + "')", driver); verifySetup("SELECT a from " + dbName + ".ptned where (b=3)", data_after_ovwrite, driver); - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".ptned where (b=2)", data_after_ovwrite, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned where (b=3)", data_after_ovwrite, driverMirror); @@ -1658,7 +1625,6 @@ public void testInsertToMultiKeyPartition() throws IOException { "location", "namelist/year=1980/month=4/day=1", driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String replDumpId = bootstrapDump.lastReplId; verifyRun("SELECT name from " + replDbName + ".namelist where (year=1980) ORDER BY name", ptn_year_1980, driverMirror); verifyRun("SELECT name from " + replDbName + ".namelist where (day=1) ORDER BY name", ptn_day_1, driverMirror); @@ -1690,8 +1656,7 @@ public void testInsertToMultiKeyPartition() throws IOException { verifyRunWithPatternMatch("SHOW TABLE EXTENDED LIKE namelist PARTITION (year=1990,month=5,day=25)", "location", "namelist/year=1990/month=5/day=25", driver); - Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, replDbName); - replDumpId = incrementalDump.lastReplId; + Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT name from " + replDbName + ".namelist where (year=1980) ORDER BY name", ptn_year_1980, driverMirror); verifyRun("SELECT name from " + replDbName + ".namelist where (day=1) ORDER BY name", ptn_day_1_2, driverMirror); @@ -1711,7 +1676,7 @@ public void testInsertToMultiKeyPartition() throws IOException { verifySetup("SELECT name from " + dbName + ".namelist where (year=1990 and month=5 and day=25)", data_after_ovwrite, driver); verifySetup("SELECT name from " + dbName + ".namelist ORDER BY name", ptn_data_3, driver); - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifySetup("SELECT name from " + replDbName + ".namelist where (year=1990 and month=5 and day=25)", data_after_ovwrite, driverMirror); verifySetup("SELECT name from " + replDbName + ".namelist ORDER BY name", ptn_data_3, driverMirror); } @@ -1724,7 +1689,6 @@ public void testIncrementalInsertDropUnpartitionedTable() throws IOException { run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String replDumpId = bootstrapDump.lastReplId; String[] unptn_data = new String[] { "eleven", "twelve" }; @@ -1736,8 +1700,7 @@ public void testIncrementalInsertDropUnpartitionedTable() throws IOException { verifySetup("SELECT a from " + dbName + ".unptned_tmp ORDER BY a", unptn_data, driver); // Get the last repl ID corresponding to all insert/alter/create events except DROP. - Tuple incrementalDump = replDumpDb(dbName, replDumpId, null, null); - String lastDumpIdWithoutDrop = incrementalDump.lastReplId; + Tuple incrementalDump = replDumpDb(dbName); // Drop all the tables run("DROP TABLE " + dbName + ".unptned", driver); @@ -1746,15 +1709,12 @@ public void testIncrementalInsertDropUnpartitionedTable() throws IOException { verifyFail("SELECT * FROM " + dbName + ".unptned_tmp", driver); // Dump all the events except DROP - incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, lastDumpIdWithoutDrop, replDbName); - replDumpId = incrementalDump.lastReplId; // Need to find the tables and data as drop is not part of this dump verifyRun("SELECT a from " + replDbName + ".unptned ORDER BY a", unptn_data, driverMirror); verifyRun("SELECT a from " + replDbName + ".unptned_tmp ORDER BY a", unptn_data, driverMirror); // Dump the drop events and check if tables are getting dropped in target as well - incrementalLoadAndVerify(dbName, replDumpId, replDbName); verifyFail("SELECT * FROM " + replDbName + ".unptned", driverMirror); verifyFail("SELECT * FROM " + replDbName + ".unptned_tmp", driverMirror); } @@ -1767,7 +1727,6 @@ public void testIncrementalInsertDropPartitionedTable() throws IOException { run("CREATE TABLE " + dbName + ".ptned(a string) PARTITIONED BY (b int) STORED AS TEXTFILE", driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String replDumpId = bootstrapDump.lastReplId; String[] ptn_data_1 = new String[] { "fifteen", "fourteen", "thirteen" }; String[] ptn_data_2 = new String[] { "fifteen", "seventeen", "sixteen" }; @@ -1789,8 +1748,7 @@ public void testIncrementalInsertDropPartitionedTable() throws IOException { verifySetup("SELECT a from " + dbName + ".ptned_tmp where (b=2) ORDER BY a", ptn_data_2, driver); // Get the last repl ID corresponding to all insert/alter/create events except DROP. - Tuple incrementalDump = replDumpDb(dbName, replDumpId, null, null); - String lastDumpIdWithoutDrop = incrementalDump.lastReplId; + Tuple incrementalDump = replDumpDb(dbName); // Drop all the tables run("DROP TABLE " + dbName + ".ptned_tmp", driver); @@ -1799,8 +1757,7 @@ public void testIncrementalInsertDropPartitionedTable() throws IOException { verifyFail("SELECT * FROM " + dbName + ".ptned", driver); // Replicate all the events except DROP - incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, lastDumpIdWithoutDrop, replDbName); - replDumpId = incrementalDump.lastReplId; + incrementalDump = incrementalLoadAndVerify(dbName, replDbName); // Need to find the tables and data as drop is not part of this dump verifyRun("SELECT a from " + replDbName + ".ptned where (b=1) ORDER BY a", ptn_data_1, driverMirror); @@ -1809,7 +1766,7 @@ public void testIncrementalInsertDropPartitionedTable() throws IOException { verifyRun("SELECT a from " + replDbName + ".ptned_tmp where (b=2) ORDER BY a", ptn_data_2, driverMirror); // Replicate the drop events and check if tables are getting dropped in target as well - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyFail("SELECT * FROM " + replDbName + ".ptned_tmp", driverMirror); verifyFail("SELECT * FROM " + replDbName + ".ptned", driverMirror); } @@ -1822,27 +1779,24 @@ public void testInsertOverwriteOnUnpartitionedTableWithCM() throws IOException { run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String replDumpId = bootstrapDump.lastReplId; // After INSERT INTO operation, get the last Repl ID String[] unptn_data = new String[] { "thirteen" }; run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')", driver); - Tuple incrementalDump = replDumpDb(dbName, replDumpId, null, null); - String insertDumpId = incrementalDump.lastReplId; + Tuple incrementalDump = replDumpDb(dbName); // Insert overwrite on unpartitioned table String[] data_after_ovwrite = new String[] { "hundred" }; run("INSERT OVERWRITE TABLE " + dbName + ".unptned values('" + data_after_ovwrite[0] + "')", driver); // Replicate only one INSERT INTO operation on the table. - incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, insertDumpId, replDbName); - replDumpId = incrementalDump.lastReplId; + incrementalDump = incrementalLoadAndVerify(dbName, replDbName); // After Load from this dump, all target tables/partitions will have initial set of data but source will have latest data. verifyRun("SELECT a from " + replDbName + ".unptned ORDER BY a", unptn_data, driverMirror); // Replicate the remaining INSERT OVERWRITE operations on the table. - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); // After load, shall see the overwritten data. verifyRun("SELECT a from " + replDbName + ".unptned ORDER BY a", data_after_ovwrite, driverMirror); @@ -1856,7 +1810,6 @@ public void testInsertOverwriteOnPartitionedTableWithCM() throws IOException { run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String replDumpId = bootstrapDump.lastReplId; // INSERT INTO 2 partitions and get the last repl ID String[] ptn_data_1 = new String[] { "fourteen" }; @@ -1865,8 +1818,7 @@ public void testInsertOverwriteOnPartitionedTableWithCM() throws IOException { run("INSERT INTO TABLE " + dbName + ".ptned partition(b=2) values('" + ptn_data_2[0] + "')", driver); run("INSERT INTO TABLE " + dbName + ".ptned partition(b=2) values('" + ptn_data_2[1] + "')", driver); - Tuple incrementalDump = replDumpDb(dbName, replDumpId, null, null); - String insertDumpId = incrementalDump.lastReplId; + Tuple incrementalDump = replDumpDb(dbName); // Insert overwrite on one partition with multiple files String[] data_after_ovwrite = new String[] { "hundred" }; @@ -1874,15 +1826,14 @@ public void testInsertOverwriteOnPartitionedTableWithCM() throws IOException { verifySetup("SELECT a from " + dbName + ".ptned where (b=2)", data_after_ovwrite, driver); // Replicate only 2 INSERT INTO operations. - incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, insertDumpId, replDbName); - replDumpId = incrementalDump.lastReplId; + incrementalDump = incrementalLoadAndVerify(dbName, replDbName); // After Load from this dump, all target tables/partitions will have initial set of data. verifyRun("SELECT a from " + replDbName + ".ptned where (b=1) ORDER BY a", ptn_data_1, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned where (b=2) ORDER BY a", ptn_data_2, driverMirror); // Replicate the remaining INSERT OVERWRITE operation on the table. - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); // After load, shall see the overwritten data. verifyRun("SELECT a from " + replDbName + ".ptned where (b=1) ORDER BY a", ptn_data_1, driverMirror); @@ -1907,13 +1858,13 @@ public void testDropPartitionEventWithPartitionOnTimestampColumn() throws IOExce run("INSERT INTO TABLE " + dbName + ".ptned PARTITION(b=\"" + ptnVal +"\") values('" + ptn_data[0] + "')", driver); // Replicate insert event and verify - Tuple incrDump = incrementalLoadAndVerify(dbName, bootstrapDump.lastReplId, replDbName); + Tuple incrDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".ptned where (b=\"" + ptnVal + "\") ORDER BY a", ptn_data, driverMirror); run("ALTER TABLE " + dbName + ".ptned DROP PARTITION(b=\"" + ptnVal + "\")", driver); // Replicate drop partition event and verify - incrementalLoadAndVerify(dbName, incrDump.lastReplId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyIfPartitionNotExist(replDbName, "ptned", new ArrayList<>(Arrays.asList(ptnVal)), metaStoreClientMirror); } @@ -1936,13 +1887,13 @@ public void testWithStringPartitionSpecialChars() throws IOException { run("INSERT INTO TABLE " + dbName + ".ptned PARTITION(p=\"" + ptnVal[1] +"\") values('" + ptn_data[1] + "')", driver); // Replicate insert event and verify - Tuple incrDump = incrementalLoadAndVerify(dbName, bootstrapDump.lastReplId, replDbName); + Tuple incrDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT p from " + replDbName + ".ptned ORDER BY p desc", ptnVal, driverMirror); run("ALTER TABLE " + dbName + ".ptned DROP PARTITION(p=\"" + ptnVal[0] + "\")", driver); // Replicate drop partition event and verify - incrementalLoadAndVerify(dbName, incrDump.lastReplId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyIfPartitionNotExist(replDbName, "ptned", new ArrayList<>(Arrays.asList(ptnVal[0])), metaStoreClientMirror); } @@ -1956,7 +1907,6 @@ public void testRenameTableWithCM() throws IOException { run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String replDumpId = bootstrapDump.lastReplId; String[] unptn_data = new String[] { "ten", "twenty" }; String[] ptn_data_1 = new String[] { "fifteen", "fourteen" }; @@ -1973,20 +1923,18 @@ public void testRenameTableWithCM() throws IOException { run("INSERT INTO TABLE " + dbName + ".ptned partition(b=2) values('" + ptn_data_2[1] + "')", driver); // Get the last repl ID corresponding to all insert events except RENAME. - Tuple incrementalDump = replDumpDb(dbName, replDumpId, null, null); - String lastDumpIdWithoutRename = incrementalDump.lastReplId; + Tuple incrementalDump = replDumpDb(dbName); run("ALTER TABLE " + dbName + ".unptned RENAME TO " + dbName + ".unptned_renamed", driver); run("ALTER TABLE " + dbName + ".ptned RENAME TO " + dbName + ".ptned_renamed", driver); - incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, lastDumpIdWithoutRename, replDbName); - replDumpId = incrementalDump.lastReplId; + incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".unptned ORDER BY a", unptn_data, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned where (b=1) ORDER BY a", ptn_data_1, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned where (b=2) ORDER BY a", ptn_data_2, driverMirror); - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyFail("SELECT a from " + replDbName + ".unptned ORDER BY a", driverMirror); verifyFail("SELECT a from " + replDbName + ".ptned where (b=1) ORDER BY a", driverMirror); verifyRun("SELECT a from " + replDbName + ".unptned_renamed ORDER BY a", unptn_data, driverMirror); @@ -2002,7 +1950,6 @@ public void testRenamePartitionWithCM() throws IOException { run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String replDumpId = bootstrapDump.lastReplId; String[] empty = new String[] {}; String[] ptn_data_1 = new String[] { "fifteen", "fourteen" }; @@ -2016,18 +1963,16 @@ public void testRenamePartitionWithCM() throws IOException { run("INSERT INTO TABLE " + dbName + ".ptned partition(b=2) values('" + ptn_data_2[1] + "')", driver); // Get the last repl ID corresponding to all insert events except RENAME. - Tuple incrementalDump = replDumpDb(dbName, replDumpId, null, null); - String lastDumpIdWithoutRename = incrementalDump.lastReplId; + Tuple incrementalDump = replDumpDb(dbName); run("ALTER TABLE " + dbName + ".ptned PARTITION (b=2) RENAME TO PARTITION (b=10)", driver); - incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, lastDumpIdWithoutRename, replDbName); - replDumpId = incrementalDump.lastReplId; + incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".ptned where (b=1) ORDER BY a", ptn_data_1, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned where (b=2) ORDER BY a", ptn_data_2, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned where (b=10) ORDER BY a", empty, driverMirror); - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".ptned where (b=1) ORDER BY a", ptn_data_1, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned where (b=10) ORDER BY a", ptn_data_2, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned where (b=2) ORDER BY a", empty, driverMirror); @@ -2060,8 +2005,8 @@ public void testRenameTableAcrossDatabases() throws IOException { verifyFail("ALTER TABLE " + dbName1 + ".unptned RENAME TO " + dbName2 + ".unptned_renamed", driver); - incrementalLoadAndVerify(dbName1, bootstrap1.lastReplId, replDbName1); - incrementalLoadAndVerify(dbName2, bootstrap2.lastReplId, replDbName2); + incrementalLoadAndVerify(dbName1, replDbName1); + incrementalLoadAndVerify(dbName2, replDbName2); verifyIfTableNotExist(replDbName1, "unptned_renamed", metaStoreClientMirror); verifyIfTableNotExist(replDbName2, "unptned_renamed", metaStoreClientMirror); @@ -2095,8 +2040,8 @@ public void testRenamePartitionedTableAcrossDatabases() throws IOException { verifyFail("ALTER TABLE " + dbName1 + ".ptned RENAME TO " + dbName2 + ".ptned_renamed", driver); - incrementalLoadAndVerify(dbName1, bootstrap1.lastReplId, replDbName1); - incrementalLoadAndVerify(dbName2, bootstrap2.lastReplId, replDbName2); + incrementalLoadAndVerify(dbName1, replDbName1); + incrementalLoadAndVerify(dbName2, replDbName2); verifyIfTableNotExist(replDbName1, "ptned_renamed", metaStoreClientMirror); verifyIfTableNotExist(replDbName2, "ptned_renamed", metaStoreClientMirror); @@ -2145,7 +2090,6 @@ public void testViewsReplication() throws IOException { //verifySetup("SELECT a from " + dbName + ".mat_view", ptn_data_1, driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String replDumpId = bootstrapDump.lastReplId; // view is referring to old database, so no data verifyRun("SELECT * from " + replDbName + ".virtual_view", empty, driverMirror); @@ -2161,8 +2105,7 @@ public void testViewsReplication() throws IOException { //verifySetup("SELECT * from " + dbName + ".mat_view2", unptn_data, driver); // Perform REPL-DUMP/LOAD - Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, replDbName); - replDumpId = incrementalDump.lastReplId; + Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT * from " + replDbName + ".unptned", unptn_data, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned where b=1", ptn_data_1, driverMirror); @@ -2178,8 +2121,7 @@ public void testViewsReplication() throws IOException { verifySetup("SELECT * from " + dbName + ".virtual_view_rename", unptn_data, driver); // Perform REPL-DUMP/LOAD - incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, replDbName); - replDumpId = incrementalDump.lastReplId; + incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT * from " + replDbName + ".virtual_view_rename", empty, driverMirror); // Test "alter table" with schema change @@ -2187,8 +2129,7 @@ public void testViewsReplication() throws IOException { verifySetup("SHOW COLUMNS FROM " + dbName + ".virtual_view_rename", new String[] {"a", "a_"}, driver); // Perform REPL-DUMP/LOAD - incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, replDbName); - replDumpId = incrementalDump.lastReplId; + incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SHOW COLUMNS FROM " + replDbName + ".virtual_view_rename", new String[] {"a", "a_"}, driverMirror); // Test "DROP VIEW" @@ -2196,7 +2137,7 @@ public void testViewsReplication() throws IOException { verifyIfTableNotExist(dbName, "virtual_view", metaStoreClient); // Perform REPL-DUMP/LOAD - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyIfTableNotExist(replDbName, "virtual_view", metaStoreClientMirror); } @@ -2207,7 +2148,7 @@ public void testDumpLimit() throws IOException { String replDbName = dbName + "_dupe"; run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver); - Tuple bootstrapDump = replDumpDb(dbName, null, null, null); + Tuple bootstrapDump = replDumpDb(dbName); String replDumpId = bootstrapDump.lastReplId; String replDumpLocn = bootstrapDump.dumpLocation; @@ -2217,34 +2158,25 @@ public void testDumpLimit() throws IOException { // x events to insert, last repl ID: replDumpId+x run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')", driver); - String firstInsertLastReplId = replDumpDb(dbName, replDumpId, null, null).lastReplId; + String firstInsertLastReplId = replDumpDb(dbName).lastReplId; Integer numOfEventsIns1 = Integer.valueOf(firstInsertLastReplId) - Integer.valueOf(replDumpId); // x events to insert, last repl ID: replDumpId+2x run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[1] + "')", driver); - String secondInsertLastReplId = replDumpDb(dbName, firstInsertLastReplId, null, null).lastReplId; + String secondInsertLastReplId = replDumpDb(dbName).lastReplId; Integer numOfEventsIns2 = Integer.valueOf(secondInsertLastReplId) - Integer.valueOf(firstInsertLastReplId); // x events to insert, last repl ID: replDumpId+3x run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[2] + "')", driver); verifyRun("SELECT a from " + dbName + ".unptned ORDER BY a", unptn_data, driver); - run("REPL LOAD " + replDbName + " FROM '" + replDumpLocn + "'", driverMirror); - Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, null, numOfEventsIns1.toString(), replDbName); + Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDbName); replDumpId = incrementalDump.lastReplId; verifyRun("SELECT a from " + replDbName + ".unptned ORDER BY a", unptn_data_load1, driverMirror); - - Integer lastReplID = Integer.valueOf(replDumpId); - lastReplID += 1000; - String toReplID = String.valueOf(lastReplID); - - incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, toReplID, numOfEventsIns2.toString(), replDbName); - replDumpId = incrementalDump.lastReplId; - + incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".unptned ORDER BY a", unptn_data_load2, driverMirror); - - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".unptned ORDER BY a", unptn_data, driverMirror); } @@ -2278,7 +2210,6 @@ public void testExchangePartition() throws IOException { verifySetup("SELECT a from " + dbName + ".ptned_src where (b=2 and c=3) ORDER BY a", ptn_data_2, driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String replDumpId = bootstrapDump.lastReplId; verifyRun("SELECT a from " + replDbName + ".ptned_src where (b=1 and c=1) ORDER BY a", ptn_data_1, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned_src where (b=2 and c=2) ORDER BY a", ptn_data_2, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned_src where (b=2 and c=3) ORDER BY a", ptn_data_2, driverMirror); @@ -2295,8 +2226,7 @@ public void testExchangePartition() throws IOException { verifySetup("SELECT a from " + dbName + ".ptned_dest where (b=2 and c=2)", empty, driver); verifySetup("SELECT a from " + dbName + ".ptned_dest where (b=2 and c=3)", empty, driver); - Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, replDbName); - replDumpId = incrementalDump.lastReplId; + Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".ptned_src where (b=1 and c=1)", empty, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned_src where (b=2 and c=2) ORDER BY a", ptn_data_2, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned_src where (b=2 and c=3) ORDER BY a", ptn_data_2, driverMirror); @@ -2313,7 +2243,7 @@ public void testExchangePartition() throws IOException { verifySetup("SELECT a from " + dbName + ".ptned_dest where (b=2 and c=2) ORDER BY a", ptn_data_2, driver); verifySetup("SELECT a from " + dbName + ".ptned_dest where (b=2 and c=3) ORDER BY a", ptn_data_2, driver); - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".ptned_src where (b=1 and c=1)", empty, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned_src where (b=2 and c=2)", empty, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned_src where (b=2 and c=3)", empty, driverMirror); @@ -2330,7 +2260,6 @@ public void testTruncateTable() throws IOException { run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String replDumpId = bootstrapDump.lastReplId; String[] unptn_data = new String[] { "eleven", "twelve" }; String[] empty = new String[] {}; @@ -2338,22 +2267,20 @@ public void testTruncateTable() throws IOException { run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[1] + "')", driver); verifySetup("SELECT a from " + dbName + ".unptned ORDER BY a", unptn_data, driver); - Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, replDbName); - replDumpId = incrementalDump.lastReplId; + Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".unptned ORDER BY a", unptn_data, driverMirror); run("TRUNCATE TABLE " + dbName + ".unptned", driver); verifySetup("SELECT a from " + dbName + ".unptned", empty, driver); - incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, replDbName); - replDumpId = incrementalDump.lastReplId; + incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".unptned", empty, driverMirror); String[] unptn_data_after_ins = new String[] { "thirteen" }; run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data_after_ins[0] + "')", driver); verifySetup("SELECT a from " + dbName + ".unptned ORDER BY a", unptn_data_after_ins, driver); - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".unptned ORDER BY a", unptn_data_after_ins, driverMirror); } @@ -2403,7 +2330,7 @@ public void testTruncatePartitionedTable() throws IOException { verifySetup("SELECT a from " + dbName + ".ptned_2 where (b=10)", empty, driver); verifySetup("SELECT a from " + dbName + ".ptned_2 where (b=20)", empty, driver); - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifySetup("SELECT a from " + replDbName + ".ptned_1 where (b=1) ORDER BY a", ptn_data_1, driverMirror); verifySetup("SELECT a from " + replDbName + ".ptned_1 where (b=2)", empty, driverMirror); verifySetup("SELECT a from " + replDbName + ".ptned_2 where (b=10)", empty, driverMirror); @@ -2417,7 +2344,7 @@ public void testTruncateWithCM() throws IOException { String replDbName = dbName + "_dupe"; run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver); - Tuple bootstrapDump = replDumpDb(dbName, null, null, null); + Tuple bootstrapDump = replDumpDb(dbName); String replDumpId = bootstrapDump.lastReplId; String replDumpLocn = bootstrapDump.dumpLocation; @@ -2428,19 +2355,19 @@ public void testTruncateWithCM() throws IOException { // x events to insert, last repl ID: replDumpId+x run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')", driver); - String firstInsertLastReplId = replDumpDb(dbName, replDumpId, null, null).lastReplId; + String firstInsertLastReplId = replDumpDb(dbName).lastReplId; Integer numOfEventsIns1 = Integer.valueOf(firstInsertLastReplId) - Integer.valueOf(replDumpId); // x events to insert, last repl ID: replDumpId+2x run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[1] + "')", driver); verifyRun("SELECT a from " + dbName + ".unptned ORDER BY a", unptn_data, driver); - String secondInsertLastReplId = replDumpDb(dbName, firstInsertLastReplId, null, null).lastReplId; + String secondInsertLastReplId = replDumpDb(dbName).lastReplId; Integer numOfEventsIns2 = Integer.valueOf(secondInsertLastReplId) - Integer.valueOf(firstInsertLastReplId); // y event to truncate, last repl ID: replDumpId+2x+y run("TRUNCATE TABLE " + dbName + ".unptned", driver); verifyRun("SELECT a from " + dbName + ".unptned ORDER BY a", empty, driver); - String thirdTruncLastReplId = replDumpDb(dbName, secondInsertLastReplId, null, null).lastReplId; + String thirdTruncLastReplId = replDumpDb(dbName).lastReplId; Integer numOfEventsTrunc3 = Integer.valueOf(thirdTruncLastReplId) - Integer.valueOf(secondInsertLastReplId); // x events to insert, last repl ID: replDumpId+3x+y @@ -2450,27 +2377,22 @@ public void testTruncateWithCM() throws IOException { run("REPL LOAD " + replDbName + " FROM '" + replDumpLocn + "'", driverMirror); // Dump and load only first insert (1 record) - Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, null, numOfEventsIns1.toString(), replDbName); - replDumpId = incrementalDump.lastReplId; + Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + dbName + "_dupe.unptned ORDER BY a", unptn_data_load1, driverMirror); // Dump and load only second insert (2 records) - Integer lastReplID = Integer.valueOf(replDumpId); - lastReplID += 1000; - String toReplID = String.valueOf(lastReplID); - incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, toReplID, numOfEventsIns2.toString(), replDbName); - replDumpId = incrementalDump.lastReplId; + incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".unptned ORDER BY a", unptn_data_load2, driverMirror); // Dump and load only truncate (0 records) - incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, null, numOfEventsTrunc3.toString(), replDbName); + incrementalDump = incrementalLoadAndVerify(dbName, replDbName); replDumpId = incrementalDump.lastReplId; verifyRun("SELECT a from " + replDbName + ".unptned ORDER BY a", empty, driverMirror); // Dump and load insert after truncate (1 record) - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".unptned ORDER BY a", unptn_data_load1, driverMirror); } @@ -2495,46 +2417,46 @@ public void testIncrementalRepeatEventOnExistingObject() throws IOException { // INSERT EVENT to unpartitioned table run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')", driver); - Tuple replDump = dumpDbFromLastDump(dbName, bootstrapDump); + Tuple replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // INSERT EVENT to partitioned table with dynamic ADD_PARTITION run("INSERT INTO TABLE " + dbName + ".ptned PARTITION(b=1) values('" + ptn_data_1[0] + "')", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // ADD_PARTITION EVENT to partitioned table run("ALTER TABLE " + dbName + ".ptned ADD PARTITION (b=2)", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // INSERT EVENT to partitioned table on existing partition run("INSERT INTO TABLE " + dbName + ".ptned PARTITION(b=2) values('" + ptn_data_2[0] + "')", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // TRUNCATE_PARTITION EVENT on partitioned table run("TRUNCATE TABLE " + dbName + ".ptned PARTITION (b=1)", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // TRUNCATE_TABLE EVENT on unpartitioned table run("TRUNCATE TABLE " + dbName + ".unptned", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // CREATE_TABLE EVENT with multiple partitions run("CREATE TABLE " + dbName + ".unptned_tmp AS SELECT * FROM " + dbName + ".ptned", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // ADD_CONSTRAINT EVENT run("ALTER TABLE " + dbName + ".unptned_tmp ADD CONSTRAINT uk_unptned UNIQUE(a) disable", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // Replicate all the events happened so far - Tuple incrDump = incrementalLoadAndVerify(dbName, bootstrapDump.lastReplId, replDbName); + Tuple incrDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT a from " + replDbName + ".unptned ORDER BY a", empty, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned where (b=1) ORDER BY a", empty, driverMirror); @@ -2576,76 +2498,76 @@ public void testIncrementalRepeatEventOnMissingObject() throws Exception { // INSERT EVENT to unpartitioned table run("INSERT INTO TABLE " + dbName + ".unptned values('" + unptn_data[0] + "')", driver); - Tuple replDump = dumpDbFromLastDump(dbName, bootstrapDump); + Tuple replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // INSERT EVENT to partitioned table with dynamic ADD_PARTITION run("INSERT INTO TABLE " + dbName + ".ptned partition(b=1) values('" + ptn_data_1[0] + "')", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // ADD_PARTITION EVENT to partitioned table run("ALTER TABLE " + dbName + ".ptned ADD PARTITION (b=2)", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // INSERT EVENT to partitioned table on existing partition run("INSERT INTO TABLE " + dbName + ".ptned partition(b=2) values('" + ptn_data_2[0] + "')", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // TRUNCATE_PARTITION EVENT on partitioned table run("TRUNCATE TABLE " + dbName + ".ptned PARTITION(b=1)", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // TRUNCATE_TABLE EVENT on unpartitioned table run("TRUNCATE TABLE " + dbName + ".unptned", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // CREATE_TABLE EVENT on partitioned table run("CREATE TABLE " + dbName + ".ptned_tmp (a string) PARTITIONED BY (b int) STORED AS TEXTFILE", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // INSERT EVENT to partitioned table with dynamic ADD_PARTITION run("INSERT INTO TABLE " + dbName + ".ptned_tmp partition(b=10) values('" + ptn_data_1[0] + "')", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // INSERT EVENT to partitioned table with dynamic ADD_PARTITION run("INSERT INTO TABLE " + dbName + ".ptned_tmp partition(b=20) values('" + ptn_data_2[0] + "')", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // DROP_PARTITION EVENT to partitioned table run("ALTER TABLE " + dbName + ".ptned DROP PARTITION (b=1)", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // RENAME_PARTITION EVENT to partitioned table run("ALTER TABLE " + dbName + ".ptned PARTITION (b=2) RENAME TO PARTITION (b=20)", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // RENAME_TABLE EVENT to unpartitioned table run("ALTER TABLE " + dbName + ".unptned RENAME TO " + dbName + ".unptned_new", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // ADD_CONSTRAINT EVENT run("ALTER TABLE " + dbName + ".ptned_tmp ADD CONSTRAINT uk_unptned UNIQUE(a) disable", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // DROP_TABLE EVENT to partitioned table run("DROP TABLE " + dbName + ".ptned_tmp", driver); - replDump = dumpDbFromLastDump(dbName, replDump); + replDump = replDumpDb(dbName); incrementalDumpList.add(replDump); // Replicate all the events happened so far - Tuple incrDump = incrementalLoadAndVerify(dbName, bootstrapDump.lastReplId, replDbName); + Tuple incrDump = incrementalLoadAndVerify(dbName, replDbName); verifyIfTableNotExist(replDbName, "unptned", metaStoreClientMirror); verifyIfTableNotExist(replDbName, "ptned_tmp", metaStoreClientMirror); @@ -2691,7 +2613,7 @@ public void testConcatenateTable() throws IOException { verifyRun("SELECT a from " + dbName + ".unptned ORDER BY a", unptn_data, driver); // Replicate all the events happened after bootstrap - incrementalLoadAndVerify(dbName, bootstrapDump.lastReplId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); // migration test is failing as CONCATENATE is not working. Its not creating the merged file. if (!isMigrationTest) { @@ -2724,7 +2646,7 @@ public void testConcatenatePartitionedTable() throws IOException { run("ALTER TABLE " + dbName + ".ptned PARTITION(b=2) CONCATENATE", driver); // Replicate all the events happened so far - incrementalLoadAndVerify(dbName, bootstrapDump.lastReplId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); // migration test is failing as CONCATENATE is not working. Its not creating the merged file. if (!isMigrationTest) { @@ -2765,7 +2687,7 @@ public void testIncrementalLoadFailAndRetry() throws IOException { // Replicate all the events happened so far. It should fail as the data files missing in // original path and not available in CM as well. - Tuple incrDump = replDumpDb(dbName, bootstrapDump.lastReplId, null, null); + Tuple incrDump = replDumpDb(dbName); verifyFail("REPL LOAD " + replDbName + " FROM '" + incrDump.dumpLocation + "'", driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned where (b=1) ORDER BY a", empty, driverMirror); @@ -2874,7 +2796,7 @@ public void testConstraints() throws IOException { run("CREATE TABLE " + dbName + ".tbl5(a string, b string, foreign key (a, b) references " + dbName + ".tbl4(a, b) disable novalidate)", driver); run("CREATE TABLE " + dbName + ".tbl6(a string, b string not null disable, unique (a) disable)", driver); - Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDumpId, replDbName); + Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDbName); replDumpId = incrementalDump.lastReplId; String pkName = null; @@ -2904,7 +2826,7 @@ public void testConstraints() throws IOException { run("ALTER TABLE " + dbName + ".tbl5 DROP CONSTRAINT `" + fkName + "`", driver); run("ALTER TABLE " + dbName + ".tbl6 DROP CONSTRAINT `" + nnName + "`", driver); - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); try { List pks = metaStoreClientMirror.getPrimaryKeys(new PrimaryKeysRequest(replDbName , "tbl4")); assertTrue(pks.isEmpty()); @@ -2954,7 +2876,6 @@ public void testRemoveStats() throws IOException { verifySetup("SELECT max(a) from " + dbName + ".ptned where b=1", new String[]{"8"}, driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); - String replDumpId = bootstrapDump.lastReplId; verifyRun("SELECT count(*) from " + replDbName + ".unptned", new String[]{"2"}, driverMirror); verifyRun("SELECT count(*) from " + replDbName + ".ptned", new String[]{"3"}, driverMirror); @@ -2970,7 +2891,7 @@ public void testRemoveStats() throws IOException { run("ANALYZE TABLE " + dbName + ".ptned2 partition(b) COMPUTE STATISTICS FOR COLUMNS", driver); run("ANALYZE TABLE " + dbName + ".ptned2 partition(b) COMPUTE STATISTICS", driver); - incrementalLoadAndVerify(dbName, replDumpId, replDbName); + incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT count(*) from " + replDbName + ".unptned2", new String[]{"2"}, driverMirror); verifyRun("SELECT count(*) from " + replDbName + ".ptned2", new String[]{"3"}, driverMirror); verifyRun("SELECT max(a) from " + replDbName + ".unptned2", new String[]{"2"}, driverMirror); @@ -2993,7 +2914,7 @@ public void testDeleteStagingDir() throws IOException { verifySetup("SELECT * from " + dbName + ".unptned", unptn_data, driver); // Perform repl - String replDumpLocn = replDumpDb(dbName, null, null, null).dumpLocation; + String replDumpLocn = replDumpDb(dbName).dumpLocation; // Reset the driver driverMirror.close(); run("REPL LOAD " + replDbName + " FROM '" + replDumpLocn + "'", driverMirror); @@ -3037,7 +2958,7 @@ public void testCMConflict() throws IOException { run("INSERT INTO TABLE " + dbName + ".unptned values('ten')", driver); // Bootstrap test - Tuple bootstrapDump = replDumpDb(dbName, null, null, null); + Tuple bootstrapDump = replDumpDb(dbName); advanceDumpDir(); run("REPL DUMP " + dbName, driver); String replDumpLocn = bootstrapDump.dumpLocation; @@ -3392,7 +3313,7 @@ public void testMoveOptimizationBootstrap() throws IOException { verifyRun("SELECT fld from " + tableNamePart + " where part = 11" , new String[]{ "3" }, driver); String replDbName = dbName + "_replica"; - Tuple dump = replDumpDb(dbName, null, null, null); + Tuple dump = replDumpDb(dbName); run("REPL LOAD " + replDbName + " FROM '" + dump.dumpLocation + "' with ('hive.repl.enable.move.optimization'='true')", driverMirror); verifyRun("REPL STATUS " + replDbName, dump.lastReplId, driverMirror); @@ -3427,7 +3348,7 @@ public void testMoveOptimizationIncremental() throws IOException { run("CREATE TABLE " + dbName + ".unptned_late AS SELECT * FROM " + dbName + ".unptned", driver); verifySetup("SELECT * from " + dbName + ".unptned_late ORDER BY a", unptn_data, driver); - Tuple incrementalDump = replDumpDb(dbName, replDumpId, null, null); + Tuple incrementalDump = replDumpDb(dbName); run("REPL LOAD " + replDbName + " FROM '" + incrementalDump.dumpLocation + "' with ('hive.repl.enable.move.optimization'='true')", driverMirror); verifyRun("REPL STATUS " + replDbName, incrementalDump.lastReplId, driverMirror); @@ -3445,7 +3366,7 @@ public void testMoveOptimizationIncremental() throws IOException { run("INSERT OVERWRITE TABLE " + dbName + ".unptned values('" + data_after_ovwrite[0] + "')", driver); verifySetup("SELECT a from " + dbName + ".unptned", data_after_ovwrite, driver); - incrementalDump = replDumpDb(dbName, replDumpId, null, null); + incrementalDump = replDumpDb(dbName); run("REPL LOAD " + replDbName + " FROM '" + incrementalDump.dumpLocation + "' with ('hive.repl.enable.move.optimization'='true')", driverMirror); verifyRun("REPL STATUS " + replDbName, incrementalDump.lastReplId, driverMirror); @@ -3497,7 +3418,7 @@ private String verifyAndReturnDbReplStatus(String dbName, String prevReplDumpId, String cmd, String replDbName) throws IOException { run(cmd, driver); - String lastReplDumpId = incrementalLoadAndVerify(dbName, prevReplDumpId, replDbName).lastReplId; + String lastReplDumpId = incrementalLoadAndVerify(dbName, replDbName).lastReplId; assertTrue(Long.parseLong(lastReplDumpId) > Long.parseLong(prevReplDumpId)); return lastReplDumpId; } @@ -3508,7 +3429,7 @@ private String verifyAndReturnTblReplStatus( String replDbName) throws IOException, TException { run(cmd, driver); String lastReplDumpId - = incrementalLoadAndVerify(dbName, lastDbReplDumpId, replDbName).lastReplId; + = incrementalLoadAndVerify(dbName, replDbName).lastReplId; verifyRun("REPL STATUS " + replDbName, lastReplDumpId, driverMirror); assertTrue(Long.parseLong(lastReplDumpId) > Long.parseLong(lastDbReplDumpId)); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java index 63b32c83db..f6728a17b3 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java @@ -116,7 +116,7 @@ public void tearDown() throws Throwable { @Test public void testAcidTablesBootstrap() throws Throwable { // Bootstrap - WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, null, null); + WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, null); replica.load(replicatedDbName, bootstrapDump.dumpLocation); verifyLoadExecution(replicatedDbName, bootstrapDump.lastReplicationId, true); @@ -125,7 +125,7 @@ public void testAcidTablesBootstrap() throws Throwable { prepareIncAcidData(primaryDbName); LOG.info(testName.getMethodName() + ": first incremental dump and load."); WarehouseInstance.Tuple incDump = primary.run("use " + primaryDbName) - .dump(primaryDbName, bootstrapDump.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); replica.load(replicatedDbName, incDump.dumpLocation); verifyIncLoad(replicatedDbName, incDump.lastReplicationId); @@ -134,14 +134,14 @@ public void testAcidTablesBootstrap() throws Throwable { prepareInc2AcidData(primaryDbName, primary.hiveConf); LOG.info(testName.getMethodName() + ": second incremental dump and load."); WarehouseInstance.Tuple inc2Dump = primary.run("use " + primaryDbName) - .dump(primaryDbName, incDump.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); replica.load(replicatedDbName, inc2Dump.dumpLocation); verifyInc2Load(replicatedDbName, inc2Dump.lastReplicationId); } @Test public void testAcidTablesMoveOptimizationBootStrap() throws Throwable { - WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, null, null); + WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, null); replica.load(replicatedDbName, bootstrapDump.dumpLocation, Collections.singletonList("'hive.repl.enable.move.optimization'='true'")); verifyLoadExecution(replicatedDbName, bootstrapDump.lastReplicationId, true); @@ -152,8 +152,7 @@ public void testAcidTablesMoveOptimizationIncremental() throws Throwable { WarehouseInstance.Tuple bootstrapDump = primary.dump(primaryDbName, null); replica.load(replicatedDbName, bootstrapDump.dumpLocation, Collections.singletonList("'hive.repl.enable.move.optimization'='true'")); - WarehouseInstance.Tuple incrDump = prepareDataAndDump(primaryDbName, - bootstrapDump.lastReplicationId, null); + WarehouseInstance.Tuple incrDump = prepareDataAndDump(primaryDbName, null); replica.load(replicatedDbName, incrDump.dumpLocation, Collections.singletonList("'hive.repl.enable.move.optimization'='true'")); verifyLoadExecution(replicatedDbName, incrDump.lastReplicationId, true); @@ -189,7 +188,7 @@ public void testAcidTablesBootstrapWithOpenTxnsTimeout() throws Throwable { "'hive.repl.bootstrap.dump.open.txn.timeout'='1s'"); WarehouseInstance.Tuple bootstrapDump = primary .run("use " + primaryDbName) - .dump(primaryDbName, null, withConfigs); + .dump(primaryDbName, withConfigs); // After bootstrap dump, all the opened txns should be aborted. Verify it. verifyAllOpenTxnsAborted(txns, primaryConf); @@ -289,7 +288,7 @@ public void run() { .verifyResults(new String[]{"1" }); // Incremental should include the concurrent write of data "2" from another txn. - WarehouseInstance.Tuple incrementalDump = primary.dump(primaryDbName, bootstrapDump.lastReplicationId); + WarehouseInstance.Tuple incrementalDump = primary.dump(primaryDbName, Collections.emptyList()); replica.load(replicatedDbName, incrementalDump.dumpLocation) .run("use " + replicatedDbName) .run("repl status " + replicatedDbName) @@ -367,7 +366,7 @@ public void run() { .run("create table t1 (id int) clustered by(id) into 3 buckets stored as orc " + "tblproperties (\"transactional\"=\"true\")") .run("insert into t1 values(100)") - .dump(primaryDbName, bootstrapDump.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); replica.load(replicatedDbName, incrementalDump.dumpLocation) .run("use " + replicatedDbName) @@ -396,7 +395,7 @@ public void testOpenTxnEvent() throws Throwable { .verifyResult("1"); WarehouseInstance.Tuple incrementalDump = - primary.dump(primaryDbName, bootStrapDump.lastReplicationId); + primary.dump(primaryDbName, Collections.emptyList()); long lastReplId = Long.parseLong(bootStrapDump.lastReplicationId); primary.testEventCounts(primaryDbName, lastReplId, null, null, 22); @@ -429,7 +428,7 @@ public void testAbortTxnEvent() throws Throwable { .verifyFailure(new String[]{tableNameFail}); WarehouseInstance.Tuple incrementalDump = - primary.dump(primaryDbName, bootStrapDump.lastReplicationId); + primary.dump(primaryDbName, Collections.emptyList()); replica.load(replicatedDbName, incrementalDump.dumpLocation) .run("REPL STATUS " + replicatedDbName) .verifyResult(incrementalDump.lastReplicationId); @@ -460,7 +459,7 @@ public void testTxnEventNonAcid() throws Throwable { .verifyResult("1"); WarehouseInstance.Tuple incrementalDump = - primary.dump(primaryDbName, bootStrapDump.lastReplicationId); + primary.dump(primaryDbName, Collections.emptyList()); replicaNonAcid.runFailure("REPL LOAD " + replicatedDbName + " FROM '" + incrementalDump.dumpLocation + "'") .run("REPL STATUS " + replicatedDbName) .verifyResult(bootStrapDump.lastReplicationId); @@ -646,7 +645,7 @@ public void testMultiDBTxn() throws Throwable { .verifyResults(resultArray) .run(txnStrCommit); - incrementalDump = primary.dump("`*`", bootStrapDump.lastReplicationId); + incrementalDump = primary.dump("`*`", Collections.emptyList()); // Due to the limitation that we can only have one instance of Persistence Manager Factory in a JVM // we are not able to create multiple embedded derby instances for two different MetaStore instances. diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTablesBootstrap.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTablesBootstrap.java index 9389f26d27..6c322ebd56 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTablesBootstrap.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTablesBootstrap.java @@ -65,7 +65,7 @@ public static void classLevelSetup() throws Exception { @Test public void testAcidTablesBootstrapDuringIncremental() throws Throwable { // Take a bootstrap dump without acid tables - WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, null, + WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, dumpWithoutAcidClause); LOG.info(testName.getMethodName() + ": loading dump without acid tables."); replica.load(replicatedDbName, bootstrapDump.dumpLocation); @@ -76,7 +76,7 @@ public void testAcidTablesBootstrapDuringIncremental() throws Throwable { prepareIncNonAcidData(primaryDbName); LOG.info(testName.getMethodName() + ": incremental dump and load dump with acid table bootstrap."); WarehouseInstance.Tuple incrementalDump = primary.run("use " + primaryDbName) - .dump(primaryDbName, bootstrapDump.lastReplicationId, dumpWithAcidBootstrapClause); + .dump(primaryDbName, dumpWithAcidBootstrapClause); replica.load(replicatedDbName, incrementalDump.dumpLocation); verifyIncLoad(replicatedDbName, incrementalDump.lastReplicationId); // Ckpt should be set on bootstrapped tables. @@ -89,14 +89,14 @@ public void testAcidTablesBootstrapDuringIncremental() throws Throwable { + ": second incremental dump and load dump after incremental with acid table " + "bootstrap."); WarehouseInstance.Tuple inc2Dump = primary.run("use " + primaryDbName) - .dump(primaryDbName, incrementalDump.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); replica.load(replicatedDbName, inc2Dump.dumpLocation); verifyInc2Load(replicatedDbName, inc2Dump.lastReplicationId); } @Test public void testRetryAcidTablesBootstrapFromDifferentDump() throws Throwable { - WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, null, + WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, dumpWithoutAcidClause); LOG.info(testName.getMethodName() + ": loading dump without acid tables."); replica.load(replicatedDbName, bootstrapDump.dumpLocation); @@ -106,7 +106,7 @@ public void testRetryAcidTablesBootstrapFromDifferentDump() throws Throwable { prepareIncNonAcidData(primaryDbName); LOG.info(testName.getMethodName() + ": first incremental dump with acid table bootstrap."); WarehouseInstance.Tuple incDump = primary.run("use " + primaryDbName) - .dump(primaryDbName, bootstrapDump.lastReplicationId, dumpWithAcidBootstrapClause); + .dump(primaryDbName, dumpWithAcidBootstrapClause); // Fail setting ckpt property for table t5 but success for earlier tables BehaviourInjection callerVerifier @@ -139,7 +139,7 @@ public Boolean apply(@Nullable CallerArguments args) { prepareInc2NonAcidData(primaryDbName, primary.hiveConf); LOG.info(testName.getMethodName() + ": second incremental dump with acid table bootstrap"); WarehouseInstance.Tuple inc2Dump = primary.run("use " + primaryDbName) - .dump(primaryDbName, bootstrapDump.lastReplicationId, dumpWithAcidBootstrapClause); + .dump(primaryDbName, dumpWithAcidBootstrapClause); // Set incorrect bootstrap dump to clean tables. Here, used the full bootstrap dump which is invalid. // So, REPL LOAD fails. @@ -176,16 +176,16 @@ public Boolean apply(@Nullable CallerArguments args) { @Test public void retryIncBootstrapAcidFromDifferentDumpWithoutCleanTablesConfig() throws Throwable { - WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, null, + WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, dumpWithoutAcidClause); replica.load(replicatedDbName, bootstrapDump.dumpLocation); prepareIncAcidData(primaryDbName); prepareIncNonAcidData(primaryDbName); WarehouseInstance.Tuple incDump = primary.run("use " + primaryDbName) - .dump(primaryDbName, bootstrapDump.lastReplicationId, dumpWithAcidBootstrapClause); + .dump(primaryDbName, dumpWithAcidBootstrapClause); WarehouseInstance.Tuple inc2Dump = primary.run("use " + primaryDbName) - .dump(primaryDbName, bootstrapDump.lastReplicationId, dumpWithAcidBootstrapClause); + .dump(primaryDbName, dumpWithAcidBootstrapClause); replica.load(replicatedDbName, incDump.dumpLocation); // Re-bootstrapping from different bootstrap dump without clean tables config should fail. @@ -196,7 +196,7 @@ public void retryIncBootstrapAcidFromDifferentDumpWithoutCleanTablesConfig() thr @Test public void testAcidTablesBootstrapDuringIncrementalWithOpenTxnsTimeout() throws Throwable { // Take a dump without ACID tables - WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, null, + WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, dumpWithoutAcidClause); LOG.info(testName.getMethodName() + ": loading dump without acid tables."); replica.load(replicatedDbName, bootstrapDump.dumpLocation); @@ -222,7 +222,7 @@ public void testAcidTablesBootstrapDuringIncrementalWithOpenTxnsTimeout() throws withConfigs.add("'hive.repl.bootstrap.dump.open.txn.timeout'='1s'"); WarehouseInstance.Tuple incDump = primary .run("use " + primaryDbName) - .dump(primaryDbName, bootstrapDump.lastReplicationId, withConfigs); + .dump(primaryDbName, withConfigs); // After bootstrap dump, all the opened txns should be aborted. Verify it. verifyAllOpenTxnsAborted(txns, primaryConf); @@ -254,7 +254,7 @@ public void testAcidTablesBootstrapDuringIncrementalWithOpenTxnsTimeout() throws @Test public void testBootstrapAcidTablesDuringIncrementalWithConcurrentWrites() throws Throwable { // Dump and load bootstrap without ACID tables. - WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, null, + WarehouseInstance.Tuple bootstrapDump = prepareDataAndDump(primaryDbName, dumpWithoutAcidClause); LOG.info(testName.getMethodName() + ": loading dump without acid tables."); replica.load(replicatedDbName, bootstrapDump.dumpLocation); @@ -304,7 +304,7 @@ public void run() { InjectableBehaviourObjectStore.setGetCurrentNotificationEventIdBehaviour(callerInjectedBehavior); WarehouseInstance.Tuple incDump = null; try { - incDump = primary.dump(primaryDbName, bootstrapDump.lastReplicationId, dumpWithAcidBootstrapClause); + incDump = primary.dump(primaryDbName, dumpWithAcidBootstrapClause); callerInjectedBehavior.assertInjectionsPerformed(true, true); } finally { // reset the behaviour @@ -321,7 +321,7 @@ public void run() { // Next Incremental should include the concurrent writes LOG.info(testName.getMethodName() + ": dumping second normal incremental dump from event id = " + incDump.lastReplicationId); - WarehouseInstance.Tuple inc2Dump = primary.dump(primaryDbName, incDump.lastReplicationId); + WarehouseInstance.Tuple inc2Dump = primary.dump(primaryDbName, Collections.emptyList()); LOG.info(testName.getMethodName() + ": loading second normal incremental dump from event id = " + incDump.lastReplicationId); replica.load(replicatedDbName, inc2Dump.dumpLocation); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java index fd4f2dc1b1..74a333aae3 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java @@ -89,7 +89,7 @@ public void testCreateFunctionIncrementalReplication() throws Throwable { + ".testFunctionTwo as 'org.apache.hadoop.hive.ql.udf.generic.GenericUDAFMax'"); WarehouseInstance.Tuple incrementalDump = - primary.dump(primaryDbName, bootStrapDump.lastReplicationId); + primary.dump(primaryDbName, Collections.emptyList()); replica.load(replicatedDbName, incrementalDump.dumpLocation) .run("REPL STATUS " + replicatedDbName) .verifyResult(incrementalDump.lastReplicationId) @@ -204,7 +204,7 @@ public void testDropFunctionIncrementalReplication() throws Throwable { primary.run("Drop FUNCTION " + primaryDbName + ".testFunctionAnother "); WarehouseInstance.Tuple incrementalDump = - primary.dump(primaryDbName, bootStrapDump.lastReplicationId); + primary.dump(primaryDbName, Collections.emptyList()); replica.load(replicatedDbName, incrementalDump.dumpLocation) .run("REPL STATUS " + replicatedDbName) .verifyResult(incrementalDump.lastReplicationId) @@ -274,7 +274,7 @@ public void testIncrementalCreateFunctionWithFunctionBinaryJarsOnHDFS() throws T + ".anotherFunction as 'hivemall.tools.string.StopwordUDF' " + "using " + jarSubString); - WarehouseInstance.Tuple tuple = primary.dump(primaryDbName, bootStrapDump.lastReplicationId); + WarehouseInstance.Tuple tuple = primary.dump(primaryDbName, Collections.emptyList()); replica.load(replicatedDbName, tuple.dumpLocation) .run("SHOW FUNCTIONS LIKE '" + replicatedDbName + "%'") @@ -402,8 +402,7 @@ public void testMetadataBootstrapDump() throws Throwable { "clustered by(key) into 2 buckets stored as orc tblproperties ('transactional'='true')") .run("create table table1 (i int, j int)") .run("insert into table1 values (1,2)") - .dump(primaryDbName, null, - Collections.singletonList("'hive.repl.dump.metadata.only'='true'")); + .dump(primaryDbName, Collections.singletonList("'hive.repl.dump.metadata.only'='true'")); replica.load(replicatedDbName, tuple.dumpLocation) .run("use " + replicatedDbName) @@ -422,8 +421,7 @@ public void testIncrementalMetadataReplication() throws Throwable { .run("create table table2 (a int, city string) partitioned by (country string)") .run("create table table3 (i int, j int)") .run("insert into table1 values (1,2)") - .dump(primaryDbName, null, - Collections.singletonList("'hive.repl.dump.metadata.only'='true'")); + .dump(primaryDbName, Collections.singletonList("'hive.repl.dump.metadata.only'='true'")); replica.load(replicatedDbName, bootstrapTuple.dumpLocation) .run("use " + replicatedDbName) @@ -500,7 +498,7 @@ public void testNonReplDBMetadataReplication() throws Throwable { .run("create table table2 (a int, city string) partitioned by (country string)") .run("create table table3 (i int, j int)") .run("insert into table1 values (1,2)") - .dump(dbName, null, Collections.singletonList("'hive.repl.dump.metadata.only'='true'")); + .dump(dbName, Collections.singletonList("'hive.repl.dump.metadata.only'='true'")); replica.load(replicatedDbName, tuple.dumpLocation) .run("use " + replicatedDbName) @@ -514,8 +512,7 @@ public void testNonReplDBMetadataReplication() throws Throwable { .run("alter table table1 rename to renamed_table1") .run("insert into table2 partition(country='india') values (1,'mumbai') ") .run("create table table4 (i int, j int)") - .dump(dbName, tuple.lastReplicationId, - Collections.singletonList("'hive.repl.dump.metadata.only'='true'")); + .dump(dbName, Collections.singletonList("'hive.repl.dump.metadata.only'='true'")); replica.load(replicatedDbName, tuple.dumpLocation) .run("use " + replicatedDbName) @@ -549,7 +546,7 @@ public void testBootStrapDumpOfWarehouse() throws Throwable { SOURCE_OF_REPLICATION + "' = '1,2,3')") .run("use " + dbTwo) .run("create table t1 (i int, j int)") - .dump("`*`", null, Collections.singletonList("'hive.repl.dump.metadata.only'='true'")); + .dump("`*`", Collections.singletonList("'hive.repl.dump.metadata.only'='true'")); /* Due to the limitation that we can only have one instance of Persistence Manager Factory in a JVM @@ -611,7 +608,7 @@ public void testIncrementalDumpOfWarehouse() throws Throwable { .run("use " + dbOne) .run("create table t1 (i int, j int) partitioned by (load_date date) " + "clustered by(i) into 2 buckets stored as orc tblproperties ('transactional'='true') ") - .dump("`*`", null, Collections.singletonList("'hive.repl.dump.metadata.only'='true'")); + .dump("`*`", Collections.singletonList("'hive.repl.dump.metadata.only'='true'")); String dbTwo = primaryDbName + randomTwo; WarehouseInstance.Tuple incrementalTuple = primary @@ -621,8 +618,7 @@ public void testIncrementalDumpOfWarehouse() throws Throwable { .run("create table t1 (i int, j int)") .run("use " + dbOne) .run("create table t2 (a int, b int)") - .dump("`*`", bootstrapTuple.lastReplicationId, - Arrays.asList("'hive.repl.dump.metadata.only'='true'")); + .dump("`*`", Arrays.asList("'hive.repl.dump.metadata.only'='true'")); /* Due to the limitation that we can only have one instance of Persistence Manager Factory in a JVM @@ -729,7 +725,7 @@ public void testReplLoadFromSourceUsingWithClause() throws Throwable { .run("create function " + primaryDbName + ".testFunctionOne as 'hivemall.tools.string.StopwordUDF' " + "using jar 'ivy://io.github.myui:hivemall:0.4.0-2'") - .dump(primaryDbName, bootstrapTuple.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); // Run load on primary itself primary.load(replicatedDbName, incrementalOneTuple.dumpLocation, withConfigs) @@ -758,7 +754,7 @@ public void testReplLoadFromSourceUsingWithClause() throws Throwable { .run("alter table table2 drop partition(country='usa')") .run("truncate table table3") .run("drop function " + primaryDbName + ".testFunctionOne ") - .dump(primaryDbName, incrementalOneTuple.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); // Run load on primary itself primary.load(replicatedDbName, secondIncremental.dumpLocation, withConfigs) @@ -805,7 +801,7 @@ public void testIncrementalReplWithEventsBatchHavingDropCreateTable() throws Thr .run("create table table2 (id int) partitioned by (country string)") .run("insert into table1 values (1)") .run("insert into table2 partition(country='india') values(1)") - .dump(primaryDbName, bootstrapTuple.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); // Second incremental dump WarehouseInstance.Tuple secondIncremental = primary.run("use " + primaryDbName) @@ -817,7 +813,7 @@ public void testIncrementalReplWithEventsBatchHavingDropCreateTable() throws Thr .run("insert into table2 partition(country='us') values(2)") .run("create table table1 (i int)") .run("insert into table1 values (2)") - .dump(primaryDbName, firstIncremental.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); // First incremental load replica.load(replicatedDbName, firstIncremental.dumpLocation) @@ -862,7 +858,7 @@ public void testIncrementalReplWithDropAndCreateTableDifferentPartitionTypeAndIn .run("insert into table1 partition(country='india') values(1)") .run("insert into table2 values(2)") .run("insert into table3 partition(country='india') values(3)") - .dump(primaryDbName, bootstrapTuple.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); // Second incremental dump WarehouseInstance.Tuple secondIncremental = primary.run("use " + primaryDbName) @@ -875,7 +871,7 @@ public void testIncrementalReplWithDropAndCreateTableDifferentPartitionTypeAndIn .run("insert into table2 partition(country='india') values(20)") .run("create table table3 (id int) partitioned by (name string, rank int)") .run("insert into table3 partition(name='adam', rank=100) values(30)") - .dump(primaryDbName, firstIncremental.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); // First incremental load replica.load(replicatedDbName, firstIncremental.dumpLocation) @@ -940,7 +936,7 @@ public void testShouldDumpMetaDataForNonNativeTableIfSetMeataDataOnly() throws T WarehouseInstance.Tuple bootstrapTuple = primary .run("use " + primaryDbName) .run(createTableQuery) - .dump(primaryDbName, null, Collections.singletonList("'hive.repl.dump.metadata.only'='true'")); + .dump(primaryDbName, Collections.singletonList("'hive.repl.dump.metadata.only'='true'")); // Bootstrap load in replica replica.load(replicatedDbName, bootstrapTuple.dumpLocation) @@ -972,7 +968,7 @@ public void testIncrementalDumpEmptyDumpDirectory() throws Throwable { .status(replicatedDbName) .verifyResult(tuple.lastReplicationId); - tuple = primary.dump(primaryDbName, tuple.lastReplicationId); + tuple = primary.dump(primaryDbName, Collections.emptyList()); replica.load(replicatedDbName, tuple.dumpLocation) .status(replicatedDbName) @@ -982,7 +978,7 @@ public void testIncrementalDumpEmptyDumpDirectory() throws Throwable { String testDbName = primaryDbName + "_test"; tuple = primary.run(" create database " + testDbName) .run("create table " + testDbName + ".tbl (fld int)") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); // Incremental load to existing database with empty dump directory should set the repl id to the last event at src. replica.load(replicatedDbName, tuple.dumpLocation) @@ -990,7 +986,7 @@ public void testIncrementalDumpEmptyDumpDirectory() throws Throwable { .verifyResult(tuple.lastReplicationId); // Incremental load to non existing db should return database not exist error. - tuple = primary.dump("someJunkDB", tuple.lastReplicationId); + tuple = primary.dump("someJunkDB", Collections.emptyList()); try { replica.runCommand("REPL LOAD someJunkDB from '" + tuple.dumpLocation + "'"); assert false; @@ -1025,7 +1021,7 @@ public void testIncrementalDumpMultiIteration() throws Throwable { .run("insert into table1 partition(country='india') values(1)") .run("insert into table2 values(2)") .run("insert into table3 partition(country='india') values(3)") - .dump(primaryDbName, bootstrapTuple.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); replica.load(replicatedDbName, incremental.dumpLocation, Collections.singletonList("'hive.repl.approx.max.load.tasks'='10'")) @@ -1045,7 +1041,7 @@ public void testIncrementalDumpMultiIteration() throws Throwable { "clustered by(key) into 2 buckets stored as orc") .run("create table table4 (i int, j int)") .run("insert into table4 values (1,2)") - .dump(primaryDbName, incremental.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); Path path = new Path(incremental.dumpLocation); FileSystem fs = path.getFileSystem(conf); @@ -1077,13 +1073,13 @@ public void testIfCkptAndSourceOfReplPropsIgnoredByReplDump() throws Throwable { .verifyResult(tuplePrimary.lastReplicationId) .run("show tblproperties t1('custom.property')") .verifyResults(new String[] { "custom.property\tcustom.value" }) - .dumpFailure(replicatedDbName, null) + .dumpFailure(replicatedDbName) .run("alter database " + replicatedDbName + " set dbproperties ('" + SOURCE_OF_REPLICATION + "' = '1, 2, 3')") - .dumpFailure(replicatedDbName, null);//can not dump the db before first successful incremental load is done. + .dumpFailure(replicatedDbName);//can not dump the db before first successful incremental load is done. // do a empty incremental load to allow dump of replicatedDbName - WarehouseInstance.Tuple temp = primary.dump(primaryDbName, tuplePrimary.lastReplicationId); + WarehouseInstance.Tuple temp = primary.dump(primaryDbName, Collections.emptyList()); replica.load(replicatedDbName, temp.dumpLocation); // first successful incremental load. // Bootstrap Repl B -> C @@ -1115,13 +1111,13 @@ public void testIfCkptAndSourceOfReplPropsIgnoredByReplDump() throws Throwable { .run("alter database " + primaryDbName + " set dbproperties('dummy_key'='dummy_val')") .run("alter table t1 set tblproperties('dummy_key'='dummy_val')") .run("alter table t1 partition(country='india') set fileformat orc") - .dump(primaryDbName, tuplePrimary.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); // Incremental Repl A -> B with alters on db/table/partition WarehouseInstance.Tuple tupleReplicaInc = replica.load(replicatedDbName, tuplePrimaryInc.dumpLocation) .run("repl status " + replicatedDbName) .verifyResult(tuplePrimaryInc.lastReplicationId) - .dump(replicatedDbName, tupleReplica.lastReplicationId); + .dump(replicatedDbName, Collections.emptyList()); // Check if DB in B have ckpt property is set to bootstrap dump location used in B and missing for table/partition. db = replica.getDatabase(replicatedDbName); @@ -1488,7 +1484,7 @@ public void testMoveOptimizationIncrementalFailureAfterCopyReplace() throws Thro tuple = primary.run("use " + primaryDbName) .run("insert overwrite table t1 select * from t2") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); testMoveOptimization(primaryDbName, replicatedDbName, replicatedDbName_CM, "t1", "ADD_PARTITION", tuple); } @@ -1509,7 +1505,7 @@ public void testMoveOptimizationIncrementalFailureAfterCopy() throws Throwable { tuple = primary.run("use " + primaryDbName) .run("insert into table t2 partition(country='india') values ('bangalore')") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, Collections.emptyList()); testMoveOptimization(primaryDbName, replicatedDbName, replicatedDbName_CM, "t2", "INSERT", tuple); } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java index 200e02c559..ac30d3edbe 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java @@ -102,7 +102,7 @@ public void replicationWithoutExternalTables() throws Throwable { .run("insert into table t2 partition(country='india') values ('bangalore')") .run("insert into table t2 partition(country='us') values ('austin')") .run("insert into table t2 partition(country='france') values ('paris')") - .dump(primaryDbName, null, dumpWithClause); + .dump(primaryDbName, dumpWithClause); // the _external_tables_file info only should be created if external tables are to be replicated not otherwise assertFalse(primary.miniDFSCluster.getFileSystem() @@ -122,7 +122,7 @@ public void replicationWithoutExternalTables() throws Throwable { .run("create external table t3 (id int)") .run("insert into table t3 values (10)") .run("insert into table t3 values (20)") - .dump(primaryDbName, tuple.lastReplicationId, dumpWithClause); + .dump(primaryDbName, dumpWithClause); // the _external_tables_file info only should be created if external tables are to be replicated not otherwise assertFalse(primary.miniDFSCluster.getFileSystem() @@ -254,7 +254,7 @@ public void externalTableReplicationWithCustomPaths() throws Throwable { .run("create external table a (i int, j int) " + "row format delimited fields terminated by ',' " + "location '" + externalTableLocation.toUri() + "'") - .dump(primaryDbName, null); + .dump(primaryDbName); replica.load(replicatedDbName, bootstrapTuple.dumpLocation, loadWithClause) .run("use " + replicatedDbName) @@ -272,7 +272,7 @@ public void externalTableReplicationWithCustomPaths() throws Throwable { } WarehouseInstance.Tuple incrementalTuple = primary.run("create table b (i int)") - .dump(primaryDbName, bootstrapTuple.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, incrementalTuple.dumpLocation, loadWithClause) .run("select i From a") @@ -285,7 +285,7 @@ public void externalTableReplicationWithCustomPaths() throws Throwable { new Path("/" + testName.getMethodName() + "/" + primaryDbName + "/new_location/a/"); incrementalTuple = primary.run("use " + primaryDbName) .run("alter table a set location '" + externalTableLocation + "'") - .dump(primaryDbName, incrementalTuple.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, incrementalTuple.dumpLocation, loadWithClause) .run("use " + replicatedDbName) @@ -332,7 +332,7 @@ public void externalTableWithPartitions() throws Throwable { tuple = primary.run("use " + primaryDbName) .run("insert into t2 partition(country='australia') values ('sydney')") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, null); assertExternalFileInfo(Collections.singletonList("t2"), new Path(tuple.dumpLocation, FILE_NAME)); @@ -360,7 +360,7 @@ public void externalTableWithPartitions() throws Throwable { tuple = primary.run("use " + primaryDbName) .run("ALTER TABLE t2 ADD PARTITION (country='france') LOCATION '" + customPartitionLocation .toString() + "'") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause) .run("use " + replicatedDbName) @@ -374,7 +374,7 @@ public void externalTableWithPartitions() throws Throwable { tuple = primary.run("use " + primaryDbName) .run("alter table t2 partition (country='france') set location '" + tmpLocation + "'") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause) .run("use " + replicatedDbName) @@ -394,7 +394,7 @@ public void externalTableWithPartitions() throws Throwable { tuple = primary.run("use " + primaryDbName) .run("insert into table t2 partition(country='france') values ('lyon')") .run("alter table t2 set location '" + tmpLocation2 + "'") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause); assertTablePartitionLocation(primaryDbName + ".t2", replicatedDbName + ".t2"); @@ -416,7 +416,7 @@ public void externalTableIncrementalReplication() throws Throwable { + "'") .run("alter table t1 add partition(country='india')") .run("alter table t1 add partition(country='us')") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, null); assertExternalFileInfo(Collections.singletonList("t1"), new Path(tuple.dumpLocation, FILE_NAME)); @@ -451,7 +451,7 @@ public void externalTableIncrementalReplication() throws Throwable { } // Repl load with zero events but external tables location info should present. - tuple = primary.dump(primaryDbName, tuple.lastReplicationId); + tuple = primary.dump(primaryDbName, null); assertExternalFileInfo(Collections.singletonList("t1"), new Path(tuple.dumpLocation, FILE_NAME)); replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause) @@ -473,7 +473,7 @@ public void externalTableIncrementalReplication() throws Throwable { tuple = primary .run("alter table t1 drop partition (country='india')") .run("alter table t1 drop partition (country='us')") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, tuple.dumpLocation) .run("select * From t1") @@ -501,7 +501,7 @@ public void bootstrapExternalTablesDuringIncrementalPhase() throws Throwable { .run("insert into table t2 partition(country='india') values ('bangalore')") .run("insert into table t2 partition(country='us') values ('austin')") .run("insert into table t2 partition(country='france') values ('paris')") - .dump(primaryDbName, null, dumpWithClause); + .dump(primaryDbName, dumpWithClause); // the _external_tables_file info only should be created if external tables are to be replicated not otherwise assertFalse(primary.miniDFSCluster.getFileSystem() @@ -525,7 +525,7 @@ public void bootstrapExternalTablesDuringIncrementalPhase() throws Throwable { .run("insert into table t3 values (10)") .run("insert into table t3 values (20)") .run("create table t4 as select * from t3") - .dump(primaryDbName, tuple.lastReplicationId, dumpWithClause); + .dump(primaryDbName, dumpWithClause); // the _external_tables_file info should be created as external tables are to be replicated. assertTrue(primary.miniDFSCluster.getFileSystem() @@ -599,7 +599,7 @@ public void retryBootstrapExternalTablesFromDifferentDump() throws Throwable { .run("insert into table t2 partition(country='india') values ('bangalore')") .run("insert into table t2 partition(country='us') values ('austin')") .run("create table t3 as select * from t1") - .dump(primaryDbName, null, dumpWithClause); + .dump(primaryDbName, dumpWithClause); replica.load(replicatedDbName, tupleBootstrapWithoutExternal.dumpLocation, loadWithClause) .status(replicatedDbName) @@ -618,7 +618,7 @@ public void retryBootstrapExternalTablesFromDifferentDump() throws Throwable { .run("create external table t4 (id int)") .run("insert into table t4 values (10)") .run("create table t5 as select * from t4") - .dump(primaryDbName, tupleBootstrapWithoutExternal.lastReplicationId, dumpWithClause); + .dump(primaryDbName, dumpWithClause); // Fail setting ckpt property for table t4 but success for t2. BehaviourInjection callerVerifier @@ -652,7 +652,7 @@ public Boolean apply(@Nullable CallerArguments args) { .run("drop table t2") .run("create table t2 as select * from t4") .run("insert into table t4 values (20)") - .dump(primaryDbName, tupleIncWithExternalBootstrap.lastReplicationId, dumpWithClause); + .dump(primaryDbName, dumpWithClause); // Set incorrect bootstrap dump to clean tables. Here, used the full bootstrap dump which is invalid. // So, REPL LOAD fails. @@ -717,7 +717,7 @@ public void testExternalTablesIncReplicationWithConcurrentDropTable() throws Thr WarehouseInstance.Tuple tupleBootstrap = primary.run("use " + primaryDbName) .run("create external table t1 (id int)") .run("insert into table t1 values (1)") - .dump(primaryDbName, null, dumpWithClause); + .dump(primaryDbName, dumpWithClause); replica.load(replicatedDbName, tupleBootstrap.dumpLocation, loadWithClause); @@ -746,7 +746,7 @@ public Table apply(@Nullable Table table) { WarehouseInstance.Tuple tupleInc; try { // The t1 table will be skipped from data location listing. - tupleInc = primary.dump(primaryDbName, tupleBootstrap.lastReplicationId, dumpWithClause); + tupleInc = primary.dump(primaryDbName, dumpWithClause); tableNuller.assertInjectionsPerformed(true, true); } finally { InjectableBehaviourObjectStore.resetGetTableBehaviour(); // reset the behaviour @@ -776,7 +776,7 @@ public void testIncrementalDumpEmptyDumpDirectory() throws Throwable { .run("create external table t1 (id int)") .run("insert into table t1 values (1)") .run("insert into table t1 values (2)") - .dump(primaryDbName, null, dumpWithClause); + .dump(primaryDbName, dumpWithClause); replica.load(replicatedDbName, tuple.dumpLocation) .status(replicatedDbName) @@ -784,7 +784,7 @@ public void testIncrementalDumpEmptyDumpDirectory() throws Throwable { // This looks like an empty dump but it has the ALTER TABLE event created by the previous // dump. We need it here so that the next dump won't have any events. - WarehouseInstance.Tuple incTuple = primary.dump(primaryDbName, tuple.lastReplicationId, dumpWithClause); + WarehouseInstance.Tuple incTuple = primary.dump(primaryDbName, dumpWithClause); replica.load(replicatedDbName, incTuple.dumpLocation, loadWithClause) .status(replicatedDbName) .verifyResult(incTuple.lastReplicationId); @@ -795,7 +795,7 @@ public void testIncrementalDumpEmptyDumpDirectory() throws Throwable { WarehouseInstance.Tuple inc2Tuple = primary.run("use " + extraPrimaryDb) .run("create table tbl (fld int)") .run("use " + primaryDbName) - .dump(primaryDbName, incTuple.lastReplicationId, dumpWithClause); + .dump(primaryDbName, dumpWithClause); Assert.assertEquals(primary.getCurrentNotificationEventId().getEventId(), Long.valueOf(inc2Tuple.lastReplicationId).longValue()); @@ -818,7 +818,7 @@ public void testExtTableBootstrapDuringIncrementalWithoutAnyEvents() throws Thro .run("insert into table t1 values (1)") .run("create table t2 (id int)") .run("insert into table t2 values (1)") - .dump(primaryDbName, null, dumpWithClause); + .dump(primaryDbName, dumpWithClause); replica.load(replicatedDbName, bootstrapDump.dumpLocation, loadWithClause) .status(replicatedDbName) @@ -832,7 +832,7 @@ public void testExtTableBootstrapDuringIncrementalWithoutAnyEvents() throws Thro // This looks like an empty dump but it has the ALTER TABLE event created by the previous // dump. We need it here so that the next dump won't have any events. - WarehouseInstance.Tuple incTuple = primary.dump(primaryDbName, bootstrapDump.lastReplicationId); + WarehouseInstance.Tuple incTuple = primary.dump(primaryDbName, null); replica.load(replicatedDbName, incTuple.dumpLocation) .status(replicatedDbName) .verifyResult(incTuple.lastReplicationId); @@ -841,7 +841,7 @@ public void testExtTableBootstrapDuringIncrementalWithoutAnyEvents() throws Thro dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'"); WarehouseInstance.Tuple inc2Tuple = primary.run("use " + primaryDbName) - .dump(primaryDbName, incTuple.lastReplicationId, dumpWithClause); + .dump(primaryDbName, dumpWithClause); replica.load(replicatedDbName, inc2Tuple.dumpLocation, loadWithClause) .status(replicatedDbName) @@ -867,7 +867,7 @@ public void replicationWithTableNameContainsKeywords() throws Throwable { .run("insert into table t2_constraints partition(country='india') values ('bangalore')") .run("insert into table t2_constraints partition(country='us') values ('austin')") .run("insert into table t2_constraints partition(country='france') values ('paris')") - .dump(primaryDbName, null); + .dump(primaryDbName); replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause) .run("repl status " + replicatedDbName) @@ -888,7 +888,7 @@ public void replicationWithTableNameContainsKeywords() throws Throwable { .run("create table t4_tables (id int)") .run("insert into table t4_tables values (10)") .run("insert into table t4_tables values (20)") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause) .run("use " + replicatedDbName) diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTablesMetaDataOnly.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTablesMetaDataOnly.java index 1d824678c1..be24d9d0f7 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTablesMetaDataOnly.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTablesMetaDataOnly.java @@ -101,7 +101,7 @@ public void replicationWithoutExternalTables() throws Throwable { .run("insert into table t2 partition(country='india') values ('bangalore')") .run("insert into table t2 partition(country='us') values ('austin')") .run("insert into table t2 partition(country='france') values ('paris')") - .dump(primaryDbName, null, dumpWithClause); + .dump(primaryDbName, dumpWithClause); // the _external_tables_file info only should be created if external tables are to be replicated not otherwise assertFalse(primary.miniDFSCluster.getFileSystem() @@ -121,7 +121,7 @@ public void replicationWithoutExternalTables() throws Throwable { .run("create external table t3 (id int)") .run("insert into table t3 values (10)") .run("insert into table t3 values (20)") - .dump(primaryDbName, tuple.lastReplicationId, dumpWithClause); + .dump(primaryDbName, dumpWithClause); // the _external_tables_file data only should be created if external tables are to be replicated not otherwise assertFalse(primary.miniDFSCluster.getFileSystem() @@ -215,7 +215,7 @@ public void externalTableReplicationWithCustomPaths() throws Throwable { .run("create external table a (i int, j int) " + "row format delimited fields terminated by ',' " + "location '" + externalTableLocation.toUri() + "'") - .dump(primaryDbName, null); + .dump(primaryDbName); replica.load(replicatedDbName, bootstrapTuple.dumpLocation, loadWithClause) .run("use " + replicatedDbName) @@ -231,7 +231,7 @@ public void externalTableReplicationWithCustomPaths() throws Throwable { } WarehouseInstance.Tuple incrementalTuple = primary.run("create table b (i int)") - .dump(primaryDbName, bootstrapTuple.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, incrementalTuple.dumpLocation, loadWithClause) .run("select i From a") @@ -244,7 +244,7 @@ public void externalTableReplicationWithCustomPaths() throws Throwable { new Path("/" + testName.getMethodName() + "/" + primaryDbName + "/new_location/a/"); incrementalTuple = primary.run("use " + primaryDbName) .run("alter table a set location '" + externalTableLocation + "'") - .dump(primaryDbName, incrementalTuple.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, incrementalTuple.dumpLocation, loadWithClause) .run("use " + replicatedDbName) @@ -287,7 +287,7 @@ public void externalTableWithPartitions() throws Throwable { tuple = primary.run("use " + primaryDbName) .run("insert into t2 partition(country='australia') values ('sydney')") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, null); assertFalseExternalFileInfo(new Path(tuple.dumpLocation, FILE_NAME)); @@ -314,7 +314,7 @@ public void externalTableWithPartitions() throws Throwable { tuple = primary.run("use " + primaryDbName) .run("ALTER TABLE t2 ADD PARTITION (country='france') LOCATION '" + customPartitionLocation .toString() + "'") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause) .run("use " + replicatedDbName) @@ -328,7 +328,7 @@ public void externalTableWithPartitions() throws Throwable { tuple = primary.run("use " + primaryDbName) .run("alter table t2 partition (country='france') set location '" + tmpLocation + "'") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause) .run("use " + replicatedDbName) @@ -344,7 +344,7 @@ public void externalTableWithPartitions() throws Throwable { tuple = primary.run("use " + primaryDbName) .run("insert into table t2 partition(country='france') values ('lyon')") .run("alter table t2 set location '" + tmpLocation2 + "'") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause); } @@ -365,7 +365,7 @@ public void externalTableIncrementalReplication() throws Throwable { + "'") .run("alter table t1 add partition(country='india')") .run("alter table t1 add partition(country='us')") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, null); assertFalseExternalFileInfo(new Path(tuple.dumpLocation, FILE_NAME)); @@ -400,7 +400,7 @@ public void externalTableIncrementalReplication() throws Throwable { } // Repl load with zero events but external tables location info should present. - tuple = primary.dump(primaryDbName, tuple.lastReplicationId); + tuple = primary.dump(primaryDbName, null); assertFalseExternalFileInfo(new Path(tuple.dumpLocation, FILE_NAME)); replica.load(replicatedDbName, tuple.dumpLocation, loadWithClause) @@ -422,7 +422,7 @@ public void externalTableIncrementalReplication() throws Throwable { tuple = primary .run("alter table t1 drop partition (country='india')") .run("alter table t1 drop partition (country='us')") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, tuple.dumpLocation) .run("select * From t1") @@ -449,7 +449,7 @@ public void bootstrapExternalTablesDuringIncrementalPhase() throws Throwable { .run("insert into table t2 partition(country='india') values ('bangalore')") .run("insert into table t2 partition(country='us') values ('austin')") .run("insert into table t2 partition(country='france') values ('paris')") - .dump(primaryDbName, null, dumpWithClause); + .dump(primaryDbName, dumpWithClause); // the _external_tables_file info only should be created if external tables are to be replicated not otherwise assertFalse(primary.miniDFSCluster.getFileSystem() @@ -474,7 +474,7 @@ public void bootstrapExternalTablesDuringIncrementalPhase() throws Throwable { .run("insert into table t3 values (10)") .run("insert into table t3 values (20)") .run("create table t4 as select * from t3") - .dump(primaryDbName, tuple.lastReplicationId, dumpWithClause); + .dump(primaryDbName, dumpWithClause); // the _external_tables_file info should be created as external tables are to be replicated. assertTrue(primary.miniDFSCluster.getFileSystem() @@ -540,7 +540,7 @@ public void testExternalTablesIncReplicationWithConcurrentDropTable() throws Thr WarehouseInstance.Tuple tupleBootstrap = primary.run("use " + primaryDbName) .run("create external table t1 (id int)") .run("insert into table t1 values (1)") - .dump(primaryDbName, null, dumpWithClause); + .dump(primaryDbName, dumpWithClause); replica.load(replicatedDbName, tupleBootstrap.dumpLocation, loadWithClause); @@ -569,7 +569,7 @@ public Table apply(@Nullable Table table) { WarehouseInstance.Tuple tupleInc; try { // The t1 table will be skipped from data location listing. - tupleInc = primary.dump(primaryDbName, tupleBootstrap.lastReplicationId, dumpWithClause); + tupleInc = primary.dump(primaryDbName, dumpWithClause); tableNuller.assertInjectionsPerformed(true, true); } finally { InjectableBehaviourObjectStore.resetGetTableBehaviour(); // reset the behaviour @@ -598,7 +598,7 @@ public void testIncrementalDumpEmptyDumpDirectory() throws Throwable { .run("create external table t1 (id int)") .run("insert into table t1 values (1)") .run("insert into table t1 values (2)") - .dump(primaryDbName, null, dumpWithClause); + .dump(primaryDbName, dumpWithClause); replica.load(replicatedDbName, tuple.dumpLocation) .status(replicatedDbName) @@ -606,7 +606,7 @@ public void testIncrementalDumpEmptyDumpDirectory() throws Throwable { // This looks like an empty dump but it has the ALTER TABLE event created by the previous // dump. We need it here so that the next dump won't have any events. - WarehouseInstance.Tuple incTuple = primary.dump(primaryDbName, tuple.lastReplicationId, dumpWithClause); + WarehouseInstance.Tuple incTuple = primary.dump(primaryDbName, dumpWithClause); replica.load(replicatedDbName, incTuple.dumpLocation, loadWithClause) .status(replicatedDbName) .verifyResult(incTuple.lastReplicationId); @@ -617,7 +617,7 @@ public void testIncrementalDumpEmptyDumpDirectory() throws Throwable { WarehouseInstance.Tuple inc2Tuple = primary.run("use " + extraPrimaryDb) .run("create table tbl (fld int)") .run("use " + primaryDbName) - .dump(primaryDbName, incTuple.lastReplicationId, dumpWithClause); + .dump(primaryDbName, dumpWithClause); Assert.assertEquals(primary.getCurrentNotificationEventId().getEventId(), Long.valueOf(inc2Tuple.lastReplicationId).longValue()); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java index be0d96e682..89cf078efd 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosIncrementalLoadAcidTables.java @@ -215,7 +215,7 @@ public void testReplCM() throws Throwable { ReplicationTestUtils.insertRecords(primary, primaryDbName, primaryDbNameExtra, tableName, null, false, ReplicationTestUtils.OperationType.REPL_TEST_ACID_INSERT); - incrementalDump = primary.dump(primaryDbName, bootStrapDump.lastReplicationId); + incrementalDump = primary.dump(primaryDbName, null); primary.run("drop table " + primaryDbName + "." + tableName); replica.loadWithoutExplain(replicatedDbName, incrementalDump.dumpLocation) .run("REPL STATUS " + replicatedDbName).verifyResult(incrementalDump.lastReplicationId); @@ -225,7 +225,7 @@ public void testReplCM() throws Throwable { ReplicationTestUtils.insertRecords(primary, primaryDbName, primaryDbNameExtra, tableNameMM, null, true, ReplicationTestUtils.OperationType.REPL_TEST_ACID_INSERT); - incrementalDump = primary.dump(primaryDbName, bootStrapDump.lastReplicationId); + incrementalDump = primary.dump(primaryDbName, null); primary.run("drop table " + primaryDbName + "." + tableNameMM); replica.loadWithoutExplain(replicatedDbName, incrementalDump.dumpLocation) .run("REPL STATUS " + replicatedDbName).verifyResult(incrementalDump.lastReplicationId); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java index e094ed23a3..37f2d1c6b9 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigration.java @@ -224,7 +224,7 @@ public void tearDown() throws Throwable { + "stored as avro tblproperties ('avro.schema.url'='" + avroSchemaFile.toUri() .toString() + "')") .run("insert into avro_table_part partition (country='india') values ('another', 13)") - .dump(primaryDbName, fromReplId); + .dump(primaryDbName, null); assertFalse(isTransactionalTable(primary.getTable(primaryDbName, "tacid"))); assertFalse(isTransactionalTable(primary.getTable(primaryDbName, "tacidpart"))); assertFalse(isTransactionalTable(primary.getTable(primaryDbName, "tflat"))); @@ -342,7 +342,7 @@ public void testBootstrapLoadMigrationManagedToAcid() throws Throwable { @Test public void testIncrementalLoadMigrationManagedToAcid() throws Throwable { - WarehouseInstance.Tuple tuple = primary.dump(primaryDbName, null); + WarehouseInstance.Tuple tuple = primary.dump(primaryDbName); replica.load(replicatedDbName, tuple.dumpLocation); tuple = prepareDataAndDump(primaryDbName, tuple.lastReplicationId); replica.load(replicatedDbName, tuple.dumpLocation); @@ -351,7 +351,7 @@ public void testIncrementalLoadMigrationManagedToAcid() throws Throwable { @Test public void testIncrementalLoadMigrationManagedToAcidFailure() throws Throwable { - WarehouseInstance.Tuple tuple = primary.dump(primaryDbName, null); + WarehouseInstance.Tuple tuple = primary.dump(primaryDbName); replica.load(replicatedDbName, tuple.dumpLocation); tuple = prepareDataAndDump(primaryDbName, tuple.lastReplicationId); loadWithFailureInAddNotification("tacid", tuple.dumpLocation); @@ -364,7 +364,7 @@ public void testIncrementalLoadMigrationManagedToAcidFailure() throws Throwable @Test public void testIncrementalLoadMigrationManagedToAcidFailurePart() throws Throwable { - WarehouseInstance.Tuple tuple = primary.dump(primaryDbName, null); + WarehouseInstance.Tuple tuple = primary.dump(primaryDbName); replica.load(replicatedDbName, tuple.dumpLocation); tuple = prepareDataAndDump(primaryDbName, tuple.lastReplicationId); loadWithFailureInAddNotification("tacidpart", tuple.dumpLocation); @@ -377,7 +377,7 @@ public void testIncrementalLoadMigrationManagedToAcidFailurePart() throws Throwa @Test public void testIncrementalLoadMigrationManagedToAcidAllOp() throws Throwable { - WarehouseInstance.Tuple bootStrapDump = primary.dump(primaryDbName, null); + WarehouseInstance.Tuple bootStrapDump = primary.dump(primaryDbName); replica.load(replicatedDbName, bootStrapDump.dumpLocation) .run("REPL STATUS " + replicatedDbName) .verifyResult(bootStrapDump.lastReplicationId); @@ -416,14 +416,14 @@ public void testBootstrapConvertedExternalTableAutoPurgeDataOnDrop() throws Thro + "'org.apache.hadoop.hive.serde2.avro.AvroSerDe' stored as avro " + "tblproperties ('avro.schema.url'='" + avroSchemaFile.toUri().toString() + "')") .run("insert into avro_tbl partition (country='india') values ('another', 13)") - .dump(primaryDbName, null); + .dump(primaryDbName); replica.load(replicatedDbName, bootstrap.dumpLocation); Path dataLocation = assertTablePath(replicatedDbName, "avro_tbl"); WarehouseInstance.Tuple incremental = primary.run("use " + primaryDbName) .run("drop table avro_tbl") - .dump(primaryDbName, bootstrap.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, incremental.dumpLocation); // After drop, the external table data location should be auto deleted as it is converted one. @@ -432,7 +432,7 @@ public void testBootstrapConvertedExternalTableAutoPurgeDataOnDrop() throws Thro @Test public void testIncConvertedExternalTableAutoDeleteDataDirOnDrop() throws Throwable { - WarehouseInstance.Tuple bootstrap = primary.dump(primaryDbName, null); + WarehouseInstance.Tuple bootstrap = primary.dump(primaryDbName); replica.load(replicatedDbName, bootstrap.dumpLocation); WarehouseInstance.Tuple incremental = primary.run("use " + primaryDbName) @@ -440,7 +440,7 @@ public void testIncConvertedExternalTableAutoDeleteDataDirOnDrop() throws Throwa + "'org.apache.hadoop.hive.serde2.avro.AvroSerDe' stored as avro " + "tblproperties ('avro.schema.url'='" + avroSchemaFile.toUri().toString() + "')") .run("insert into avro_tbl values ('str', 13)") - .dump(primaryDbName, bootstrap.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, incremental.dumpLocation); // Data location is valid and is under default external warehouse directory. @@ -451,7 +451,7 @@ public void testIncConvertedExternalTableAutoDeleteDataDirOnDrop() throws Throwa incremental = primary.run("use " + primaryDbName) .run("drop table avro_tbl") - .dump(primaryDbName, incremental.lastReplicationId); + .dump(primaryDbName, null); replica.load(replicatedDbName, incremental.dumpLocation); // After drop, the external table data location should be auto deleted as it is converted one. @@ -471,7 +471,7 @@ public void testBootstrapLoadMigrationToAcidWithMoveOptimization() throws Throwa public void testIncrementalLoadMigrationToAcidWithMoveOptimization() throws Throwable { List withConfigs = Collections.singletonList("'hive.repl.enable.move.optimization'='true'"); - WarehouseInstance.Tuple tuple = primary.dump(primaryDbName, null); + WarehouseInstance.Tuple tuple = primary.dump(primaryDbName); replica.load(replicatedDbName, tuple.dumpLocation); tuple = prepareDataAndDump(primaryDbName, tuple.lastReplicationId); replica.load(replicatedDbName, tuple.dumpLocation, withConfigs); @@ -513,7 +513,7 @@ public void testMigrationWithUpgrade() throws Throwable { .run("insert into tacid values (3)") .run("create table texternal (id int) ") .run("insert into texternal values (1)") - .dump(primaryDbName, null); + .dump(primaryDbName); replica.load(replicatedDbName, tuple.dumpLocation) .run("use " + replicatedDbName) .run("repl status " + replicatedDbName) @@ -557,7 +557,7 @@ public void testMigrationWithUpgrade() throws Throwable { withConfigs.add("'hive.repl.dump.include.acid.tables'='true'"); withConfigs.add("'hive.repl.include.external.tables'='true'"); withConfigs.add("'hive.distcp.privileged.doAs' = '" + UserGroupInformation.getCurrentUser().getUserName() + "'"); - tuple = primary.dump(primaryDbName, tuple.lastReplicationId, withConfigs); + tuple = primary.dump(primaryDbName, withConfigs); replica.load(replicatedDbName, tuple.dumpLocation, withConfigs); replica.run("use " + replicatedDbName) .run("repl status " + replicatedDbName) diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java index cefdca620f..e1894464a1 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationWithTableMigrationEx.java @@ -191,7 +191,7 @@ public void testConcurrentOpDuringBootStrapDumpCreateTableReplay() throws Throwa assertTrue(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters())); // next incremental dump - tuple = primary.dump(primaryDbName, tuple.lastReplicationId); + tuple = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation); verifyLoadExecution(replicatedDbName, tuple.lastReplicationId); assertFalse(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters())); @@ -208,7 +208,7 @@ public void testConcurrentOpDuringBootStrapDumpInsertReplay() throws Throwable { assertTrue(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters())); // next incremental dump - tuple = primary.dump(primaryDbName, tuple.lastReplicationId); + tuple = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation); verifyLoadExecution(replicatedDbName, tuple.lastReplicationId); assertFalse(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters())); @@ -226,7 +226,7 @@ public void testTableLevelDumpMigration() throws Throwable { tuple = primary.run("use " + primaryDbName) .run("insert into t1 values (1, 2)") - .dump(primaryDbName+".'t1'", tuple.lastReplicationId); + .dump(primaryDbName+".'t1'", null); replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation); assertFalse(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters())); } @@ -256,7 +256,7 @@ public void testConcurrentOpDuringBootStrapDumpInsertOverwrite() throws Throwabl assertTrue(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters())); // next incremental dump - tuple = primary.dump(primaryDbName, tuple.lastReplicationId); + tuple = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation); replica.run("use " + replicatedDbName) .run("show tables") @@ -315,7 +315,7 @@ public void testIncLoadPenFlagPropAlterDB() throws Throwable { tuple = primary.run("use " + primaryDbName) .run("alter database " + primaryDbName + " set dbproperties('dummy_key'='dummy_val')") .run("create table tbl_temp (fld int)") - .dump(primaryDbName, tuple.lastReplicationId); + .dump(primaryDbName, null); loadWithFailureInAddNotification("tbl_temp", tuple.dumpLocation); Database replDb = replica.getDatabase(replicatedDbName); @@ -324,7 +324,7 @@ public void testIncLoadPenFlagPropAlterDB() throws Throwable { assertTrue(replDb.getParameters().get("dummy_key").equalsIgnoreCase("dummy_val")); // next incremental dump - tuple = primary.dump(primaryDbName, tuple.lastReplicationId); + tuple = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation); assertFalse(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters())); } @@ -342,7 +342,7 @@ public void testIncLoadPenFlagWithMoveOptimization() throws Throwable { assertTrue(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters())); // next incremental dump - tuple = primary.dump(primaryDbName, tuple.lastReplicationId); + tuple = primary.dump(primaryDbName, null); replica.load(replicatedDbName, tuple.dumpLocation, withClause); assertFalse(ReplUtils.isFirstIncPending(replica.getDatabase(replicatedDbName).getParameters())); } @@ -388,7 +388,7 @@ public void testOnwerPropagation() throws Throwable { // test incremental alterUserName("hive1"); - tuple = primary.dump(primaryDbName, tuple.lastReplicationId); + tuple = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation); verifyUserName("hive1"); } @@ -409,7 +409,7 @@ public void testOnwerPropagationInc() throws Throwable { // test incremental when table is getting created in the same load alterUserName("hive"); - tuple = primary.dump(primaryDbName, tuple.lastReplicationId); + tuple = primary.dump(primaryDbName, null); replica.loadWithoutExplain(replicatedDbName, tuple.dumpLocation); verifyUserName("hive"); } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java new file mode 100644 index 0000000000..744acc34e1 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestScheduledReplicationScenarios.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder; +import org.apache.hadoop.hive.ql.exec.repl.ReplDumpWork; +import org.apache.hadoop.hive.ql.scheduled.ScheduledQueryExecutionService; +import org.apache.hadoop.hive.shims.Utils; +import org.junit.Before; +import org.junit.After; +import org.junit.Test; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.Map; +import java.util.HashMap; +import java.util.List; + + +/** + * TestScheduledReplicationScenarios - test scheduled replication . + */ +public class TestScheduledReplicationScenarios extends BaseReplicationScenariosAcidTables { + + private static final String REPLICA_EXTERNAL_BASE = "/replica_external_base"; + + @BeforeClass + public static void classLevelSetup() throws Exception { + Map overrides = new HashMap<>(); + overrides.put(MetastoreConf.ConfVars.EVENT_MESSAGE_FACTORY.getHiveName(), + GzipJSONMessageEncoder.class.getCanonicalName()); + overrides.put(HiveConf.ConfVars.HIVE_SCHEDULED_QUERIES_EXECUTOR_IDLE_SLEEP_TIME.varname, "1s"); + overrides.put(HiveConf.ConfVars.HIVE_SCHEDULED_QUERIES_EXECUTOR_PROGRESS_REPORT_INTERVAL.varname, + "1s"); + overrides.put(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname, "true"); + internalBeforeClassSetup(overrides, TestScheduledReplicationScenarios.class); + } + + static void internalBeforeClassSetup(Map overrides, + Class clazz) throws Exception { + + conf = new HiveConf(clazz); + conf.set("dfs.client.use.datanode.hostname", "true"); + conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); + MiniDFSCluster miniDFSCluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build(); + Map acidEnableConf = new HashMap() {{ + put("fs.defaultFS", miniDFSCluster.getFileSystem().getUri().toString()); + put("hive.support.concurrency", "true"); + put("hive.txn.manager", "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); + put("hive.metastore.client.capability.check", "false"); + put("hive.repl.bootstrap.dump.open.txn.timeout", "1s"); + put("hive.strict.checks.bucketing", "false"); + put("hive.mapred.mode", "nonstrict"); + put("mapred.input.dir.recursive", "true"); + put("hive.metastore.disallow.incompatible.col.type.changes", "false"); + put("hive.in.repl.test", "true"); + }}; + + acidEnableConf.putAll(overrides); + + primary = new WarehouseInstance(LOG, miniDFSCluster, acidEnableConf); + replica = new WarehouseInstance(LOG, miniDFSCluster, acidEnableConf); + Map overridesForHiveConf1 = new HashMap() {{ + put("fs.defaultFS", miniDFSCluster.getFileSystem().getUri().toString()); + put("hive.support.concurrency", "false"); + put("hive.txn.manager", "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager"); + put("hive.metastore.client.capability.check", "false"); + }}; + replicaNonAcid = new WarehouseInstance(LOG, miniDFSCluster, overridesForHiveConf1); + } + + @Before + public void setup() throws Throwable { + super.setup(); + } + + @After + public void tearDown() throws Throwable { + primary.run("drop database if exists " + primaryDbName + " cascade"); + replica.run("drop database if exists " + replicatedDbName + " cascade"); + replicaNonAcid.run("drop database if exists " + replicatedDbName + " cascade"); + primary.run("drop database if exists " + primaryDbName + "_extra cascade"); + } + + @Test + public void testAcidTablesBootstrapIncr() throws Throwable { + // Bootstrap + primary.run("use " + primaryDbName) + .run("create table t1 (id int) clustered by(id) into 3 buckets stored as orc " + + "tblproperties (\"transactional\"=\"true\")") + .run("insert into t1 values(1)") + .run("insert into t1 values(2)"); + try (ScheduledQueryExecutionService schqS = + ScheduledQueryExecutionService.startScheduledQueryExecutorService(primary.hiveConf)) { + int next = 0; + ReplDumpWork.injectNextDumpDirForTest(String.valueOf(next)); + primary.run("create scheduled query s1 every 10 minutes as repl dump " + primaryDbName); + primary.run("alter scheduled query s1 execute"); + Thread.sleep(6000); + Path dumpRoot = new Path(primary.hiveConf.getVar(HiveConf.ConfVars.REPLDIR), primaryDbName); + Path currdumpRoot = new Path(dumpRoot, String.valueOf(next)); + replica.load(replicatedDbName, currdumpRoot.toString()); + replica.run("use " + replicatedDbName) + .run("show tables like 't1'") + .verifyResult("t1") + .run("select id from t1 order by id") + .verifyResults(new String[]{"1", "2"}); + + // First incremental, after bootstrap + + primary.run("use " + primaryDbName) + .run("insert into t1 values(3)") + .run("insert into t1 values(4)"); + next++; + ReplDumpWork.injectNextDumpDirForTest(String.valueOf(next)); + primary.run("alter scheduled query s1 execute"); + Thread.sleep(20000); + Path incrdumpRoot = new Path(dumpRoot, String.valueOf(next)); + replica.load(replicatedDbName, incrdumpRoot.toString()); + replica.run("use " + replicatedDbName) + .run("show tables like 't1'") + .verifyResult("t1") + .run("select id from t1 order by id") + .verifyResults(new String[]{"1", "2", "3", "4"}) + .run("drop table t1"); + + + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + } finally { + primary.run("drop scheduled query s1"); + } + } + + @Test + public void testExternalTablesBootstrapIncr() throws Throwable { + List loadWithCause = ReplicationTestUtils.externalTableBasePathWithClause(REPLICA_EXTERNAL_BASE, replica); + // Bootstrap + primary.run("use " + primaryDbName) + .run("create external table t1 (id int)") + .run("insert into t1 values(1)") + .run("insert into t1 values(2)"); + try (ScheduledQueryExecutionService schqS = + ScheduledQueryExecutionService.startScheduledQueryExecutorService(primary.hiveConf)) { + int next = 0; + ReplDumpWork.injectNextDumpDirForTest(String.valueOf(next)); + primary.run("create scheduled query s1 every 10 minutes as repl dump " + primaryDbName); + primary.run("alter scheduled query s1 execute"); + Thread.sleep(6000); + Path dumpRoot = new Path(primary.hiveConf.getVar(HiveConf.ConfVars.REPLDIR), primaryDbName); + Path currdumpRoot = new Path(dumpRoot, String.valueOf(next)); + replica.load(replicatedDbName, currdumpRoot.toString(), loadWithCause); + replica.run("use " + replicatedDbName) + .run("show tables like 't1'") + .verifyResult("t1") + .run("select id from t1 order by id") + .verifyResults(new String[]{"1", "2"}); + + // First incremental, after bootstrap + + primary.run("use " + primaryDbName) + .run("insert into t1 values(3)") + .run("insert into t1 values(4)"); + next++; + ReplDumpWork.injectNextDumpDirForTest(String.valueOf(next)); + primary.run("alter scheduled query s1 execute"); + Thread.sleep(20000); + Path incrdumpRoot = new Path(dumpRoot, String.valueOf(next)); + replica.load(replicatedDbName, incrdumpRoot.toString(), loadWithCause); + replica.run("use " + replicatedDbName) + .run("show tables like 't1'") + .verifyResult("t1") + .run("select id from t1 order by id") + .verifyResults(new String[]{"1", "2", "3", "4"}) + .run("drop table t1"); + + + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + } finally { + primary.run("drop scheduled query s1"); + } + } +} diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java index 94eb1ffa87..44a3805dee 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java @@ -318,7 +318,7 @@ private String dumpLoadVerify(List tableNames, String lastReplicationId, // Take dump WarehouseInstance.Tuple dumpTuple = primary.run("use " + primaryDbName) - .dump(primaryDbName, lastReplicationId, withClauseList); + .dump(primaryDbName, withClauseList); // Load, if necessary changing configuration. diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java index 78f505b65c..312479a9ee 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestTableLevelReplicationScenarios.java @@ -153,7 +153,7 @@ private String replicateAndVerify(String replPolicy, String oldReplPolicy, Strin if (lastReplId == null) { replica.run("drop database if exists " + replicatedDbName + " cascade"); } - WarehouseInstance.Tuple tuple = primary.dump(replPolicy, oldReplPolicy, lastReplId, dumpWithClause); + WarehouseInstance.Tuple tuple = primary.dump(replPolicy, oldReplPolicy, dumpWithClause); if (bootstrappedTables != null) { verifyBootstrapDirInIncrementalDump(tuple.dumpLocation, bootstrappedTables); @@ -263,7 +263,7 @@ public void testBasicBootstrapWithIncludeAndExcludeList() throws Throwable { @Test public void testBasicIncrementalWithIncludeList() throws Throwable { WarehouseInstance.Tuple tupleBootstrap = primary.run("use " + primaryDbName) - .dump(primaryDbName, null); + .dump(primaryDbName); replica.load(replicatedDbName, tupleBootstrap.dumpLocation); String[] originalNonAcidTables = new String[] {"t1", "t2"}; @@ -282,7 +282,7 @@ public void testBasicIncrementalWithIncludeList() throws Throwable { @Test public void testBasicIncrementalWithIncludeAndExcludeList() throws Throwable { WarehouseInstance.Tuple tupleBootstrap = primary.run("use " + primaryDbName) - .dump(primaryDbName, null); + .dump(primaryDbName); replica.load(replicatedDbName, tupleBootstrap.dumpLocation); String[] originalTables = new String[] {"t1", "t11", "t2", "t3", "t111"}; @@ -326,7 +326,7 @@ public void testIncorrectTablePolicyInReplDump() throws Throwable { // Test incremental replication with invalid replication policies in REPLACE clause. String replPolicy = primaryDbName; WarehouseInstance.Tuple tupleBootstrap = primary.run("use " + primaryDbName) - .dump(primaryDbName, null); + .dump(primaryDbName); replica.load(replicatedDbName, tupleBootstrap.dumpLocation); String lastReplId = tupleBootstrap.lastReplicationId; for (String oldReplPolicy : invalidReplPolicies) { @@ -447,7 +447,7 @@ public void testBootstrapExternalTablesWithIncludeAndExcludeList() throws Throwa String replPolicy = primaryDbName + ".'(a[0-9]+)|(b2)'.'a1'"; String[] replicatedTables = new String[] {"a2", "b2"}; WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName) - .dump(replPolicy, null, dumpWithClause); + .dump(replPolicy, dumpWithClause); // the _external_tables_file info should be created as external tables are to be replicated. Assert.assertTrue(primary.miniDFSCluster.getFileSystem() @@ -486,7 +486,7 @@ public void testBootstrapExternalTablesIncrementalPhaseWithIncludeAndExcludeList dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'"); WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName) - .dump(replPolicy, lastReplId, dumpWithClause); + .dump(replPolicy, dumpWithClause); // the _external_tables_file info should be created as external tables are to be replicated. Assert.assertTrue(primary.miniDFSCluster.getFileSystem() @@ -635,7 +635,7 @@ public void testReplacePolicyOnBootstrapExternalTablesIncrementalPhase() throws dumpWithClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname + "'='true'", "'" + HiveConf.ConfVars.REPL_BOOTSTRAP_EXTERNAL_TABLES.varname + "'='true'"); WarehouseInstance.Tuple tuple = primary.run("use " + primaryDbName) - .dump(replPolicy, oldReplPolicy, lastReplId, dumpWithClause); + .dump(replPolicy, oldReplPolicy, dumpWithClause); // the _external_tables_file info should be created as external tables are to be replicated. Assert.assertTrue(primary.miniDFSCluster.getFileSystem() diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java index 43effeb64e..19aaabd946 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java @@ -262,22 +262,21 @@ WarehouseInstance runFailure(String command, int errorCode) throws Throwable { } } - Tuple dump(String dbName, String lastReplicationId, List withClauseOptions) + Tuple dump(String dbName, List withClauseOptions) throws Throwable { String dumpCommand = - "REPL DUMP " + dbName + (lastReplicationId == null ? "" : " FROM " + lastReplicationId); + "REPL DUMP " + dbName; if (!withClauseOptions.isEmpty()) { dumpCommand += " with (" + StringUtils.join(withClauseOptions, ",") + ")"; } return dump(dumpCommand); } - Tuple dump(String replPolicy, String oldReplPolicy, String lastReplicationId, List withClauseOptions) + Tuple dump(String replPolicy, String oldReplPolicy, List withClauseOptions) throws Throwable { String dumpCommand = "REPL DUMP " + replPolicy - + (oldReplPolicy == null ? "" : " REPLACE " + oldReplPolicy) - + (lastReplicationId == null ? "" : " FROM " + lastReplicationId); + + (oldReplPolicy == null ? "" : " REPLACE " + oldReplPolicy); if (!withClauseOptions.isEmpty()) { dumpCommand += " with (" + StringUtils.join(withClauseOptions, ",") + ")"; } @@ -292,13 +291,9 @@ Tuple dump(String dumpCommand) throws Throwable { return new Tuple(dumpLocation, lastDumpId); } - Tuple dump(String dbName, String lastReplicationId) throws Throwable { - return dump(dbName, lastReplicationId, Collections.emptyList()); - } - - WarehouseInstance dumpFailure(String dbName, String lastReplicationId) throws Throwable { + WarehouseInstance dumpFailure(String dbName) throws Throwable { String dumpCommand = - "REPL DUMP " + dbName + (lastReplicationId == null ? "" : " FROM " + lastReplicationId); + "REPL DUMP " + dbName; advanceDumpDir(); runFailure(dumpCommand); return this; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java index 977abb74cc..0109ed9f3f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.exec.repl; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; @@ -75,13 +76,14 @@ import java.io.IOException; import java.io.Serializable; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.UUID; import java.util.HashSet; import java.util.Set; +import java.util.Comparator; +import java.util.ArrayList; import java.util.concurrent.TimeUnit; import static org.apache.hadoop.hive.ql.exec.repl.ReplExternalTables.Writer; @@ -119,18 +121,21 @@ public String getName() { public int execute() { try { Hive hiveDb = getHive(); - Path dumpRoot = new Path(conf.getVar(HiveConf.ConfVars.REPLDIR), getNextDumpDir()); - DumpMetaData dmd = new DumpMetaData(dumpRoot, conf); + Path dumpRoot = new Path(conf.getVar(HiveConf.ConfVars.REPLDIR), work.dbNameOrPattern); + Path currentDumpPath = new Path(dumpRoot, getNextDumpDir()); + DumpMetaData dmd = new DumpMetaData(currentDumpPath, conf); // Initialize ReplChangeManager instance since we will require it to encode file URI. ReplChangeManager.getInstance(conf); Path cmRoot = new Path(conf.getVar(HiveConf.ConfVars.REPLCMDIR)); Long lastReplId; - if (work.isBootStrapDump()) { - lastReplId = bootStrapDump(dumpRoot, dmd, cmRoot, hiveDb); + if (!dumpRoot.getFileSystem(conf).exists(dumpRoot) + || dumpRoot.getFileSystem(conf).listStatus(dumpRoot).length == 0) { + lastReplId = bootStrapDump(currentDumpPath, dmd, cmRoot, hiveDb); } else { - lastReplId = incrementalDump(dumpRoot, dmd, cmRoot, hiveDb); + work.setEventFrom(getEventFromPreviousDumpMetadata(dumpRoot)); + lastReplId = incrementalDump(currentDumpPath, dmd, cmRoot, hiveDb); } - prepareReturnValues(Arrays.asList(dumpRoot.toUri().toString(), String.valueOf(lastReplId))); + prepareReturnValues(Arrays.asList(currentDumpPath.toUri().toString(), String.valueOf(lastReplId))); } catch (Exception e) { LOG.error("failed", e); setException(e); @@ -139,6 +144,22 @@ public int execute() { return 0; } + private Long getEventFromPreviousDumpMetadata(Path dumpRoot) throws IOException, SemanticException { + FileStatus[] statuses = dumpRoot.getFileSystem(conf).listStatus(dumpRoot); + if (statuses.length > 0) { + //sort based on last modified. Recent one is at the top + Arrays.sort(statuses, new Comparator() { + public int compare(FileStatus f1, FileStatus f2) { + return Long.compare(f2.getModificationTime(), f1.getModificationTime()); + } + }); + FileStatus recentDump = statuses[0]; + DumpMetaData dmd = new DumpMetaData(recentDump.getPath(), conf); + return dmd.getEventTo(); + } + return 0L; + } + private void prepareReturnValues(List values) throws SemanticException { LOG.debug("prepareReturnValues : " + dumpSchema); for (String s : values) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java index 7bae9ac66d..9b11bae6ba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpWork.java @@ -32,32 +32,25 @@ final ReplScope replScope; final ReplScope oldReplScope; final String dbNameOrPattern, astRepresentationForErrorMsg, resultTempPath; - final Long eventFrom; Long eventTo; - private Integer maxEventLimit; + Long eventFrom; static String testInjectDumpDir = null; + private Integer maxEventLimit; public static void injectNextDumpDirForTest(String dumpDir) { testInjectDumpDir = dumpDir; } public ReplDumpWork(ReplScope replScope, ReplScope oldReplScope, - Long eventFrom, Long eventTo, String astRepresentationForErrorMsg, Integer maxEventLimit, + String astRepresentationForErrorMsg, String resultTempPath) { this.replScope = replScope; this.oldReplScope = oldReplScope; this.dbNameOrPattern = replScope.getDbName(); - this.eventFrom = eventFrom; - this.eventTo = eventTo; this.astRepresentationForErrorMsg = astRepresentationForErrorMsg; - this.maxEventLimit = maxEventLimit; this.resultTempPath = resultTempPath; } - boolean isBootStrapDump() { - return eventFrom == null; - } - int maxEventLimit() throws Exception { if (eventTo < eventFrom) { throw new Exception("Invalid event ID input received in TO clause"); @@ -69,6 +62,10 @@ int maxEventLimit() throws Exception { return maxEventLimit; } + void setEventFrom(long eventId) { + eventFrom = eventId; + } + // Override any user specification that changes the last event to be dumped. void overrideLastEventToDump(Hive fromDb, long bootstrapLastId) throws Exception { // If we are bootstrapping ACID tables, we need to dump all the events upto the event id at @@ -77,7 +74,6 @@ void overrideLastEventToDump(Hive fromDb, long bootstrapLastId) throws Exception // bootstrampDump() for more details. if (bootstrapLastId > 0) { eventTo = bootstrapLastId; - maxEventLimit = null; LoggerFactory.getLogger(this.getClass()) .debug("eventTo restricted to event id : {} because of bootstrap of ACID tables", eventTo); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java index 810a4c5284..2243cb69b7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java @@ -56,25 +56,18 @@ import static org.apache.hadoop.hive.ql.exec.repl.ReplExternalTables.Reader; import static org.apache.hadoop.hive.ql.exec.repl.ExternalTableCopyTaskBuilder.DirCopyWork; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DBNAME; -import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_FROM; -import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_LIMIT; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPLACE; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_CONFIG; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_DUMP; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_LOAD; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_STATUS; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_TABLES; -import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TO; public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { // Replication Scope private ReplScope replScope = new ReplScope(); private ReplScope oldReplScope = null; - private Long eventFrom; - private Long eventTo; - private Integer maxEventLimit; - // Base path for REPL LOAD private String path; // Added conf member to set the REPL command specific config entries without affecting the configs @@ -207,27 +200,6 @@ private void initReplDump(ASTNode ast) throws HiveException { case TOK_REPLACE: setOldReplPolicy(currNode); break; - case TOK_FROM: - // TOK_FROM subtree - Tree fromNode = currNode; - eventFrom = Long.parseLong(PlanUtils.stripQuotes(fromNode.getChild(0).getText())); - - // Skip the first, which is always required - int fromChildIdx = 1; - while (fromChildIdx < fromNode.getChildCount()) { - if (fromNode.getChild(fromChildIdx).getType() == TOK_TO) { - eventTo = Long.parseLong(PlanUtils.stripQuotes(fromNode.getChild(fromChildIdx + 1).getText())); - // Skip the next child, since we already took care of it - fromChildIdx++; - } else if (fromNode.getChild(fromChildIdx).getType() == TOK_LIMIT) { - maxEventLimit = Integer.parseInt(PlanUtils.stripQuotes(fromNode.getChild(fromChildIdx + 1).getText())); - // Skip the next child, since we already took care of it - fromChildIdx++; - } - // move to the next child in FROM tree - fromChildIdx++; - } - break; default: throw new SemanticException("Unrecognized token " + currNode.getType() + " in REPL DUMP statement."); } @@ -263,10 +235,7 @@ private void analyzeReplDump(ASTNode ast) throws SemanticException { .get(new ReplDumpWork( replScope, oldReplScope, - eventFrom, - eventTo, ASTErrorUtils.getMsg(ErrorMsg.INVALID_PATH.getMsg(), ast), - maxEventLimit, ctx.getResFile().toUri().toString() ), conf); rootTasks.add(replDumpWorkTask); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestReplDumpTask.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestReplDumpTask.java index aacd29591d..9f90a364a1 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestReplDumpTask.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/TestReplDumpTask.java @@ -139,9 +139,7 @@ void dumpTable(String dbName, String tblName, String validTxnList, Path dbRoot, task.initialize(queryState, null, null, null); task.setWork( - new ReplDumpWork(replScope, null, - Long.MAX_VALUE, Long.MAX_VALUE, "", - Integer.MAX_VALUE, "") + new ReplDumpWork(replScope, null, "", "") ); try {