diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index 0aee012..21f09ae 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -386,7 +386,6 @@ public void testBootstrapWithConcurrentDropTable() throws IOException {
run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)");
verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2);
-
advanceDumpDir();
BehaviourInjection
ptnedTableNuller = new BehaviourInjection(){
@@ -414,12 +413,14 @@ public Table apply(@Nullable Table table) {
LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
- // The ptned table should miss in target as the table was marked cisrtually as dropped
+ // The ptned table should miss in target as the table was marked virtually as dropped
verifyRun("SELECT * from " + dbName + "_dupe.unptned", unptn_data);
verifyFail("SELECT a from " + dbName + "_dupe.ptned WHERE b=1");
+ verifyIfTableNotExist(dbName + "_dupe", "ptned");
// Verify if Drop table on a non-existing table is idempotent
run("DROP TABLE " + dbName + ".ptned");
+ verifyIfTableNotExist(dbName, "ptned");
advanceDumpDir();
run("REPL DUMP " + dbName + " FROM " + replDumpId);
@@ -429,10 +430,82 @@ public Table apply(@Nullable Table table) {
assert(run("REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'", true));
verifyRun("SELECT * from " + dbName + "_dupe.unptned", unptn_data);
+ verifyIfTableNotExist(dbName + "_dupe", "ptned");
verifyFail("SELECT a from " + dbName + "_dupe.ptned WHERE b=1");
}
@Test
+ public void testBootstrapWithConcurrentDropPartition() throws IOException {
+ String name = testName.getMethodName();
+ String dbName = createDB(name);
+ run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE");
+
+ String[] ptn_data_1 = new String[]{ "thirteen", "fourteen", "fifteen"};
+ String[] ptn_data_2 = new String[]{ "fifteen", "sixteen", "seventeen"};
+ String[] empty = new String[]{};
+
+ String ptn_locn_1 = new Path(TEST_PATH, name + "_ptn1").toUri().getPath();
+ String ptn_locn_2 = new Path(TEST_PATH, name + "_ptn2").toUri().getPath();
+
+ createTestDataFile(ptn_locn_1, ptn_data_1);
+ createTestDataFile(ptn_locn_2, ptn_data_2);
+
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=1)");
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1);
+ run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)");
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2);
+
+ advanceDumpDir();
+
+ BehaviourInjection, List> listPartitionNamesNuller
+ = new BehaviourInjection, List>(){
+ @Nullable
+ @Override
+ public List apply(@Nullable List partitions) {
+ injectionPathCalled = true;
+ return new ArrayList();
+ }
+ };
+ InjectableBehaviourObjectStore.setListPartitionNamesBehaviour(listPartitionNamesNuller);
+
+ // None of the partitions will be dumped as the partitions list was empty
+ run("REPL DUMP " + dbName);
+ listPartitionNamesNuller.assertInjectionsPerformed(true, false);
+ InjectableBehaviourObjectStore.resetListPartitionNamesBehaviour(); // reset the behaviour
+
+ String replDumpLocn = getResult(0, 0);
+ String replDumpId = getResult(0, 1, true);
+ LOG.info("Bootstrap-Dump: Dumped to {} with id {}", replDumpLocn, replDumpId);
+ run("REPL LOAD " + dbName + "_dupe FROM '" + replDumpLocn + "'");
+
+ // All partitions should miss in target as it was marked virtually as dropped
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", empty);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", empty);
+ verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("1")));
+ verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("2")));
+
+ // Verify if drop partition on a non-existing partition is idempotent and just a noop.
+ run("ALTER TABLE " + dbName + ".ptned DROP PARTITION (b=1)");
+ run("ALTER TABLE " + dbName + ".ptned DROP PARTITION (b=2)");
+ verifyIfPartitionNotExist(dbName, "ptned", new ArrayList<>(Arrays.asList("1")));
+ verifyIfPartitionNotExist(dbName, "ptned", new ArrayList<>(Arrays.asList("2")));
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", empty);
+ verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", empty);
+
+ advanceDumpDir();
+ run("REPL DUMP " + dbName + " FROM " + replDumpId);
+ String postDropReplDumpLocn = getResult(0,0);
+ String postDropReplDumpId = getResult(0,1,true);
+ LOG.info("Dumped to {} with id {}->{}", postDropReplDumpLocn, replDumpId, postDropReplDumpId);
+ assert(run("REPL LOAD " + dbName + "_dupe FROM '" + postDropReplDumpLocn + "'", true));
+
+ verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("1")));
+ verifyIfPartitionNotExist(dbName + "_dupe", "ptned", new ArrayList<>(Arrays.asList("2")));
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=1", empty);
+ verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=2", empty);
+ }
+
+ @Test
public void testIncrementalAdds() throws IOException {
String name = testName.getMethodName();
String dbName = createDB(name);
@@ -599,30 +672,14 @@ public void testDrops() throws IOException {
// not existing, and thus, throwing a NoSuchObjectException, or returning nulls
// or select * returning empty, depending on what we're testing.
- Exception e = null;
- try {
- Table tbl = metaStoreClient.getTable(dbName + "_dupe", "unptned");
- assertNull(tbl);
- } catch (TException te) {
- e = te;
- }
- assertNotNull(e);
- assertEquals(NoSuchObjectException.class, e.getClass());
+ verifyIfTableNotExist(dbName + "_dupe", "unptned");
verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b='2'", empty);
verifyRun("SELECT a from " + dbName + "_dupe.ptned", ptn_data_1);
verifyRun("SELECT a from " + dbName + "_dupe.ptned3 WHERE b=1", empty);
verifyRun("SELECT a from " + dbName + "_dupe.ptned3", ptn_data_2);
- Exception e2 = null;
- try {
- Table tbl = metaStoreClient.getTable(dbName+"_dupe","ptned2");
- assertNull(tbl);
- } catch (TException te) {
- e2 = te;
- }
- assertNotNull(e2);
- assertEquals(NoSuchObjectException.class, e.getClass());
+ verifyIfTableNotExist(dbName + "_dupe", "ptned2");
}
@Test
@@ -732,15 +789,7 @@ public void testDropsWithCM() throws IOException {
run("SELECT a from " + dbName + "_dupe.ptned");
verifyResults(ptn_data_1);
- Exception e2 = null;
- try {
- Table tbl = metaStoreClient.getTable(dbName+"_dupe","ptned2");
- assertNull(tbl);
- } catch (TException te) {
- e2 = te;
- }
- assertNotNull(e2);
- assertEquals(NoSuchObjectException.class, e.getClass());
+ verifyIfTableNotExist(dbName +"_dupe", "ptned2");
run("SELECT a from " + dbName + "_dupe.unptned_copy");
verifyResults(unptn_data);
@@ -874,15 +923,7 @@ public void testAlters() throws IOException {
// Replication done, we now do the following verifications:
// verify that unpartitioned table rename succeeded.
- Exception e = null;
- try {
- Table tbl = metaStoreClient.getTable(dbName + "_dupe" , "unptned");
- assertNull(tbl);
- } catch (TException te) {
- e = te;
- }
- assertNotNull(e);
- assertEquals(NoSuchObjectException.class, e.getClass());
+ verifyIfTableNotExist(dbName + "_dupe", "unptned");
verifyRun("SELECT * from " + dbName + "_dupe.unptned_rn", unptn_data);
// verify that partition rename succeded.
@@ -898,15 +939,7 @@ public void testAlters() throws IOException {
verifyRun("SELECT a from " + dbName + "_dupe.ptned WHERE b=22", ptn_data_2);
// verify that ptned table rename succeded.
- Exception e2 = null;
- try {
- Table tbl = metaStoreClient.getTable(dbName + "_dupe" , "ptned2");
- assertNull(tbl);
- } catch (TException te) {
- e2 = te;
- }
- assertNotNull(e2);
- assertEquals(NoSuchObjectException.class, e.getClass());
+ verifyIfTableNotExist(dbName + "_dupe", "ptned2");
verifyRun("SELECT a from " + dbName + "_dupe.ptned2_rn WHERE b=2", ptn_data_2);
// verify that ptned table property set worked
@@ -1970,6 +2003,50 @@ private void printOutput() throws IOException {
}
}
+ private void verifyIfTableNotExist(String dbName, String tableName){
+ Exception e = null;
+ try {
+ Table tbl = metaStoreClient.getTable(dbName, tableName);
+ assertNull(tbl);
+ } catch (TException te) {
+ e = te;
+ }
+ assertNotNull(e);
+ assertEquals(NoSuchObjectException.class, e.getClass());
+ }
+
+ private void verifyIfTableExist(String dbName, String tableName){
+ Exception e = null;
+ try {
+ Table tbl = metaStoreClient.getTable(dbName, tableName);
+ assertNotNull(tbl);
+ } catch (TException te) {
+ assert(false);
+ }
+ }
+
+ private void verifyIfPartitionNotExist(String dbName, String tableName, List partValues){
+ Exception e = null;
+ try {
+ Partition ptn = metaStoreClient.getPartition(dbName, tableName, partValues);
+ assertNull(ptn);
+ } catch (TException te) {
+ e = te;
+ }
+ assertNotNull(e);
+ assertEquals(NoSuchObjectException.class, e.getClass());
+ }
+
+ private void verifyIfPartitionExist(String dbName, String tableName, List partValues){
+ Exception e = null;
+ try {
+ Partition ptn = metaStoreClient.getPartition(dbName, tableName, partValues);
+ assertNotNull(ptn);
+ } catch (TException te) {
+ assert(false);
+ }
+ }
+
private void verifySetup(String cmd, String[] data) throws IOException {
if (VERIFY_SETUP_STEPS){
run(cmd);
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
index a832c78..ed6d4be 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/InjectableBehaviourObjectStore.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.metastore;
+import java.util.List;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -50,6 +51,8 @@ public void assertInjectionsPerformed(
private static com.google.common.base.Function getTableModifier =
com.google.common.base.Functions.identity();
+ private static com.google.common.base.Function, List> listPartitionNamesModifier =
+ com.google.common.base.Functions.identity();
public static void setGetTableBehaviour(com.google.common.base.Function modifier){
getTableModifier = (modifier == null)? com.google.common.base.Functions.identity() : modifier;
@@ -63,8 +66,25 @@ public static void resetGetTableBehaviour(){
return getTableModifier;
}
+ public static void setListPartitionNamesBehaviour(com.google.common.base.Function, List> modifier){
+ listPartitionNamesModifier = (modifier == null)? com.google.common.base.Functions.identity() : modifier;
+ }
+
+ public static void resetListPartitionNamesBehaviour(){
+ setListPartitionNamesBehaviour(null);
+ }
+
+ public static com.google.common.base.Function, List> getListPartitionNamesBehaviour() {
+ return listPartitionNamesModifier;
+ }
+
@Override
public Table getTable(String dbName, String tableName) throws MetaException {
return getTableModifier.apply(super.getTable(dbName, tableName));
}
+
+ @Override
+ public List listPartitionNames(String dbName, String tableName, short max) throws MetaException {
+ return listPartitionNamesModifier.apply(super.listPartitionNames(dbName, tableName, max));
+ }
}