diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index 2e880c7a7f..31d7f39e02 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -3028,8 +3028,8 @@ public void testSkipTables() throws IOException { // Perform REPL-DUMP/LOAD advanceDumpDir(); run("REPL DUMP " + dbName + " FROM " + replDumpId, driver); - String incrementalDumpLocn = getResult(0,0,driver); - String incrementalDumpId = getResult(0,1,true,driver); + String incrementalDumpLocn = getResult(0, 0, driver); + String incrementalDumpId = getResult(0, 1, true, driver); LOG.info("Incremental-dump: Dumped to {} with id {}", incrementalDumpLocn, incrementalDumpId); run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror); printOutput(driverMirror); @@ -3043,9 +3043,29 @@ public void testSkipTables() throws IOException { // Perform REPL-DUMP/LOAD advanceDumpDir(); - run("REPL DUMP " + dbName + " FROM " + replDumpId, driver); - incrementalDumpLocn = getResult(0,0,driver); - incrementalDumpId = getResult(0,1,true,driver); + run("REPL DUMP " + dbName + " FROM " + incrementalDumpId, driver); + incrementalDumpLocn = getResult(0, 0, driver); + incrementalDumpId = getResult(0, 1, true, driver); + LOG.info("Incremental-dump: Dumped to {} with id {}", incrementalDumpLocn, incrementalDumpId); + run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror); + printOutput(driverMirror); + run("REPL LOAD " + dbName + "_dupe FROM '"+incrementalDumpLocn+"'", driverMirror); + verifyIfTableNotExist(dbName + "_dupe", "acid_table_incremental", metaStoreClientMirror); + + // Test adding a constraint + run("ALTER TABLE " + dbName + ".acid_table_incremental ADD CONSTRAINT key_pk PRIMARY KEY (key) DISABLE NOVALIDATE", driver); + try { + List pks = metaStoreClient.getPrimaryKeys(new PrimaryKeysRequest(dbName, "acid_table_incremental")); + assertEquals(pks.size(), 1); + } catch (TException te) { + assertNull(te); + } + + // Perform REPL-DUMP/LOAD + advanceDumpDir(); + run("REPL DUMP " + dbName + " FROM " + incrementalDumpId, driver); + incrementalDumpLocn = getResult(0, 0, driver); + incrementalDumpId = getResult(0, 1, true, driver); LOG.info("Incremental-dump: Dumped to {} with id {}", incrementalDumpLocn, incrementalDumpId); run("EXPLAIN REPL LOAD " + dbName + "_dupe FROM '" + incrementalDumpLocn + "'", driverMirror); printOutput(driverMirror); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java index 95d394ef69..715a4d3742 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java @@ -430,4 +430,19 @@ public static Boolean shouldExportTable(ReplicationSpec replicationSpec, Table t return true; } + + /** + * Verify if a table should be exported or not by talking to metastore to fetch table info. + * Return true when running into errors with metastore call. + */ + public static Boolean tryValidateShouldExportTable(Hive db, String dbName, String tableName, ReplicationSpec replicationSpec) { + try { + Table table = db.getTable(dbName, tableName); + return EximUtil.shouldExportTable(replicationSpec, table); + } catch (Exception e) { + // Swallow the exception + LOG.error("Failed to validate if the table should be exported or not", e); + } + return true; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java index ccd95fe798..d0cbd4a772 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events; import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; @@ -29,6 +31,12 @@ @Override public void handle(Context withinContext) throws Exception { LOG.info("Processing#{} ADD_FOREIGNKEY_MESSAGE message : {}", fromEventId(), event.getMessage()); + + if (!EximUtil.tryValidateShouldExportTable(withinContext.db, event.getDbName(), event.getTableName(), withinContext.replicationSpec)) + { + return; + } + DumpMetaData dmd = withinContext.createDmd(this); dmd.setPayload(event.getMessage()); dmd.write(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java index 7973eb3c24..aa7f4ef0a6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events; import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; @@ -29,6 +31,12 @@ @Override public void handle(Context withinContext) throws Exception { LOG.info("Processing#{} ADD_NOTNULLCONSTRAINT_MESSAGE message : {}", fromEventId(), event.getMessage()); + + if (!EximUtil.tryValidateShouldExportTable(withinContext.db, event.getDbName(), event.getTableName(), withinContext.replicationSpec)) + { + return; + } + DumpMetaData dmd = withinContext.createDmd(this); dmd.setPayload(event.getMessage()); dmd.write(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java index 3fb3e8b393..344fac9df9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events; import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; @@ -29,6 +31,12 @@ @Override public void handle(Context withinContext) throws Exception { LOG.info("Processing#{} ADD_PRIMARYKEY_MESSAGE message : {}", fromEventId(), event.getMessage()); + + if (!EximUtil.tryValidateShouldExportTable(withinContext.db, event.getDbName(), event.getTableName(), withinContext.replicationSpec)) + { + return; + } + DumpMetaData dmd = withinContext.createDmd(this); dmd.setPayload(event.getMessage()); dmd.write(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java index e4f541532a..4cc75a7d01 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events; import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; @@ -29,6 +31,12 @@ @Override public void handle(Context withinContext) throws Exception { LOG.info("Processing#{} ADD_UNIQUECONSTRAINT_MESSAGE message : {}", fromEventId(), event.getMessage()); + + if (!EximUtil.tryValidateShouldExportTable(withinContext.db, event.getDbName(), event.getTableName(), withinContext.replicationSpec)) + { + return; + } + DumpMetaData dmd = withinContext.createDmd(this); dmd.setPayload(event.getMessage()); dmd.write();