diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index de355ad104..0eff339543 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -540,6 +540,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal REPL_DUMP_METADATA_ONLY("hive.repl.dump.metadata.only", false, "Indicates whether replication dump only metadata information or data + metadata. \n" + "This config makes hive.repl.include.external.tables config ineffective."), + REPL_INCLUDE_MATERIALIZED_VIEWS("hive.repl.include.materialized.views", false, + "Indicates whether replication of materialized views is enabled."), REPL_DUMP_SKIP_IMMUTABLE_DATA_COPY("hive.repl.dump.skip.immutable.data.copy", false, "Indicates whether replication dump can skip copyTask and refer to \n" + " original path instead. This would retain all table and partition meta"), diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index 15adac0ee5..8df9156f48 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -136,7 +136,7 @@ protected static final Logger LOG = LoggerFactory.getLogger(TestReplicationScenarios.class); private ArrayList lastResults; - private final boolean VERIFY_SETUP_STEPS = false; + private boolean verifySetupSteps = false; // if verifySetup is set to true, all the test setup we do will perform additional // verifications as well, which is useful to verify that our setup occurred // correctly when developing and debugging tests. These verifications, however @@ -1349,7 +1349,7 @@ public void testTableAlters() throws IOException { String testKey = "blah"; String testVal = "foo"; run("ALTER TABLE " + dbName + ".unptned2 SET TBLPROPERTIES ('" + testKey + "' = '" + testVal + "')", driver); - if (VERIFY_SETUP_STEPS){ + if (verifySetupSteps){ try { Table unptn2 = metaStoreClient.getTable(dbName,"unptned2"); assertTrue(unptn2.getParameters().containsKey(testKey)); @@ -1366,7 +1366,7 @@ public void testTableAlters() throws IOException { // alter partitioned table set table property run("ALTER TABLE " + dbName + ".ptned SET TBLPROPERTIES ('" + testKey + "' = '" + testVal + "')", driver); - if (VERIFY_SETUP_STEPS){ + if (verifySetupSteps){ try { Table ptned = metaStoreClient.getTable(dbName,"ptned"); assertTrue(ptned.getParameters().containsKey(testKey)); @@ -2609,6 +2609,109 @@ public void testViewsReplication() throws IOException { verifyIfTableNotExist(replDbName, "virtual_view", metaStoreClientMirror); } + @Test + public void testMaterializedViewsReplication() throws Exception { + boolean verifySetup_tmp = verifySetupSteps; + verifySetupSteps = true; + String testName = "materializedviewsreplication"; + String testName2 = testName + "2"; + String dbName = createDB(testName, driver); + String dbName2 = createDB(testName2, driver); //for creating multi-db materialized view + String replDbName = dbName + "_dupe"; + + run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver); + run("CREATE TABLE " + dbName2 + ".unptned(a string) STORED AS TEXTFILE", driver); + run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver); + + String[] unptn_data = new String[]{ "eleven", "twelve" }; + String[] ptn_data_1 = new String[]{ "thirteen", "fourteen", "fifteen"}; + String[] ptn_data_2 = new String[]{ "fifteen", "sixteen", "seventeen"}; + String[] empty = new String[]{}; + + String unptn_locn = new Path(TEST_PATH, testName + "_unptn").toUri().getPath(); + String ptn_locn_1 = new Path(TEST_PATH, testName + "_ptn1").toUri().getPath(); + String ptn_locn_2 = new Path(TEST_PATH, testName + "_ptn2").toUri().getPath(); + + createTestDataFile(unptn_locn, unptn_data); + createTestDataFile(ptn_locn_1, ptn_data_1); + createTestDataFile(ptn_locn_2, ptn_data_2); + + verifySetup("SELECT a from " + dbName + ".ptned", empty, driver); + verifySetup("SELECT * from " + dbName + ".unptned", empty, driver); + verifySetup("SELECT * from " + dbName2 + ".unptned", empty, driver); + + run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned", driver); + run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName2 + ".unptned", driver); + verifySetup("SELECT * from " + dbName + ".unptned", unptn_data, driver); + verifySetup("SELECT * from " + dbName2 + ".unptned", unptn_data, driver); + + run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=1)", driver); + verifySetup("SELECT a from " + dbName + ".ptned WHERE b=1", ptn_data_1, driver); + run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)", driver); + verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2, driver); + + + run("CREATE MATERIALIZED VIEW " + dbName + ".mat_view_boot disable rewrite stored as textfile AS SELECT a FROM " + dbName + ".ptned where b=1", driver); + verifySetup("SELECT a from " + dbName + ".mat_view_boot", ptn_data_1, driver); + + run("CREATE MATERIALIZED VIEW " + dbName + ".mat_view_boot2 disable rewrite stored as textfile AS SELECT t1.a FROM " + dbName + ".unptned as t1 join " + dbName2 + ".unptned as t2 on t1.a = t2.a", driver); + verifySetup("SELECT a from " + dbName + ".mat_view_boot2", unptn_data, driver); + + Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); + + verifyRun("SELECT * from " + replDbName + ".unptned", unptn_data, driverMirror); + verifyRun("SELECT a from " + replDbName + ".ptned where b=1", ptn_data_1, driverMirror); + + //verify source MVs are not on replica + verifyIfTableNotExist(replDbName, "mat_view_boot", metaStoreClientMirror); + verifyIfTableNotExist(replDbName, "mat_view_boot2", metaStoreClientMirror); + + //test alter materialized view with rename + run("ALTER TABLE " + dbName + ".mat_view_boot RENAME TO " + dbName + ".mat_view_rename", driver); + + //verify rename, i.e. new MV exists and old MV does not exist + verifyIfTableNotExist(dbName, "mat_view_boot", metaStoreClient); + verifyIfTableExist(dbName, "mat_view_rename", metaStoreClient); + //verifySetup("SELECT a from " + dbName + ".mat_view_rename", ptn_data_1, driver); + + // Perform REPL-DUMP/LOAD + Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDbName); + + //verify source MVs are not on replica + verifyIfTableNotExist(replDbName, "mat_view_rename", metaStoreClientMirror); + verifyIfTableNotExist(replDbName, "mat_view_boot2", metaStoreClientMirror); + + //test alter materialized view rebuild + run("ALTER MATERIALIZED VIEW " + dbName + ".mat_view_boot2 REBUILD" , driver); + verifyRun("SELECT a from " + dbName + ".mat_view_boot2", unptn_data, driver); + + // Perform REPL-DUMP/LOAD + incrementalDump = incrementalLoadAndVerify(dbName, replDbName); + + //verify source MVs are not on replica + verifyIfTableNotExist(replDbName, "mat_view_rename", metaStoreClientMirror); + verifyIfTableNotExist(replDbName, "mat_view_boot2", metaStoreClientMirror); + + run("CREATE MATERIALIZED VIEW " + dbName + ".mat_view_inc disable rewrite stored as textfile AS SELECT a FROM " + dbName + ".ptned where b=2", driver); + verifySetup("SELECT a from " + dbName + ".mat_view_inc", ptn_data_2, driver); + + run("CREATE MATERIALIZED VIEW " + dbName + ".mat_view_inc2 disable rewrite stored as textfile AS SELECT t1.a FROM " + dbName + ".unptned as t1 join " + dbName2 + ".unptned as t2 on t1.a = t2.a", driver); + verifySetup("SELECT a from " + dbName + ".mat_view_inc2", unptn_data, driver); + + // Perform REPL-DUMP/LOAD + incrementalDump = incrementalLoadAndVerify(dbName, replDbName); + + //verify source MVs are not on replica + verifyIfTableNotExist(replDbName, "mat_view_rename", metaStoreClientMirror); + verifyIfTableNotExist(replDbName, "mat_view_boot2", metaStoreClientMirror); + verifyIfTableNotExist(replDbName, "mat_view_inc", metaStoreClientMirror); + verifyIfTableNotExist(replDbName, "mat_view_inc2", metaStoreClientMirror); + + verifySetupSteps = verifySetup_tmp; + } + + + @Test public void testExchangePartition() throws IOException { String testName = "exchangePartition"; @@ -3921,7 +4024,7 @@ private void verifyIfPartitionExist(String dbName, String tableName, List tabletuple = new HiveWrapper(hiveDb, dbName).table(tblName, conf); + if (tabletuple != null && TableType.MATERIALIZED_VIEW.equals(tabletuple.object.getTableType())){ + LOG.debug("Attempt to dump materialized view : " + tblName); + continue; + } + } catch (InvalidTableException te) { + LOG.debug(te.getMessage()); + } + } + } + Path evRoot = new Path(dumpRoot, String.valueOf(lastReplId)); dumpEvent(ev, evRoot, dumpRoot, cmRoot, hiveDb); Utils.writeOutput(String.valueOf(lastReplId), ackFile, conf); @@ -828,12 +852,19 @@ Long bootStrapDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot, Hive hiveDb) Exception caught = null; try (Writer writer = new Writer(dbRoot, conf)) { for (String tblName : Utils.matchesTbl(hiveDb, dbName, work.replScope)) { - LOG.debug("Dumping table: " + tblName + " to db root " + dbRoot.toUri()); Table table = null; - try { HiveWrapper.Tuple tableTuple = new HiveWrapper(hiveDb, dbName).table(tblName, conf); table = tableTuple != null ? tableTuple.object : null; + + //disable materialized-view replication if not configured + if(tableTuple != null && !isMaterializedViewsReplEnabled() + && TableType.MATERIALIZED_VIEW.equals(tableTuple.object.getTableType())){ + LOG.debug("Attempt to dump materialized view : " + tblName); + continue; + } + + LOG.debug("Dumping table: " + tblName + " to db root " + dbRoot.toUri()); if (shouldDumpExternalTableLocation() && TableType.EXTERNAL_TABLE.equals(tableTuple.object.getTableType())) { LOG.debug("Adding table {} to external tables list", tblName);