diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 58615e1391..16581727c9 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -545,6 +545,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal REPL_DUMP_METADATA_ONLY("hive.repl.dump.metadata.only", false, "Indicates whether replication dump only metadata information or data + metadata. \n" + "This config makes hive.repl.include.external.tables config ineffective."), + REPL_MATERIALIZED_VIEWS_ENABLED("hive.repl.materialized.views.enabled", false, + "Indicates whether replication of materialized views is enabled."), REPL_DUMP_SKIP_IMMUTABLE_DATA_COPY("hive.repl.dump.skip.immutable.data.copy", false, "Indicates whether replication dump can skip copyTask and refer to \n" + " original path instead. This would retain all table and partition meta"), diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index b8e91dda2f..c0026f2f0b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -137,7 +137,7 @@ protected static final Logger LOG = LoggerFactory.getLogger(TestReplicationScenarios.class); private ArrayList lastResults; - private final boolean VERIFY_SETUP_STEPS = false; + private boolean VERIFY_SETUP_STEPS = false; // if verifySetup is set to true, all the test setup we do will perform additional // verifications as well, which is useful to verify that our setup occurred // correctly when developing and debugging tests. These verifications, however @@ -2533,11 +2533,16 @@ public void testRenamePartitionedTableAcrossDatabases() throws IOException { @Test public void testViewsReplication() throws IOException { + boolean verify_setup_tmp = VERIFY_SETUP_STEPS; + VERIFY_SETUP_STEPS = true; // set flag to true for this test String testName = "viewsReplication"; + String testName2 = testName + "2"; String dbName = createDB(testName, driver); + String dbName2 = createDB(testName2, driver); //for creating multi-db materialized view String replDbName = dbName + "_dupe"; run("CREATE TABLE " + dbName + ".unptned(a string) STORED AS TEXTFILE", driver); + run("CREATE TABLE " + dbName2 + ".unptned(a string) STORED AS TEXTFILE", driver); run("CREATE TABLE " + dbName + ".ptned(a string) partitioned by (b int) STORED AS TEXTFILE", driver); run("CREATE VIEW " + dbName + ".virtual_view AS SELECT * FROM " + dbName + ".unptned", driver); @@ -2556,10 +2561,13 @@ public void testViewsReplication() throws IOException { verifySetup("SELECT a from " + dbName + ".ptned", empty, driver); verifySetup("SELECT * from " + dbName + ".unptned", empty, driver); + verifySetup("SELECT * from " + dbName2 + ".unptned", empty, driver); verifySetup("SELECT * from " + dbName + ".virtual_view", empty, driver); run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName + ".unptned", driver); + run("LOAD DATA LOCAL INPATH '" + unptn_locn + "' OVERWRITE INTO TABLE " + dbName2 + ".unptned", driver); verifySetup("SELECT * from " + dbName + ".unptned", unptn_data, driver); + verifySetup("SELECT * from " + dbName2 + ".unptned", unptn_data, driver); verifySetup("SELECT * from " + dbName + ".virtual_view", unptn_data, driver); run("LOAD DATA LOCAL INPATH '" + ptn_locn_1 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=1)", driver); @@ -2567,10 +2575,11 @@ public void testViewsReplication() throws IOException { run("LOAD DATA LOCAL INPATH '" + ptn_locn_2 + "' OVERWRITE INTO TABLE " + dbName + ".ptned PARTITION(b=2)", driver); verifySetup("SELECT a from " + dbName + ".ptned WHERE b=2", ptn_data_2, driver); + // TODO: This does not work because materialized views need the creation metadata // to be updated in case tables used were replicated to a different database. - //run("CREATE MATERIALIZED VIEW " + dbName + ".mat_view AS SELECT a FROM " + dbName + ".ptned where b=1", driver); - //verifySetup("SELECT a from " + dbName + ".mat_view", ptn_data_1, driver); + run("CREATE MATERIALIZED VIEW " + dbName + ".mat_view disable rewrite stored as textfile AS SELECT a FROM " + dbName + ".ptned where b=1", driver); + verifySetup("SELECT a from " + dbName + ".mat_view", ptn_data_1, driver); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName); @@ -2586,9 +2595,12 @@ public void testViewsReplication() throws IOException { //run("CREATE MATERIALIZED VIEW " + dbName + ".mat_view2 AS SELECT * FROM " + dbName + ".unptned", driver); //verifySetup("SELECT * from " + dbName + ".mat_view2", unptn_data, driver); + run("CREATE MATERIALIZED VIEW " + dbName + ".mat_view2 disable rewrite stored as textfile AS SELECT t1.a FROM " + dbName + ".unptned as t1 join " + dbName2 + ".unptned as t2 on t1.a = t2.a", driver); + verifySetup("SELECT a from " + dbName + ".mat_view2", unptn_data, driver); // Perform REPL-DUMP/LOAD Tuple incrementalDump = incrementalLoadAndVerify(dbName, replDbName); + incrementalDump = incrementalLoadAndVerify(dbName, replDbName); verifyRun("SELECT * from " + replDbName + ".unptned", unptn_data, driverMirror); verifyRun("SELECT a from " + replDbName + ".ptned where b=1", ptn_data_1, driverMirror); @@ -2622,6 +2634,7 @@ public void testViewsReplication() throws IOException { // Perform REPL-DUMP/LOAD incrementalLoadAndVerify(dbName, replDbName); verifyIfTableNotExist(replDbName, "virtual_view", metaStoreClientMirror); + VERIFY_SETUP_STEPS = verify_setup_tmp; } @Test diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java index dc8d7903fc..3466156c04 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java @@ -416,6 +416,13 @@ private boolean shouldBootstrapDumpExternalTable(String tableName) { || !ReplUtils.tableIncludedInReplScope(work.oldReplScope, tableName)); } + /** + * Decide whether to dump materialized views. + */ + private boolean isMaterializedViewsReplEnabled() { + return conf.getBoolVar(HiveConf.ConfVars.REPL_MATERIALIZED_VIEWS_ENABLED); + } + /** * Decide whether to dump ACID tables. * @param tableName - Name of ACID table to be replicated @@ -539,6 +546,24 @@ private Long incrementalDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot, Hive if (ev.getEventId() <= resumeFrom) { continue; } + + //disable materialized-view replication if configured + String tblName = null; + tblName = ev.getTableName(); + if(tblName != null) { + try { + Table table = null; + HiveWrapper.Tuple TableTuple = new HiveWrapper(hiveDb, dbName).table(tblName, conf); + table = TableTuple != null ? TableTuple.object : null; + if (TableType.MATERIALIZED_VIEW.equals(TableTuple.object.getTableType()) && !isMaterializedViewsReplEnabled()) { + LOG.debug("Attempt to dump materialized view : " + tblName); + continue; + } + } catch (InvalidTableException te) { + LOG.debug(te.getMessage()); + } + } + Path evRoot = new Path(dumpRoot, String.valueOf(lastReplId)); dumpEvent(ev, evRoot, dumpRoot, cmRoot, hiveDb); Utils.writeOutput(String.valueOf(lastReplId), ackFile, conf); @@ -829,12 +854,15 @@ Long bootStrapDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot, Hive hiveDb) Exception caught = null; try (Writer writer = new Writer(dbRoot, conf)) { for (String tblName : Utils.matchesTbl(hiveDb, dbName, work.replScope)) { - LOG.debug("Dumping table: " + tblName + " to db root " + dbRoot.toUri()); Table table = null; - try { HiveWrapper.Tuple
tableTuple = new HiveWrapper(hiveDb, dbName).table(tblName, conf); table = tableTuple != null ? tableTuple.object : null; + if(TableType.MATERIALIZED_VIEW.equals(tableTuple.object.getTableType()) && !isMaterializedViewsReplEnabled()){ + LOG.debug("Attempt to dump materialized view : " + tblName); + continue; + } + LOG.debug("Dumping table: " + tblName + " to db root " + dbRoot.toUri()); if (shouldDumpExternalTableLocation() && TableType.EXTERNAL_TABLE.equals(tableTuple.object.getTableType())) { LOG.debug("Adding table {} to external tables list", tblName);