diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 928221d14f..3ff938c4f6 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -504,8 +504,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Turn on ChangeManager, so delete files will go to cmrootdir."), REPLCMDIR("hive.repl.cmrootdir","/user/${system:user.name}/cmroot/", "Root dir for ChangeManager, used for deleted files."), - REPLCMRETIAN("hive.repl.cm.retain","24h", - new TimeValidator(TimeUnit.HOURS), + REPLCMRETIAN("hive.repl.cm.retain","7d", + new TimeValidator(TimeUnit.DAYS), "Time to retain removed files in cmrootdir."), REPLCMENCRYPTEDDIR("hive.repl.cm.encryptionzone.rootdir", ".cmroot", "Root dir for ChangeManager if encryption zones are enabled, used for deleted files."), @@ -564,7 +564,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal REPL_ADD_RAW_RESERVED_NAMESPACE("hive.repl.add.raw.reserved.namespace", false, "For TDE with same encryption keys on source and target, allow Distcp super user to access \n" + "the raw bytes from filesystem without decrypting on source and then encrypting on target."), - REPL_INCLUDE_EXTERNAL_TABLES("hive.repl.include.external.tables", false, + REPL_INCLUDE_EXTERNAL_TABLES("hive.repl.include.external.tables", true, "Indicates if repl dump should include information about external tables. It should be \n" + "used in conjunction with 'hive.repl.dump.metadata.only' set to false. if 'hive.repl.dump.metadata.only' \n" + " is set to true then this config parameter has no effect as external table meta data is flushed \n" diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java index 8f1bc40407..e46a52045b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java @@ -1316,16 +1316,16 @@ public void testClearerEncrypted() throws Exception { fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part11.getName(), fileChksum11, ReplChangeManager.getInstance(conf).getCmRoot(part11).toString()), - now - 86400*1000*2, now - 86400*1000*2); + now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2); fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part21.getName(), fileChksum21, ReplChangeManager.getInstance(conf).getCmRoot(part21).toString()), - now - 86400*1000*2, now - 86400*1000*2); + now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2); fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part31.getName(), fileChksum31, ReplChangeManager.getInstance(conf).getCmRoot(part31).toString()), - now - 86400*1000*2, now - 86400*1000*2); + now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2); fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part32.getName(), fileChksum32, ReplChangeManager.getInstance(conf).getCmRoot(part32).toString()), - now - 86400*1000*2, now - 86400*1000*2); + now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2); ReplChangeManager.scheduleCMClearer(hiveConfCmClearer); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java index d3891cf9bb..9b728b8693 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestReplChangeManager.java @@ -310,13 +310,13 @@ public void testClearer() throws Exception { assertTrue(fs.exists(ReplChangeManager.getCMPath(hiveConf, part32.getName(), fileChksum32, cmroot.toString()))); fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part11.getName(), fileChksum11, cmroot.toString()), - now - 86400*1000*2, now - 86400*1000*2); + now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2); fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part21.getName(), fileChksum21, cmroot.toString()), - now - 86400*1000*2, now - 86400*1000*2); + now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2); fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part31.getName(), fileChksum31, cmroot.toString()), - now - 86400*1000*2, now - 86400*1000*2); + now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2); fs.setTimes(ReplChangeManager.getCMPath(hiveConf, part32.getName(), fileChksum32, cmroot.toString()), - now - 86400*1000*2, now - 86400*1000*2); + now - 7 * 86400 * 1000 * 2, now - 7 * 86400 * 1000 * 2); ReplChangeManager.scheduleCMClearer(hiveConf); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java index 6f5c8a96fc..3f4cacfe58 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java @@ -54,6 +54,7 @@ public static void classLevelSetup() throws Exception { Configuration conf = new Configuration(); conf.set("dfs.client.use.datanode.hostname", "true"); conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); + conf.set("hive.repl.include.external.tables", "false"); MiniDFSCluster miniDFSCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build(); HashMap overridesForHiveConf = new HashMap() {{ diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java index 2953c2214e..b4666ba3a1 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java @@ -1000,7 +1000,7 @@ public void testIncrementalDumpMultiIteration() throws Throwable { "clustered by(key) into 2 buckets stored as orc") .run("create table table4 (i int, j int)") .run("insert into table4 values (1,2)") - .dump(primaryDbName, Collections.emptyList()); + .dump(primaryDbName, Collections.singletonList("'hive.repl.include.external.tables'='false'")); String hiveDumpDir = incremental.dumpLocation + File.separator + ReplUtils.REPL_HIVE_BASE_DIR; Path path = new Path(hiveDumpDir); @@ -1009,7 +1009,7 @@ public void testIncrementalDumpMultiIteration() throws Throwable { int numEvents = fileStatus.length - 3; //for _metadata, _finished_dump and _events_dump replica.load(replicatedDbName, primaryDbName, - Collections.singletonList("'hive.repl.approx.max.load.tasks'='1'")) + Arrays.asList("'hive.repl.approx.max.load.tasks'='1','hive.repl.include.external.tables'='false'")) .run("use " + replicatedDbName) .run("show tables") .verifyResults(new String[] {"table1", "table2", "table3", "table4", "table5" }) diff --git a/ql/src/test/queries/clientpositive/exim_10_external_managed.q b/ql/src/test/queries/clientpositive/exim_10_external_managed.q index 5aadbb3e05..cda243fb95 100644 --- a/ql/src/test/queries/clientpositive/exim_10_external_managed.q +++ b/ql/src/test/queries/clientpositive/exim_10_external_managed.q @@ -1,4 +1,5 @@ set hive.test.mode=true; +set hive.repl.include.external.tables=false; set hive.test.mode.prefix=; set hive.test.mode.nosamplelist=exim_department_n4,exim_employee; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 454a4d533d..79102d2c6d 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -597,7 +597,7 @@ public static ConfVars getMetaConf(String name) { + "present in HMS Notification. Any key-value pair whose key is matched with any regex will" +" be removed from Parameters map during Serialization of Table/Partition object."), EVENT_DB_LISTENER_TTL("metastore.event.db.listener.timetolive", - "hive.metastore.event.db.listener.timetolive", 86400, TimeUnit.SECONDS, + "hive.metastore.event.db.listener.timetolive", 7, TimeUnit.DAYS, "time after which events will be removed from the database listener queue"), EVENT_CLEAN_MAX_EVENTS("metastore.event.db.clean.maxevents", "hive.metastore.event.db.clean.maxevents", 10000,