diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index d58326f628..7b0b076f77 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -531,11 +531,12 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal + "task increment that would cross the specified limit."), REPL_PARTITIONS_DUMP_PARALLELISM("hive.repl.partitions.dump.parallelism",100, "Number of threads that will be used to dump partition data information during repl dump."), - REPL_DATA_COPY_LAZY("hive.repl.data.copy.lazy", true, + REPL_RUN_DATA_COPY_TASKS_ON_TARGET("hive.repl.run.data.copy.tasks.on.target", true, "Indicates whether replication should run data copy tasks during repl load operation."), REPL_FILE_LIST_CACHE_SIZE("hive.repl.file.list.cache.size", 10000, "This config indicates threshold for the maximum number of data copy locations to be kept in memory. \n" - + "When the config 'hive.repl.data.copy.lazy' is set to true, this config is not considered."), + + "When the config 'hive.repl.run.data.copy.tasks.on.target' is set to true, this config " + + "is not considered."), REPL_DUMP_METADATA_ONLY("hive.repl.dump.metadata.only", false, "Indicates whether replication dump only metadata information or data + metadata. \n" + "This config makes hive.repl.include.external.tables config ineffective."), diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/commands/TestCommands.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/commands/TestCommands.java index f803bc210f..6156e041dc 100644 --- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/commands/TestCommands.java +++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/commands/TestCommands.java @@ -77,7 +77,7 @@ public static void setUpBeforeClass() throws Exception { TestHCatClient.startMetaStoreServer(); hconf = TestHCatClient.getConf(); hconf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname,""); - hconf.set(HiveConf.ConfVars.REPL_DATA_COPY_LAZY.varname, "false"); + hconf.set(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname, "false"); hconf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java index e46a52045b..7bfbbd8945 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java @@ -32,10 +32,9 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.utils.EncryptionZoneUtils; import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hdfs.DFSTestUtil; -import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; @@ -65,7 +64,6 @@ private static MiniDFSCluster miniDFSCluster; private static String cmroot; private static FileSystem fs; - private static HadoopShims.HdfsEncryptionShim shimCm; private static String cmrootEncrypted; private static String jksFile = System.getProperty("java.io.tmpdir") + "/test.jks"; private static String cmrootFallBack; @@ -92,8 +90,6 @@ public static void setUp() throws Exception { hiveConf.set(HiveConf.ConfVars.REPLCMENCRYPTEDDIR.varname, cmrootEncrypted); hiveConf.set(HiveConf.ConfVars.REPLCMFALLBACKNONENCRYPTEDDIR.varname, cmrootFallBack); initReplChangeManager(); - //Create cm in encrypted zone - shimCm = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, conf); try { client = new HiveMetaStoreClient(hiveConf); @@ -156,7 +152,7 @@ public void dropTableWithDifferentEncryptionZonesDifferentKey() throws Throwable Path dirDb1 = new Path(warehouse.getWhRoot(), dbName1 +".db"); warehouseFs.delete(dirDb1, true); warehouseFs.mkdirs(dirDb1); - shimCm.createEncryptionZone(dirDb1, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirDb1, "test_key_db", conf); Path dirTbl1 = new Path(dirDb1, tblName1); warehouseFs.mkdirs(dirTbl1); Path part11 = new Path(dirTbl1, "part1"); @@ -165,7 +161,7 @@ public void dropTableWithDifferentEncryptionZonesDifferentKey() throws Throwable Path dirDb2 = new Path(warehouse.getWhRoot(), dbName2 +".db"); warehouseFs.delete(dirDb2, true); warehouseFs.mkdirs(dirDb2); - shimCm.createEncryptionZone(dirDb2, "test_key_cm"); + EncryptionZoneUtils.createEncryptionZone(dirDb2, "test_key_cm", conf); Path dirTbl2 = new Path(dirDb2, tblName2); warehouseFs.mkdirs(dirTbl2); Path part12 = new Path(dirTbl2, "part1"); @@ -274,13 +270,13 @@ public void dropTableWithTableAtEncryptionZoneRoot() throws Throwable { warehouseFs.mkdirs(dirDb); Path dirTbl1 = new Path(dirDb, tblName1); warehouseFs.mkdirs(dirTbl1); - shimCm.createEncryptionZone(dirTbl1, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirTbl1, "test_key_db", conf); Path part11 = new Path(dirTbl1, "part1"); createFile(part11, "testClearer11"); Path dirTbl2 = new Path(dirDb, tblName2); warehouseFs.mkdirs(dirTbl2); - shimCm.createEncryptionZone(dirTbl2, "test_key_cm"); + EncryptionZoneUtils.createEncryptionZone(dirTbl2, "test_key_cm", conf); Path part12 = new Path(dirTbl2, "part1"); createFile(part12, "testClearer12"); @@ -350,7 +346,7 @@ public void dropTableWithDifferentEncryptionZonesSameKey() throws Throwable { Path dirDb1 = new Path(warehouse.getWhRoot(), dbName1 +".db"); warehouseFs.mkdirs(dirDb1); - shimCm.createEncryptionZone(dirDb1, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirDb1, "test_key_db", conf); Path dirTbl1 = new Path(dirDb1, tblName1); warehouseFs.mkdirs(dirTbl1); Path part11 = new Path(dirTbl1, "part1"); @@ -358,7 +354,7 @@ public void dropTableWithDifferentEncryptionZonesSameKey() throws Throwable { Path dirDb2 = new Path(warehouse.getWhRoot(), dbName2 +".db"); warehouseFs.mkdirs(dirDb2); - shimCm.createEncryptionZone(dirDb2, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirDb2, "test_key_db", conf); Path dirTbl2 = new Path(dirDb2, tblName2); warehouseFs.mkdirs(dirTbl2); Path part12 = new Path(dirTbl2, "part1"); @@ -465,7 +461,7 @@ public void dropTableWithSameEncryptionZones() throws Throwable { Path dirDb = new Path(warehouse.getWhRoot(), dbName +".db"); warehouseFs.delete(dirDb, true); warehouseFs.mkdirs(dirDb); - shimCm.createEncryptionZone(dirDb, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirDb, "test_key_db", conf); Path dirTbl1 = new Path(dirDb, tblName1); warehouseFs.mkdirs(dirTbl1); Path part11 = new Path(dirTbl1, "part1"); @@ -614,7 +610,7 @@ public void dropExternalTableWithSameEncryptionZonesForCm() throws Throwable { Path dirDb = new Path(warehouse.getWhRoot(), dbName +".db"); warehouseFs.delete(dirDb, true); warehouseFs.mkdirs(dirDb); - shimCm.createEncryptionZone(dirDb, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirDb, "test_key_db", conf); Path dirTbl1 = new Path(dirDb, tblName1); warehouseFs.mkdirs(dirTbl1); Path part11 = new Path(dirTbl1, "part1"); @@ -707,13 +703,13 @@ public void dropExternalTableWithDifferentEncryptionZones() throws Throwable { warehouseFs.mkdirs(dirDb); Path dirTbl1 = new Path(dirDb, tblName1); warehouseFs.mkdirs(dirTbl1); - shimCm.createEncryptionZone(dirTbl1, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirTbl1, "test_key_db", conf); Path part11 = new Path(dirTbl1, "part1"); createFile(part11, "testClearer11"); Path dirTbl2 = new Path(dirDb, tblName2); warehouseFs.mkdirs(dirTbl2); - shimCm.createEncryptionZone(dirTbl2, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirTbl2, "test_key_db", conf); Path part12 = new Path(dirTbl2, "part1"); createFile(part12, "testClearer12"); @@ -799,13 +795,13 @@ public void dropExternalTableWithDifferentEncryptionZonesDifferentKey() throws T warehouseFs.mkdirs(dirDb); Path dirTbl1 = new Path(dirDb, tblName1); warehouseFs.mkdirs(dirTbl1); - shimCm.createEncryptionZone(dirTbl1, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirTbl1, "test_key_db", conf); Path part11 = new Path(dirTbl1, "part1"); createFile(part11, "testClearer11"); Path dirTbl2 = new Path(dirDb, tblName2); warehouseFs.mkdirs(dirTbl2); - shimCm.createEncryptionZone(dirTbl2, "test_key_cm"); + EncryptionZoneUtils.createEncryptionZone(dirTbl2, "test_key_cm", conf); Path part12 = new Path(dirTbl2, "part1"); createFile(part12, "testClearer12"); @@ -930,7 +926,7 @@ public void truncateTableWithDifferentEncryptionZones() throws Throwable { Path dirDb1 = new Path(warehouse.getWhRoot(), dbName1 +".db"); warehouseFs.delete(dirDb1, true); warehouseFs.mkdirs(dirDb1); - shimCm.createEncryptionZone(dirDb1, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirDb1, "test_key_db", conf); Path dirTbl1 = new Path(dirDb1, tblName1); warehouseFs.mkdirs(dirTbl1); Path part11 = new Path(dirTbl1, "part1"); @@ -939,7 +935,7 @@ public void truncateTableWithDifferentEncryptionZones() throws Throwable { Path dirDb2 = new Path(warehouse.getWhRoot(), dbName2 +".db"); warehouseFs.delete(dirDb2, true); warehouseFs.mkdirs(dirDb2); - shimCm.createEncryptionZone(dirDb2, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirDb2, "test_key_db", conf); Path dirTbl2 = new Path(dirDb2, tblName2); warehouseFs.mkdirs(dirTbl2); Path part12 = new Path(dirTbl2, "part1"); @@ -1020,7 +1016,7 @@ public void truncateTableWithDifferentEncryptionZonesDifferentKey() throws Throw Path dirDb1 = new Path(warehouse.getWhRoot(), dbName1 +".db"); warehouseFs.mkdirs(dirDb1); - shimCm.createEncryptionZone(dirDb1, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirDb1, "test_key_db", conf); Path dirTbl1 = new Path(dirDb1, tblName1); warehouseFs.mkdirs(dirTbl1); Path part11 = new Path(dirTbl1, "part1"); @@ -1028,7 +1024,7 @@ public void truncateTableWithDifferentEncryptionZonesDifferentKey() throws Throw Path dirDb2 = new Path(warehouse.getWhRoot(), dbName2 +".db"); warehouseFs.mkdirs(dirDb2); - shimCm.createEncryptionZone(dirDb2, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirDb2, "test_key_db", conf); Path dirTbl2 = new Path(dirDb2, tblName2); warehouseFs.mkdirs(dirTbl2); Path part12 = new Path(dirTbl2, "part1"); @@ -1125,7 +1121,7 @@ public void truncateTableWithSameEncryptionZones() throws Throwable { Path dirDb = new Path(warehouse.getWhRoot(), dbName +".db"); warehouseFs.delete(dirDb, true); warehouseFs.mkdirs(dirDb); - shimCm.createEncryptionZone(dirDb, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirDb, "test_key_db", conf); Path dirTbl1 = new Path(dirDb, tblName1); warehouseFs.mkdirs(dirTbl1); Path part11 = new Path(dirTbl1, "part1"); @@ -1233,7 +1229,7 @@ public void recycleFailureWithDifferentEncryptionZonesForCm() throws Throwable { warehouseFs.mkdirs(dirDb); Path dirTbl1 = new Path(dirDb, "tbl1"); warehouseFs.mkdirs(dirTbl1); - shimCm.createEncryptionZone(dirTbl1, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirTbl1, "test_key_db", conf); Path part11 = new Path(dirTbl1, "part1"); createFile(part11, "testClearer11"); @@ -1262,8 +1258,6 @@ public void testClearerEncrypted() throws Exception { FileSystem cmfs = new Path(cmrootCmClearer).getFileSystem(hiveConfCmClearer); cmfs.mkdirs(warehouseCmClearer.getWhRoot()); - HadoopShims.HdfsEncryptionShim shimCmEncrypted = ShimLoader.getHadoopShims().createHdfsEncryptionShim(cmfs, conf); - FileSystem fsWarehouse = warehouseCmClearer.getWhRoot().getFileSystem(hiveConfCmClearer); long now = System.currentTimeMillis(); Path dirDb = new Path(warehouseCmClearer.getWhRoot(), "db1"); @@ -1271,7 +1265,7 @@ public void testClearerEncrypted() throws Exception { fsWarehouse.mkdirs(dirDb); Path dirTbl1 = new Path(dirDb, "tbl1"); fsWarehouse.mkdirs(dirTbl1); - shimCmEncrypted.createEncryptionZone(dirTbl1, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirTbl1, "test_key_db", conf); Path part11 = new Path(dirTbl1, "part1"); createFile(part11, "testClearer11"); String fileChksum11 = ReplChangeManager.checksumFor(part11, fsWarehouse); @@ -1280,7 +1274,7 @@ public void testClearerEncrypted() throws Exception { String fileChksum12 = ReplChangeManager.checksumFor(part12, fsWarehouse); Path dirTbl2 = new Path(dirDb, "tbl2"); fsWarehouse.mkdirs(dirTbl2); - shimCmEncrypted.createEncryptionZone(dirTbl2, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirTbl2, "test_key_db", conf); Path part21 = new Path(dirTbl2, "part1"); createFile(part21, "testClearer21"); String fileChksum21 = ReplChangeManager.checksumFor(part21, fsWarehouse); @@ -1289,7 +1283,7 @@ public void testClearerEncrypted() throws Exception { String fileChksum22 = ReplChangeManager.checksumFor(part22, fsWarehouse); Path dirTbl3 = new Path(dirDb, "tbl3"); fsWarehouse.mkdirs(dirTbl3); - shimCmEncrypted.createEncryptionZone(dirTbl3, "test_key_cm"); + EncryptionZoneUtils.createEncryptionZone(dirTbl3, "test_key_cm", conf); Path part31 = new Path(dirTbl3, "part1"); createFile(part31, "testClearer31"); String fileChksum31 = ReplChangeManager.checksumFor(part31, fsWarehouse); @@ -1371,8 +1365,7 @@ public void testCmrootEncrypted() throws Exception { encryptedHiveConf.set(HiveConf.ConfVars.REPLCMFALLBACKNONENCRYPTEDDIR.varname, cmrootFallBack); //Create cm in encrypted zone - HadoopShims.HdfsEncryptionShim shimCmEncrypted = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, conf); - shimCmEncrypted.createEncryptionZone(new Path(cmrootdirEncrypted), "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(new Path(cmrootdirEncrypted), "test_key_db", conf); ReplChangeManager.resetReplChangeManagerInstance(); Warehouse warehouseEncrypted = new Warehouse(encryptedHiveConf); FileSystem warehouseFsEncrypted = warehouseEncrypted.getWhRoot().getFileSystem(encryptedHiveConf); @@ -1384,7 +1377,7 @@ public void testCmrootEncrypted() throws Exception { warehouseFsEncrypted.mkdirs(dirDb); Path dirTbl1 = new Path(dirDb, "tbl1"); warehouseFsEncrypted.mkdirs(dirTbl1); - shimCmEncrypted.createEncryptionZone(dirTbl1, "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(dirTbl1, "test_key_db", conf); Path part11 = new Path(dirTbl1, "part1"); createFile(part11, "testClearer11"); @@ -1436,9 +1429,8 @@ public void testCmrootFallbackEncrypted() throws Exception { encryptedHiveConf.set(HiveConf.ConfVars.REPLCMFALLBACKNONENCRYPTEDDIR.varname, cmRootFallbackEncrypted); //Create cm in encrypted zone - HadoopShims.HdfsEncryptionShim shimCmEncrypted = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, conf); - shimCmEncrypted.createEncryptionZone(new Path(cmrootdirEncrypted), "test_key_db"); - shimCmEncrypted.createEncryptionZone(new Path(cmRootFallbackEncrypted), "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(new Path(cmrootdirEncrypted), "test_key_db", conf); + EncryptionZoneUtils.createEncryptionZone(new Path(cmRootFallbackEncrypted), "test_key_db", conf); ReplChangeManager.resetReplChangeManagerInstance(); boolean exceptionThrown = false; try { @@ -1474,8 +1466,7 @@ public void testCmrootFallbackRelative() throws Exception { encryptedHiveConf.set(HiveConf.ConfVars.REPLCMFALLBACKNONENCRYPTEDDIR.varname, cmRootFallbackEncrypted); //Create cm in encrypted zone - HadoopShims.HdfsEncryptionShim shimCmEncrypted = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, conf); - shimCmEncrypted.createEncryptionZone(new Path(cmrootdirEncrypted), "test_key_db"); + EncryptionZoneUtils.createEncryptionZone(new Path(cmrootdirEncrypted), "test_key_db", conf); ReplChangeManager.resetReplChangeManagerInstance(); boolean exceptionThrown = false; diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java index a7cd3a64af..203ac5e8a1 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationScenariosAcidTables.java @@ -98,7 +98,7 @@ static void internalBeforeClassSetup(Map overrides, Class clazz) put("hive.metastore.disallow.incompatible.col.type.changes", "false"); put("hive.in.repl.test", "true"); put("metastore.warehouse.tenant.colocation", "true"); - put(HiveConf.ConfVars.REPL_DATA_COPY_LAZY.varname, "false"); + put(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname, "false"); }}; acidEnableConf.putAll(overrides); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java index 24104c709b..21490e4c3e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java @@ -55,7 +55,7 @@ public static void classLevelSetup() throws Exception { conf.set("dfs.client.use.datanode.hostname", "true"); conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); conf.set("hive.repl.include.external.tables", "false"); - conf.set(HiveConf.ConfVars.REPL_DATA_COPY_LAZY.varname, "false"); + conf.set(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname, "false"); MiniDFSCluster miniDFSCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build(); HashMap overridesForHiveConf = new HashMap() {{ diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index eb0776d505..15adac0ee5 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -181,7 +181,7 @@ static void internalBeforeClassSetup(Map additionalProperties) hconf.setBoolVar(HiveConf.ConfVars.HIVEOPTIMIZEMETADATAQUERIES, true); hconf.setBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER, true); hconf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE, true); - hconf.setBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY, false); + hconf.setBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET, false); System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); @@ -602,7 +602,7 @@ public void testBasicWithCMLazyCopy() throws Exception { verifySetup("SELECT a from " + dbName + ".ptned_empty", empty, driver); verifySetup("SELECT * from " + dbName + ".unptned_empty", empty, driver); - String lazyCopyClause = " with ('" + HiveConf.ConfVars.REPL_DATA_COPY_LAZY.varname + "'='true')"; + String lazyCopyClause = " with ('" + HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname + "'='true')"; advanceDumpDir(); run("REPL DUMP " + dbName + lazyCopyClause, driver); @@ -1650,7 +1650,7 @@ public void testIncrementalLoadLazyCopy() throws IOException { run("CREATE TABLE " + dbName + ".ptned_empty(a string) partitioned by (b int) STORED AS TEXTFILE", driver); - List lazyCopyClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_DATA_COPY_LAZY.varname + "'='true'"); + List lazyCopyClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname + "'='true'"); Tuple bootstrapDump = bootstrapLoadAndVerify(dbName, replDbName, lazyCopyClause); String[] unptnData = new String[] {"eleven", "twelve"}; diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java index a7a5bdfd09..2c176a2902 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.AbortTxnsRequest; -import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.messaging.json.gzip.GzipJSONMessageEncoder; import org.apache.hadoop.hive.metastore.txn.TxnStore; @@ -103,7 +102,7 @@ static void internalBeforeClassSetup(Map overrides, put("hive.metastore.disallow.incompatible.col.type.changes", "false"); put("metastore.warehouse.tenant.colocation", "true"); put("hive.in.repl.test", "true"); - put(HiveConf.ConfVars.REPL_DATA_COPY_LAZY.varname, "false"); + put(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname, "false"); }}; acidEnableConf.putAll(overrides); @@ -1808,7 +1807,7 @@ public void testCheckPointingWithNewTablesAdded() throws Throwable { @Test public void testManagedTableLazyCopy() throws Throwable { List withClause = Arrays.asList( - "'" + HiveConf.ConfVars.REPL_DATA_COPY_LAZY.varname + "'='true'"); + "'" + HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname + "'='true'"); WarehouseInstance.Tuple bootstrapDump = primary.run("use " + primaryDbName) .run("CREATE TABLE t1(a string) STORED AS TEXTFILE") diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java index d735c9bd20..869cca5a18 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java @@ -35,7 +35,6 @@ import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.repl.incremental.IncrementalLoadTasksBuilder; import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils; -import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.parse.repl.PathBuilder; import org.apache.hadoop.hive.ql.processors.CommandProcessorException; import org.apache.hadoop.hive.ql.util.DependencyResolver; @@ -160,7 +159,7 @@ public void testCreateFunctionOnHDFSIncrementalReplicationLazyCopy() throws Thro Path identityUdf2HdfsPath = new Path(primary.functionsRoot, "idFunc2" + File.separator + "identity_udf2.jar"); setupUDFJarOnHDFS(identityUdfLocalPath, identityUdf1HdfsPath); setupUDFJarOnHDFS(identityUdfLocalPath, identityUdf2HdfsPath); - List withClasuse = Arrays.asList("'" + HiveConf.ConfVars.REPL_DATA_COPY_LAZY.varname + "'='true'"); + List withClasuse = Arrays.asList("'" + HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname + "'='true'"); primary.run("CREATE FUNCTION " + primaryDbName + ".idFunc1 as 'IdentityStringUDF' " diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java index a5678f26e8..a2cb793848 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java @@ -77,7 +77,7 @@ public static void classLevelSetup() throws Exception { overrides.put(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname, "true"); overrides.put(HiveConf.ConfVars.HIVE_DISTCP_DOAS_USER.varname, UserGroupInformation.getCurrentUser().getUserName()); - overrides.put(HiveConf.ConfVars.REPL_DATA_COPY_LAZY.varname, "false"); + overrides.put(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname, "false"); internalBeforeClassSetup(overrides, TestReplicationScenarios.class); } @@ -213,7 +213,7 @@ public void externalTableReplicationWithDefaultPaths() throws Throwable { @Test public void externalTableReplicationWithDefaultPathsLazyCopy() throws Throwable { - List lazyCopyClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_DATA_COPY_LAZY.varname + "'='true'"); + List lazyCopyClause = Arrays.asList("'" + HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname + "'='true'"); //creates external tables with partitions WarehouseInstance.Tuple tuple = primary .run("use " + primaryDbName) @@ -280,7 +280,7 @@ public void externalTableReplicationWithCustomPathsLazyCopy() throws Throwable { List withClause = Arrays.asList( "'distcp.options.update'=''", - "'" + HiveConf.ConfVars.REPL_DATA_COPY_LAZY.varname + "'='true'" + "'" + HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname + "'='true'" ); primary.run("use " + primaryDbName) diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java index 44eead0b30..6df024a01f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenarios.java @@ -97,7 +97,7 @@ static void internalBeforeClassSetup(Map primaryOverrides, Map additionalOverrides = new HashMap() {{ put("fs.defaultFS", miniDFSCluster.getFileSystem().getUri().toString()); put(HiveConf.ConfVars.HIVE_IN_TEST_REPL.varname, "true"); - put(HiveConf.ConfVars.REPL_DATA_COPY_LAZY.varname, "false"); + put(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET.varname, "false"); }}; Map replicatedOverrides = new HashMap<>(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java index 214c12d015..264bc3b179 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java @@ -104,7 +104,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.Base64; -import java.util.LinkedList; import java.util.UUID; import java.util.ArrayList; import java.util.Map; @@ -176,7 +175,7 @@ public int execute() { ReplChangeManager.getInstance(conf); Path cmRoot = new Path(conf.getVar(HiveConf.ConfVars.REPLCMDIR)); Long lastReplId; - LOG.info("Data copy at load enabled : {}", conf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY)); + LOG.info("Data copy at load enabled : {}", conf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET)); if (isBootstrap) { lastReplId = bootStrapDump(hiveDumpRoot, dmd, cmRoot, getHive()); } else { @@ -573,7 +572,7 @@ private Long incrementalDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot, Hive } Path dbRootMetadata = new Path(metadataPath, dbName); Path dbRootData = new Path(bootstrapRoot, EximUtil.DATA_PATH_NAME + File.separator + dbName); - boolean dataCopyAtLoad = conf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean dataCopyAtLoad = conf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); try (Writer writer = new Writer(dumpRoot, conf)) { for (String tableName : Utils.matchesTbl(hiveDb, dbName, work.replScope)) { try { @@ -610,7 +609,7 @@ private Long incrementalDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot, Hive } private void setDataCopyIterators(FileList extTableFileList, FileList managedTableFileList) { - boolean dataCopyAtLoad = conf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean dataCopyAtLoad = conf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); if (dataCopyAtLoad) { work.setManagedTableCopyPathIterator(Collections.emptyList().iterator()); work.setExternalTblCopyPathIterator(Collections.emptyList().iterator()); @@ -815,7 +814,7 @@ Long bootStrapDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot, Hive hiveDb) work.getMetricCollector().reportStageStart(getName(), metricMap); Path dbRoot = dumpDbMetadata(dbName, metadataPath, bootDumpBeginReplId, hiveDb); Path dbDataRoot = new Path(new Path(dumpRoot, EximUtil.DATA_PATH_NAME), dbName); - boolean dataCopyAtLoad = conf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean dataCopyAtLoad = conf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); functionsBinaryCopyPaths = dumpFunctionMetadata(dbName, dbRoot, dbDataRoot, hiveDb, dataCopyAtLoad); String uniqueKey = Utils.setDbBootstrapDumpState(hiveDb, dbName); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java index 8029b72294..ec67fbdd38 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java @@ -119,7 +119,7 @@ public int execute() { if (shouldLoadAuthorizationMetadata()) { initiateAuthorizationLoadTask(); } - LOG.info("Data copy at load enabled : {}", conf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY)); + LOG.info("Data copy at load enabled : {}", conf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET)); if (work.isIncrementalLoad()) { return executeIncrementalLoad(); } else { @@ -338,7 +338,7 @@ a database ( directory ) } private void addLazyDataCopyTask(TaskTracker loadTaskTracker) throws IOException { - boolean dataCopyAtLoad = conf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean dataCopyAtLoad = conf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); if (dataCopyAtLoad) { if (work.getExternalTableDataCopyItr() == null) { Path extTableBackingFile = new Path(work.dumpDirectory, EximUtil.FILE_LIST_EXTERNAL); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java index d751794181..635fd6dac2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java @@ -275,7 +275,7 @@ private void addPartition(boolean hasMorePartitions, AlterTableAddPartitionDesc loadFileType = event.replicationSpec().isReplace() ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING; stagingDir = PathUtils.getExternalTmpPath(replicaWarehousePartitionLocation, context.pathInfo); } - boolean copyAtLoad = context.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean copyAtLoad = context.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); Task copyTask = ReplCopyTask.getLoadCopyTask( event.replicationSpec(), new Path(event.dataPath() + Path.SEPARATOR + getPartitionName(sourceWarehousePartitionLocation)), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java index 8572f081c7..45fca07ae0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java @@ -18,8 +18,6 @@ package org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.table; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.ValidReaderWriteIdList; -import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; @@ -57,7 +55,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.BitSet; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -284,7 +281,7 @@ static TableLocationTuple tableLocation(ImportTableDesc tblDesc, Database parent + table.getCompleteName() + " with source location: " + dataPath.toString() + " and target location " + tgtPath.toString()); - boolean copyAtLoad = context.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean copyAtLoad = context.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); Task copyTask = ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, tmpPath, context.hiveConf, copyAtLoad, false); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/FileList.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/FileList.java index b834521b56..50c621b782 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/FileList.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/FileList.java @@ -169,7 +169,7 @@ private static String getNextID() { } public int getThreshold(int cacheSize) { - boolean copyAtLoad = conf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean copyAtLoad = conf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); return copyAtLoad ? 0 : (int)(cacheSize * thresholdFactor); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 79ccbc5034..614453bf1f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -460,7 +460,7 @@ private static ImportTableDesc getBaseCreateTableDescFromTable(String dbName, Task copyTask = null; if (replicationSpec.isInReplicationScope()) { - boolean copyAtLoad = x.getConf().getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean copyAtLoad = x.getConf().getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); copyTask = ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, destPath, x.getConf(), isSkipTrash, needRecycle, copyAtLoad); } else { @@ -603,7 +603,7 @@ private static ImportTableDesc getBaseCreateTableDescFromTable(String dbName, Task copyTask = null; if (replicationSpec.isInReplicationScope()) { - boolean copyAtLoad = x.getConf().getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean copyAtLoad = x.getConf().getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); copyTask = ReplCopyTask.getLoadCopyTask(replicationSpec, new Path(srcLocation), destPath, x.getConf(), isSkipTrash, needRecycle, copyAtLoad); } else { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java index 3120c960e7..334d1b7efd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java @@ -96,7 +96,7 @@ public void handle(Context withinContext) throws Exception { withinContext.replicationSpec, withinContext.hiveConf); - boolean copyAtLoad = withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean copyAtLoad = withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); Iterator partitionFilesIter = apm.getPartitionFilesIter().iterator(); // We expect one to one mapping between partitions and file iterators. For external table, this diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CommitTxnHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CommitTxnHandler.java index fcb919a3e6..265fefe628 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CommitTxnHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CommitTxnHandler.java @@ -60,7 +60,7 @@ CommitTxnMessage eventMessage(String stringRepresentation) { private void writeDumpFiles(Table qlMdTable, Partition ptn, Iterable files, Context withinContext, Path dataPath) throws IOException, LoginException, MetaException, HiveFatalException { - boolean copyAtLoad = withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean copyAtLoad = withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); if (copyAtLoad) { // encoded filename/checksum of files, write into _files writeEncodedDumpFiles(withinContext, files, dataPath); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateFunctionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateFunctionHandler.java index 69671b9290..0d66128e2a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateFunctionHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateFunctionHandler.java @@ -55,7 +55,7 @@ public void handle(Context withinContext) throws Exception { Path metadataPath = new Path(withinContext.eventRoot, EximUtil.METADATA_NAME); Path dataPath = new Path(withinContext.eventRoot, EximUtil.DATA_PATH_NAME); FileSystem fileSystem = metadataPath.getFileSystem(withinContext.hiveConf); - boolean copyAtLoad = withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean copyAtLoad = withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); List functionBinaryCopyPaths = new ArrayList<>(); try (JsonWriter jsonWriter = new JsonWriter(fileSystem, metadataPath)) { FunctionSerializer serializer = new FunctionSerializer(eventMessage.getFunctionObj(), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java index ed78bc03c1..d9ade2e88a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java @@ -75,7 +75,7 @@ public void handle(Context withinContext) throws Exception { withinContext.replicationSpec, withinContext.hiveConf); - boolean copyAtLoad = withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean copyAtLoad = withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); Iterable files = eventMessage.getFiles(); if (files != null) { if (copyAtLoad) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/InsertHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/InsertHandler.java index 704b9960e4..b0e9bcf74f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/InsertHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/InsertHandler.java @@ -77,7 +77,7 @@ public void handle(Context withinContext) throws Exception { withinContext.hiveConf); Iterable files = eventMessage.getFiles(); - boolean copyAtLoad = withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean copyAtLoad = withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); /* * Insert into/overwrite operation shall operate on one or more partitions or even partitions from multiple tables. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java index f42290ea0a..e65769aa04 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java @@ -42,7 +42,6 @@ import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork; import java.io.IOException; -import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -202,7 +201,7 @@ ResourceUri destinationResourceUri(ResourceUri resourceUri) } private Task getCopyTask(String sourceUri, Path dest) { - boolean copyAtLoad = context.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY); + boolean copyAtLoad = context.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET); if (copyAtLoad ) { return ReplCopyTask.getLoadCopyTask(metadata.getReplicationSpec(), new Path(sourceUri), dest, context.hiveConf); } else { diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/util/TestFileList.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/util/TestFileList.java index 37ac5d6080..67000a0cb3 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/util/TestFileList.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/util/TestFileList.java @@ -144,7 +144,7 @@ private void waitForStreamingClosure(FileListStreamer fileListStreamer) throws I private Object[] setupAndGetTuple(int cacheSize, boolean lazyDataCopy) throws Exception { HiveConf hiveConf = Mockito.mock(HiveConf.class); - Mockito.when(hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DATA_COPY_LAZY)).thenReturn(lazyDataCopy); + Mockito.when(hiveConf.getBoolVar(HiveConf.ConfVars.REPL_RUN_DATA_COPY_TASKS_ON_TARGET)).thenReturn(lazyDataCopy); Path backingFile = new Path("/tmp/backingFile"); LinkedBlockingQueue cache = new LinkedBlockingQueue<>(cacheSize); FileListStreamer fileListStreamer = Mockito.spy(new FileListStreamer(cache, backingFile, hiveConf)); diff --git a/ql/src/test/queries/clientpositive/repl_2_exim_basic.q b/ql/src/test/queries/clientpositive/repl_2_exim_basic.q index 3d5961b75c..2a9ca27013 100644 --- a/ql/src/test/queries/clientpositive/repl_2_exim_basic.q +++ b/ql/src/test/queries/clientpositive/repl_2_exim_basic.q @@ -4,7 +4,7 @@ set hive.test.mode.prefix=; set hive.test.mode.nosamplelist=managed_t,ext_t,managed_t_imported,managed_t_r_imported,ext_t_imported,ext_t_r_imported; set hive.repl.include.external.tables=true; set hive.repl.dump.metadata.only.for.external.table=false; -set hive.repl.data.copy.lazy=false; +set hive.repl.run.data.copy.tasks.on.target=false; drop table if exists managed_t; drop table if exists ext_t; diff --git a/ql/src/test/queries/clientpositive/repl_3_exim_metadata.q b/ql/src/test/queries/clientpositive/repl_3_exim_metadata.q index 46973615bd..6226d2cb2c 100644 --- a/ql/src/test/queries/clientpositive/repl_3_exim_metadata.q +++ b/ql/src/test/queries/clientpositive/repl_3_exim_metadata.q @@ -2,7 +2,7 @@ set hive.mapred.mode=nonstrict; set hive.test.mode=true; set hive.test.mode.prefix=; set hive.test.mode.nosamplelist=replsrc,repldst,repldst_md; -set hive.repl.data.copy.lazy=false; +set hive.repl.run.data.copy.tasks.on.target=false; drop table if exists replsrc; drop table if exists repldst; diff --git a/ql/src/test/queries/clientpositive/repl_4_exim_nocolstat.q b/ql/src/test/queries/clientpositive/repl_4_exim_nocolstat.q index 4bb172e98e..50e2993d89 100644 --- a/ql/src/test/queries/clientpositive/repl_4_exim_nocolstat.q +++ b/ql/src/test/queries/clientpositive/repl_4_exim_nocolstat.q @@ -4,7 +4,7 @@ set hive.test.mode.prefix=; set hive.test.mode.nosamplelist=replsrc,repldst; set metastore.try.direct.sql=false; set hive.metastore.rawstore.impl=org.apache.hadoop.hive.metastore.ObjectStore; -set hive.repl.data.copy.lazy=false; +set hive.repl.run.data.copy.tasks.on.target=false; drop table if exists replsrc; drop table if exists repldst; diff --git a/standalone-metastore/metastore-common/pom.xml b/standalone-metastore/metastore-common/pom.xml index 827c1e91e5..4922ae9aa9 100644 --- a/standalone-metastore/metastore-common/pom.xml +++ b/standalone-metastore/metastore-common/pom.xml @@ -264,11 +264,6 @@ mockito-core test - - org.apache.hive - hive-shims - ${project.version} - diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java index 3af74ba281..67c74d040a 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java @@ -40,16 +40,14 @@ import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.utils.EncryptionZoneUtils; import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.metastore.utils.Retry; import org.apache.hadoop.hive.metastore.utils.StringUtils; -import org.apache.hadoop.hive.shims.HadoopShims; -import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.hive.shims.HadoopShims.HdfsEncryptionShim; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,7 +59,6 @@ private static boolean inited = false; private static boolean enabled = false; private static Map encryptionZoneToCmrootMapping = new HashMap<>(); - private HadoopShims hadoopShims; private static Configuration conf; private String msUser; private String msGroup; @@ -161,7 +158,6 @@ private ReplChangeManager(Configuration conf) throws MetaException { if (MetastoreConf.getBoolVar(conf, ConfVars.REPLCMENABLED)) { ReplChangeManager.enabled = true; ReplChangeManager.conf = conf; - hadoopShims = ShimLoader.getHadoopShims(); cmRootDir = MetastoreConf.getVar(conf, ConfVars.REPLCMDIR); encryptedCmRootDir = MetastoreConf.getVar(conf, ConfVars.REPLCMENCRYPTEDDIR); fallbackNonEncryptedCmRootDir = MetastoreConf.getVar(conf, ConfVars.REPLCMFALLBACKNONENCRYPTEDDIR); @@ -174,12 +170,10 @@ private ReplChangeManager(Configuration conf) throws MetaException { Path cmroot = new Path(cmRootDir); createCmRoot(cmroot); FileSystem cmRootFs = cmroot.getFileSystem(conf); - HdfsEncryptionShim pathEncryptionShim = hadoopShims - .createHdfsEncryptionShim(cmRootFs, conf); - if (pathEncryptionShim.isPathEncrypted(cmroot)) { + if (EncryptionZoneUtils.isPathEncrypted(cmroot, conf)) { //If cm root is encrypted we keep using it for the encryption zone String encryptionZonePath = cmRootFs.getUri() - + pathEncryptionShim.getEncryptionZoneForPath(cmroot).getPath(); + + EncryptionZoneUtils.getEncryptionZoneForPath(cmroot, conf).getPath(); encryptionZoneToCmrootMapping.put(encryptionZonePath, cmRootDir); } else { encryptionZoneToCmrootMapping.put(NO_ENCRYPTION, cmRootDir); @@ -190,7 +184,7 @@ private ReplChangeManager(Configuration conf) throws MetaException { throw new MetaException(ConfVars.REPLCMENCRYPTEDDIR.getHiveName() + " should be absolute path"); } createCmRoot(cmRootFallback); - if (pathEncryptionShim.isPathEncrypted(cmRootFallback)) { + if (EncryptionZoneUtils.isPathEncrypted(cmRootFallback, conf)) { throw new MetaException(ConfVars.REPLCMFALLBACKNONENCRYPTEDDIR.getHiveName() + " should not be encrypted"); } @@ -566,10 +560,9 @@ Path getCmRoot(Path path) throws IOException { String cmrootDir = fallbackNonEncryptedCmRootDir; String encryptionZonePath = NO_ENCRYPTION; if (enabled) { - HdfsEncryptionShim pathEncryptionShim = hadoopShims.createHdfsEncryptionShim(path.getFileSystem(conf), conf); - if (pathEncryptionShim.isPathEncrypted(path)) { + if (EncryptionZoneUtils.isPathEncrypted(path, conf)) { encryptionZonePath = path.getFileSystem(conf).getUri() - + pathEncryptionShim.getEncryptionZoneForPath(path).getPath(); + + EncryptionZoneUtils.getEncryptionZoneForPath(path, conf).getPath(); //For encryption zone, create cm at the relative path specified by hive.repl.cm.encryptionzone.rootdir //at the root of the encryption zone cmrootDir = encryptionZonePath + Path.SEPARATOR + encryptedCmRootDir; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/EncryptionZoneUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/EncryptionZoneUtils.java new file mode 100644 index 0000000000..22a3892518 --- /dev/null +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/EncryptionZoneUtils.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.utils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.hdfs.client.HdfsAdmin; +import org.apache.hadoop.hdfs.protocol.EncryptionZone; + +import java.io.IOException; +import java.net.URI; + +public class EncryptionZoneUtils { + + private EncryptionZoneUtils() { + + } + + public static boolean isPathEncrypted(Path path, Configuration conf) throws IOException { + Path fullPath; + if (path.isAbsolute()) { + fullPath = path; + } else { + fullPath = path.getFileSystem(conf).makeQualified(path); + } + return (EncryptionZoneUtils.getEncryptionZoneForPath(fullPath, conf) != null); + } + + public static EncryptionZone getEncryptionZoneForPath(Path path, Configuration conf) throws IOException { + URI uri = path.getFileSystem(conf).getUri(); + if ("hdfs".equals(uri.getScheme())) { + HdfsAdmin hdfsAdmin = new HdfsAdmin(uri, conf); + if (path.getFileSystem(conf).exists(path)) { + return hdfsAdmin.getEncryptionZoneForPath(path); + } else if (!path.getParent().equals(path)) { + return getEncryptionZoneForPath(path.getParent(), conf); + } else { + return null; + } + } + return null; + } + + public static void createEncryptionZone(Path path, String keyName, Configuration conf) throws IOException { + URI uri = path.getFileSystem(conf).getUri(); + if ("hdfs".equals(uri.getScheme())) { + HdfsAdmin hdfsAdmin = new HdfsAdmin(uri, conf); + hdfsAdmin.createEncryptionZone(path, keyName); + } else { + throw new UnsupportedOperationException("Cannot create encryption zone for scheme {}" + uri.getScheme()); + } + } +}