diff --git ql/src/java/org/apache/hadoop/hive/ql/ddl/misc/msck/MsckOperation.java ql/src/java/org/apache/hadoop/hive/ql/ddl/misc/msck/MsckOperation.java index c05d699bd8..3ffc58f6ca 100644 --- ql/src/java/org/apache/hadoop/hive/ql/ddl/misc/msck/MsckOperation.java +++ ql/src/java/org/apache/hadoop/hive/ql/ddl/misc/msck/MsckOperation.java @@ -52,7 +52,7 @@ public MsckOperation(DDLOperationContext context, MsckDesc desc) { public int execute() throws HiveException, IOException, TException { try { Msck msck = new Msck(false, false); - msck.init(context.getDb().getConf()); + msck.init(Msck.getMsckConf(context.getDb().getConf())); String[] names = Utilities.getDbTableName(desc.getTableName()); diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckCreatePartitionsInBatches.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckCreatePartitionsInBatches.java index 7821f40a82..869249a690 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckCreatePartitionsInBatches.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckCreatePartitionsInBatches.java @@ -83,7 +83,7 @@ public static void setupClass() throws HiveException, MetaException { throw new HiveException(e); } msck = new Msck( false, false); - msck.init(hiveConf); + msck.init(Msck.getMsckConf(hiveConf)); } @Before diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckDropPartitionsInBatches.java ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckDropPartitionsInBatches.java index 8be31128a1..e7318bf6d3 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckDropPartitionsInBatches.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckDropPartitionsInBatches.java @@ -80,7 +80,7 @@ public static void setupClass() throws Exception { SessionState.start(hiveConf); db = new HiveMetaStoreClient(hiveConf); msck = new Msck( false, false); - msck.init(hiveConf); + msck.init(Msck.getMsckConf(hiveConf)); } @Before diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java index f4e109d1b0..59a96e8462 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java @@ -84,15 +84,19 @@ public void setConf(final Configuration conf) { public void init(Configuration conf) throws MetaException { if (msc == null) { - // the only reason we are using new conf here is to override EXPRESSION_PROXY_CLASS - Configuration metastoreConf = MetastoreConf.newMetastoreConf(new Configuration(conf)); - metastoreConf.set(MetastoreConf.ConfVars.EXPRESSION_PROXY_CLASS.getVarname(), - MsckPartitionExpressionProxy.class.getCanonicalName()); - setConf(metastoreConf); - this.msc = new HiveMetaStoreClient(metastoreConf); + setConf(conf); + this.msc = new HiveMetaStoreClient(conf); } } + public static Configuration getMsckConf(Configuration conf) { + // the only reason we are using new conf here is to override EXPRESSION_PROXY_CLASS + Configuration metastoreConf = MetastoreConf.newMetastoreConf(new Configuration(conf)); + metastoreConf.set(MetastoreConf.ConfVars.EXPRESSION_PROXY_CLASS.getVarname(), + MsckPartitionExpressionProxy.class.getCanonicalName()); + return metastoreConf; + } + /** * MetastoreCheck, see if the data in the metastore matches what is on the * dfs. Current version checks for tables and partitions that are either diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionManagementTask.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionManagementTask.java index e4488f4709..612ac87c72 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionManagementTask.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PartitionManagementTask.java @@ -146,7 +146,8 @@ public void run() { new ThreadFactoryBuilder().setDaemon(true).setNameFormat("PartitionDiscoveryTask-%d").build()); CountDownLatch countDownLatch = new CountDownLatch(candidateTables.size()); LOG.info("Found {} candidate tables for partition discovery", candidateTables.size()); - setupMsckConf(); + setupMsckPathInvalidation(); + Configuration msckConf = Msck.getMsckConf(conf); for (Table table : candidateTables) { qualifiedTableName = Warehouse.getCatalogQualifiedTableName(table); long retentionSeconds = getRetentionPeriodInSeconds(table); @@ -155,7 +156,7 @@ public void run() { // this always runs in 'sync' mode where partitions can be added and dropped MsckInfo msckInfo = new MsckInfo(table.getCatName(), table.getDbName(), table.getTableName(), null, null, true, true, true, retentionSeconds); - executorService.submit(new MsckThread(msckInfo, conf, qualifiedTableName, countDownLatch)); + executorService.submit(new MsckThread(msckInfo, msckConf, qualifiedTableName, countDownLatch)); } countDownLatch.await(); executorService.shutdownNow(); @@ -196,7 +197,7 @@ public static long getRetentionPeriodInSeconds(final Table table) { return retentionSeconds; } - private void setupMsckConf() { + private void setupMsckPathInvalidation() { // if invalid partition directory appears, we just skip and move on. We don't want partition management to throw // when invalid path is encountered as these are background threads. We just want to skip and move on. Users will // have to fix the invalid paths via external means.