diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index a8aaa5c..bc1b411 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -940,6 +940,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal METASTORE_METRICS("hive.metastore.metrics.enabled", false, "Enable metrics on the metastore."), METASTORE_INIT_METADATA_COUNT_ENABLED("hive.metastore.initial.metadata.count.enabled", true, "Enable a metadata count at metastore startup for metrics."), + HIVE_METASTORE_DROP_PARTITIONS_BATCH_SIZE("hive.metastore.drop.partitions.batch.size", 1000, ""), // Metastore SSL settings HIVE_METASTORE_USE_SSL("hive.metastore.use.SSL", false, diff --git metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index c3f2e99..639e90e 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -2074,16 +2074,16 @@ public void dropPartitions(String dbName, String tblName, List partNames boolean success = false; openTransaction(); try { - // Delete all things. - dropPartitionGrantsNoTxn(dbName, tblName, partNames); - dropPartitionAllColumnGrantsNoTxn(dbName, tblName, partNames); - dropPartitionColumnStatisticsNoTxn(dbName, tblName, partNames); - - // CDs are reused; go thry partition SDs, detach all CDs from SDs, then remove unused CDs. - for (MColumnDescriptor mcd : detachCdsFromSdsNoTxn(dbName, tblName, partNames)) { - removeUnusedColumnDescriptor(mcd); + int batchSize = HiveConf.getIntVar(hiveConf, HiveConf.ConfVars.HIVE_METASTORE_DROP_PARTITIONS_BATCH_SIZE); + if (batchSize > 0) { + List> partNamesParts = Lists.partition(partNames, batchSize); + for (List partNamesPart : partNamesParts) { + dropPartitionsInternal(dbName, tblName, partNamesPart); + } + } else { + dropPartitionsInternal(dbName, tblName, partNames); } - dropPartitionsNoTxn(dbName, tblName, partNames); + if (!(success = commitTransaction())) { throw new MetaException("Failed to drop partitions"); // Should not happen? } @@ -2094,6 +2094,18 @@ public void dropPartitions(String dbName, String tblName, List partNames } } + private void dropPartitionsInternal(String dbName, String tblName, List partNames) throws MetaException { + dropPartitionGrantsNoTxn(dbName, tblName, partNames); + dropPartitionAllColumnGrantsNoTxn(dbName, tblName, partNames); + dropPartitionColumnStatisticsNoTxn(dbName, tblName, partNames); + + // CDs are reused; go thry partition SDs, detach all CDs from SDs, then remove unused CDs. + for (MColumnDescriptor mcd : detachCdsFromSdsNoTxn(dbName, tblName, partNames)) { + removeUnusedColumnDescriptor(mcd); + } + dropPartitionsNoTxn(dbName, tblName, partNames); + } + /** * Drop an MPartition and cascade deletes (e.g., delete partition privilege grants, * drop the storage descriptor cleanly, etc.)