diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index ccadac1ada6aaae884ab39f5d99e91b8c542404e..b25f62ba3fa0a34aa1610c6aec885ae323194285 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -57,6 +57,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -260,8 +261,21 @@ public void alterTable(RawStore msdb, Warehouse wh, String dbname, msdb.alterTable(dbname, name, newt); // alterPartition is only for changing the partition location in the table rename if (dataWasMoved) { - for (Partition part : parts) { - msdb.alterPartition(newDbName, newTblName, part.getValues(), part); + + int partsToProcess = parts.size(); + int partitionBatchSize = MetastoreConf.getIntVar(hiveConf, + MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX); + int batchStart = 0; + while (partsToProcess > 0) { + int batchEnd = Math.min(batchStart + partitionBatchSize, parts.size()); + List partBatch = parts.subList(batchStart, batchEnd); + partsToProcess -= partBatch.size(); + batchStart += partBatch.size(); + List> partValues = new LinkedList<>(); + for (Partition part : partBatch) { + partValues.add(part.getValues()); + } + msdb.alterPartitions(newDbName, newTblName, partValues, partBatch); } } diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 7334a0c9fa87b1fe5b4f6c9d2073a477bc0f11ad..4275ac6ea28832e6fa9cb942569183dd15a4533d 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -3584,7 +3584,13 @@ public void alterTable(String dbname, String name, Table newTable) oldt.setOwner(newt.getOwner()); // Fully copy over the contents of the new SD into the old SD, // so we don't create an extra SD in the metastore db that has no references. + MColumnDescriptor oldCD = null; + MStorageDescriptor oldSD = oldt.getSd(); + if (oldSD != null) { + oldCD = oldSD.getCD(); + } copyMSD(newt.getSd(), oldt.getSd()); + removeUnusedColumnDescriptor(oldCD); oldt.setRetention(newt.getRetention()); oldt.setPartitionKeys(newt.getPartitionKeys()); oldt.setTableType(newt.getTableType()); @@ -3633,12 +3639,27 @@ public void alterIndex(String dbname, String baseTblName, String name, Index new } } - private void alterPartitionNoTxn(String dbname, String name, List part_vals, + /** + * Alters an existing partition. Initiates copy of SD. Returns the old CD. + * @param dbname + * @param name + * @param part_vals Partition values (of the original partition instance) + * @param newPart Partition object containing new information + * @return The column descriptor of the old partition instance (null if table is a view) + * @throws InvalidObjectException + * @throws MetaException + */ + private MColumnDescriptor alterPartitionNoTxn(String dbname, String name, List part_vals, Partition newPart) throws InvalidObjectException, MetaException { name = normalizeIdentifier(name); dbname = normalizeIdentifier(dbname); MPartition oldp = getMPartition(dbname, name, part_vals); MPartition newp = convertToMPart(newPart, false); + MColumnDescriptor oldCD = null; + MStorageDescriptor oldSD = oldp.getSd(); + if (oldSD != null) { + oldCD = oldSD.getCD(); + } if (oldp == null || newp == null) { throw new InvalidObjectException("partition does not exist."); } @@ -3654,6 +3675,7 @@ private void alterPartitionNoTxn(String dbname, String name, List part_v if (newp.getLastAccessTime() != oldp.getLastAccessTime()) { oldp.setLastAccessTime(newp.getLastAccessTime()); } + return oldCD; } @Override @@ -3663,7 +3685,8 @@ public void alterPartition(String dbname, String name, List part_vals, P Exception e = null; try { openTransaction(); - alterPartitionNoTxn(dbname, name, part_vals, newPart); + MColumnDescriptor oldCd = alterPartitionNoTxn(dbname, name, part_vals, newPart); + removeUnusedColumnDescriptor(oldCd); // commit the changes success = commitTransaction(); } catch (Exception exception) { @@ -3689,9 +3712,16 @@ public void alterPartitions(String dbname, String name, List> part_ try { openTransaction(); Iterator> part_val_itr = part_vals.iterator(); + Set oldCds = new HashSet<>(); for (Partition tmpPart: newParts) { List tmpPartVals = part_val_itr.next(); - alterPartitionNoTxn(dbname, name, tmpPartVals, tmpPart); + MColumnDescriptor oldCd = alterPartitionNoTxn(dbname, name, tmpPartVals, tmpPart); + if (oldCd != null) { + oldCds.add(oldCd); + } + } + for (MColumnDescriptor oldCd : oldCds) { + removeUnusedColumnDescriptor(oldCd); } // commit the changes success = commitTransaction(); @@ -3712,7 +3742,6 @@ public void alterPartitions(String dbname, String name, List> part_ private void copyMSD(MStorageDescriptor newSd, MStorageDescriptor oldSd) { oldSd.setLocation(newSd.getLocation()); - MColumnDescriptor oldCD = oldSd.getCD(); // If the columns of the old column descriptor != the columns of the new one, // then change the old storage descriptor's column descriptor. // Convert the MFieldSchema's to their thrift object counterparts, because we maintain @@ -3728,9 +3757,6 @@ private void copyMSD(MStorageDescriptor newSd, MStorageDescriptor oldSd) { oldSd.setCD(newSd.getCD()); } - //If oldCd does not have any more references, then we should delete it - // from the backend db - removeUnusedColumnDescriptor(oldCD); oldSd.setBucketCols(newSd.getBucketCols()); oldSd.setCompressed(newSd.isCompressed()); oldSd.setInputFormat(newSd.getInputFormat());