diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index ff97522..9c46cf7 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -295,6 +295,11 @@ public boolean dropPartition(String catName, String dbName, String tableName, Li } @Override + public Map getPartitionLocations(String catName, String dbName, String tblName) { + return objectStore.getPartitionLocations(catName, dbName, tblName); + } + + @Override public void updateCreationMetadata(String catName, String dbname, String tablename, CreationMetadata cm) throws MetaException { objectStore.updateCreationMetadata(catName, dbname, tablename, cm); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index b9f5fb8..8c1945f 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -1543,7 +1543,7 @@ private void drop_database_core(RawStore ms, String catName, // For each partition in each table, drop the partitions and get a list of // partitions' locations which might need to be deleted partitionPaths = dropPartitionsAndGetLocations(ms, catName, name, table.getTableName(), - tablePath, table.getPartitionKeys(), deleteData && !isExternal(table)); + tablePath, deleteData && !isExternal(table)); // Drop the table but not its data drop_table(MetaStoreUtils.prependCatalogToDbName(table.getCatName(), table.getDbName(), conf), @@ -2423,7 +2423,7 @@ private boolean drop_table_core(final RawStore ms, final String catName, final S // Drop the partitions and get a list of locations which need to be deleted partPaths = dropPartitionsAndGetLocations(ms, catName, dbname, name, tblPath, - tbl.getPartitionKeys(), deleteData && !isExternal); + deleteData && !isExternal); // Drop any constraints on the table ms.dropConstraint(catName, dbname, name, null, true); @@ -2508,78 +2508,71 @@ private void deletePartitionData(List partPaths, boolean ifPurge, Database } /** - * Retrieves the partitions specified by partitionKeys. If checkLocation, for locations of - * partitions which may not be subdirectories of tablePath checks to make the locations are - * writable. + * Deletes the partitions specified by catName, dbName, tableName. If checkLocation, for + * locations of partitions which may not be subdirectories of tablePath checks to make sure the + * locations are writable. * * Drops the metadata for each partition. * * Provides a list of locations of partitions which may not be subdirectories of tablePath. * - * @param ms - * @param dbName - * @param tableName - * @param tablePath - * @param partitionKeys - * @param checkLocation - * @return + * @param ms RawStore to use for metadata retrieval and delete + * @param catName The catName + * @param dbName The dbName + * @param tableName The tableName + * @param tablePath The tablePath of which subdirectories does not have to be checked + * @param checkLocation Should we check the locations at all + * @return The list of the Path objects to delete (only in case checkLocation is true) * @throws MetaException * @throws IOException - * @throws InvalidInputException - * @throws InvalidObjectException * @throws NoSuchObjectException */ private List dropPartitionsAndGetLocations(RawStore ms, String catName, String dbName, - String tableName, Path tablePath, List partitionKeys, boolean checkLocation) - throws MetaException, IOException, NoSuchObjectException, InvalidObjectException, - InvalidInputException { - int partitionBatchSize = MetastoreConf.getIntVar(conf, - ConfVars.BATCH_RETRIEVE_MAX); + String tableName, Path tablePath, boolean checkLocation) + throws MetaException, IOException, NoSuchObjectException { Path tableDnsPath = null; if (tablePath != null) { tableDnsPath = wh.getDnsPath(tablePath); } - List partPaths = new ArrayList<>(); - Table tbl = ms.getTable(catName, dbName, tableName); - // call dropPartition on each of the table's partitions to follow the - // procedure for cleanly dropping partitions. - while (true) { - List partsToDelete = ms.getPartitions(catName, dbName, tableName, partitionBatchSize); - if (partsToDelete == null || partsToDelete.isEmpty()) { - break; - } - List partNames = new ArrayList<>(); - for (Partition part : partsToDelete) { - if (checkLocation && part.getSd() != null && - part.getSd().getLocation() != null) { + Map partitionLocations = ms.getPartitionLocations(catName, dbName, tableName); + if (partitionLocations == null || partitionLocations.isEmpty()) { + return Collections.emptyList(); + } - Path partPath = wh.getDnsPath(new Path(part.getSd().getLocation())); + List partPaths; + if (checkLocation) { + partPaths = new ArrayList(partitionLocations.size()); + for (String partName : partitionLocations.keySet()) { + String pathString = partitionLocations.get(partName); + if (pathString!=null) { + Path partPath = wh.getDnsPath(new Path(pathString)); if (tableDnsPath == null || - (partPath != null && !isSubdirectory(tableDnsPath, partPath))) { + (partPath != null && !isSubdirectory(tableDnsPath, partPath))) { if (!wh.isWritable(partPath.getParent())) { - throw new MetaException("Table metadata not deleted since the partition " + - Warehouse.makePartName(partitionKeys, part.getValues()) + - " has parent location " + partPath.getParent() + " which is not writable " + - "by " + SecurityUtils.getUser()); + throw new MetaException("Table metadata not deleted since the partition " + + partName + " has parent location " + partPath.getParent() + + " which is not writable by " + SecurityUtils.getUser()); } partPaths.add(partPath); } } - partNames.add(Warehouse.makePartName(tbl.getPartitionKeys(), part.getValues())); } - for (MetaStoreEventListener listener : listeners) { - //No drop part listener events fired for public listeners historically, for drop table case. - //Limiting to internal listeners for now, to avoid unexpected calls for public listeners. - if (listener instanceof HMSMetricsListener) { - for (@SuppressWarnings("unused") Partition part : partsToDelete) { - listener.onDropPartition(null); - } + } else { + partPaths = Collections.emptyList(); + } + + for (MetaStoreEventListener listener : listeners) { + //No drop part listener events fired for public listeners historically, for drop table case. + //Limiting to internal listeners for now, to avoid unexpected calls for public listeners. + if (listener instanceof HMSMetricsListener) { + for (@SuppressWarnings("unused") String partName : partitionLocations.keySet()) { + listener.onDropPartition(null); } } - ms.dropPartitions(catName, dbName, tableName, partNames); } + ms.dropPartitions(catName, dbName, tableName, new ArrayList<>(partitionLocations.keySet())); return partPaths; } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index b3a8dd0..ac3e947 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -2762,6 +2762,39 @@ private boolean dropPartitionCommon(MPartition part) throws NoSuchObjectExceptio return getPartitionsInternal(catName, dbName, tableName, maxParts, true, true); } + @Override + public Map getPartitionLocations(String catName, String dbName, String tblName) { + catName = normalizeIdentifier(catName); + dbName = normalizeIdentifier(dbName); + tblName = normalizeIdentifier(tblName); + + boolean success = false; + Map partLocations = new HashMap<>(); + try { + openTransaction(); + LOG.debug("Executing getPartitionLocations"); + + Query query = pm.newQuery(MPartition.class); + query.setFilter("this.table.database.catalogName == t1 && this.table.database.name == t2 " + + "&& this.table.tableName == t3"); + query.declareParameters("String t1, String t2, String t3"); + query.setResult("this.partitionName, this.sd.location"); + + List result = (List)query.execute(catName, dbName, tblName); + for(Object[] row:result) { + partLocations.put((String)row[0], (String)row[1]); + } + LOG.debug("Done executing query for getPartitionLocations"); + success = commitTransaction(); + query.closeAll(); + } finally { + if (!success) { + rollbackTransaction(); + } + } + return partLocations; + } + protected List getPartitionsInternal(String catName, String dbName, String tblName, final int maxParts, boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index f350aa9..e7cdbe9 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -361,6 +361,15 @@ boolean dropPartition(String catName, String dbName, String tableName, String tableName, int max) throws MetaException, NoSuchObjectException; /** + * Get the location for every partition of a given table. + * @param catName catalog name. + * @param dbName database name. + * @param tblName table name. + * @return The map of the partitionName, location pairs + */ + Map getPartitionLocations(String catName, String dbName, String tblName); + + /** * Alter a table. * @param catName catalog the table is in. * @param dbname database the table is in. diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index d9356b8..80c5391 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -1037,6 +1037,11 @@ public void dropPartitions(String catName, String dbName, String tblName, List getPartitionLocations(String catName, String dbName, String tblName) { + return rawStore.getPartitionLocations(catName, dbName, tblName); + } + + @Override public void alterTable(String catName, String dbName, String tblName, Table newTable) throws InvalidObjectException, MetaException { rawStore.alterTable(catName, dbName, tblName, newTable); diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 8c3ada3..b070e83 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -272,6 +272,11 @@ public boolean dropPartition(String catName, String dbName, String tableName, Li } @Override + public Map getPartitionLocations(String catName, String dbName, String tblName) { + return objectStore.getPartitionLocations(catName, dbName, tblName); + } + + @Override public void alterTable(String catName, String dbName, String name, Table newTable) throws InvalidObjectException, MetaException { objectStore.alterTable(catName, dbName, name, newTable); diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index f98e8de..dcf483a 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -270,6 +270,11 @@ public boolean dropPartition(String catName, String dbName, String tableName, Li } @Override + public Map getPartitionLocations(String catName, String dbName, String tblName) { + return Collections.emptyMap(); + } + + @Override public void alterTable(String catName, String dbname, String name, Table newTable) throws InvalidObjectException, MetaException { }