diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index d1db106270..ecea1d94e8 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -759,7 +759,9 @@ public static ConfVars getMetaConf(String name) { "get_partitions_with_auth, \n" + "get_partitions_by_filter, \n" + "get_partitions_spec_by_filter, \n" + - "get_partitions_by_expr.\n" + + "get_partitions_by_expr,\n" + + "get_partitions_ps,\n" + + "get_partitions_ps_with_auth.\n" + "The default value \"-1\" means no limit."), LOG4J_FILE("metastore.log4j.file", "hive.log4j.file", "", "Hive log4j configuration file.\n" + diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index e20fdaf03d..8d22ebc518 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -5183,6 +5183,15 @@ private void checkLimitNumberOfPartitionsByExpr(String catName, String dbName, S } } + private void checkLimitNumberOfPartitionsByPs(String catName, String dbName, String tblName, + List part_vals, int maxParts) + throws TException { + if (isPartitionLimitEnabled()) { + checkLimitNumberOfPartitions(tblName, get_num_partitions_by_ps(catName, dbName, tblName, + part_vals), maxParts); + } + } + private boolean isPartitionLimitEnabled() { int partitionLimit = MetastoreConf.getIntVar(conf, ConfVars.LIMIT_PARTITION_REQUEST); return partitionLimit > -1; @@ -6182,6 +6191,8 @@ public boolean drop_partition_by_name_with_environment_context(final String db_n List ret = null; Exception ex = null; try { + checkLimitNumberOfPartitionsByPs(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], + tbl_name, part_vals, max_parts); authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); ret = getMS().listPartitionsPsWithAuth(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, part_vals, max_parts, userName, groupNames); @@ -6842,6 +6853,26 @@ private int get_num_partitions_by_expr(final String catName, final String dbName return ret; } + private int get_num_partitions_by_ps(final String catName, final String dbName, + final String tblName, List part_vals) + throws TException { + String[] parsedDbName = parseDbName(dbName, conf); + startTableFunction("get_num_partitions_by_ps", parsedDbName[CAT_NAME], + parsedDbName[DB_NAME], tblName); + + int ret = -1; + Exception ex = null; + try { + ret = getMS().getNumPartitionsByPs(catName, dbName, tblName, part_vals); + } catch (Exception e) { + ex = e; + rethrowException(e); + } finally { + endFunction("get_num_partitions_by_ps", ret != -1, ex, tblName); + } + return ret; + } + @Override public List get_partitions_by_names(final String dbName, final String tblName, final List partNames) throws TException { diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 3922282ea9..dfdb3b4183 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -3363,6 +3363,40 @@ private PartitionValuesResponse getDistinctValuesForPartitionsNoTxn( return pns; } + @Override + public int getNumPartitionsByPs(String catName, String dbName, String tblName, List part_vals) + throws MetaException, NoSuchObjectException { + boolean success = false; + Long result; + QueryWrapper queryWrapper = new QueryWrapper(); + + try { + openTransaction(); + LOG.debug("executing getNumPartitionsByPs"); + catName = normalizeIdentifier(catName); + dbName = normalizeIdentifier(dbName); + tblName = normalizeIdentifier(tblName); + Table table = getTable(catName, dbName, tblName); + if (table == null) { + throw new NoSuchObjectException(TableName.getQualified(catName, dbName, tblName) + + " table not found"); + } + // size is known since it contains dbName, catName, tblName and partialRegex pattern + Map params = new HashMap<>(4); + String filter = getJDOFilterStrForPartitionVals(table, part_vals, params); + Query query = queryWrapper.query = pm.newQuery( + "select count(partitionName) from org.apache.hadoop.hive.metastore.model.MPartition" + ); + query.setFilter(filter); + query.declareParameters(makeParameterDeclarationString(params)); + result = (Long) query.executeWithMap(params); + success = commitTransaction(); + } finally { + rollbackAndCleanup(success, queryWrapper); + } + return result.intValue(); + } + /** * Retrieves a Collection of partition-related results from the database that match * the partial specification given for a specific table. diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 8b66a46b6f..c3a43a4b43 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -641,6 +641,20 @@ int getNumPartitionsByFilter(String catName, String dbName, String tblName, Stri int getNumPartitionsByExpr(String catName, String dbName, String tblName, byte[] expr) throws MetaException, NoSuchObjectException; + /** + * Get the number of partitions that match a given partial specification. + * @param catName catalog name. + * @param dbName database name. + * @param tblName table name. + * @param part_vals A partial list of values for partitions in order of the table's partition keys. + * Entries can be empty if you need to specify latter partitions. + * @return number of matching partitions. + * @throws MetaException error accessing the RDBMS or working with the specification. + * @throws NoSuchObjectException no such table. + */ + int getNumPartitionsByPs(String catName, String dbName, String tblName, List part_vals) + throws MetaException, NoSuchObjectException; + /** * Get partitions by name. * @param catName catalog name. diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 8d28c1fd5a..af9e510634 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -1886,6 +1886,12 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, Str return partitionNames; } + @Override + public int getNumPartitionsByPs(String catName, String dbName, String tblName, List part_vals) + throws MetaException, NoSuchObjectException { + return rawStore.getNumPartitionsByPs(catName, dbName, tblName, part_vals); + } + @Override public List listPartitionsPsWithAuth(String catName, String dbName, String tblName, List partSpecs, short maxParts, String userName, List groupNames) throws MetaException, InvalidObjectException, NoSuchObjectException { diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index e31dc064c1..8eeca2eb12 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -415,6 +415,12 @@ public int getNumPartitionsByExpr(String catName, String dbName, String tblName, return objectStore.getNumPartitionsByExpr(catName, dbName, tblName, expr); } + @Override + public int getNumPartitionsByPs(String catName, String dbName, String tblName, List part_vals) + throws MetaException, NoSuchObjectException { + return objectStore.getNumPartitionsByPs(catName, dbName, tblName, part_vals); + } + @Override public List getPartitionsByNames(String catName, String dbName, String tblName, List partNames) throws MetaException, NoSuchObjectException { diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 89acdcc55b..d112ff1c80 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -431,6 +431,12 @@ public int getNumPartitionsByExpr(String catName, String dbName, String tblName, return -1; } + @Override + public int getNumPartitionsByPs(String catName, String dbName, String tblName, List part_vals) + throws MetaException, NoSuchObjectException { + return -1; + } + @Override public Table markPartitionForEvent(String catName, String dbName, String tblName, Map partVals, PartitionEventType evtType) throws MetaException, UnknownTableException, diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index dc91b71db9..5342f5d1c4 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -556,43 +556,64 @@ public void testListPartitionsWihtLimitEnabled() throws Throwable { String dbName = "compdb"; String tblName = "comptbl"; String typeName = "Person"; + String ds = "2008-07-01 14:13:12"; cleanUp(dbName, tblName, typeName); // Create too many partitions, just enough to validate over limit requests List> values = new ArrayList<>(); for (int i=0; i partitions; + List partVal = new ArrayList<>(); + partVal.add(ds); short maxParts; // Requesting more partitions than allowed should throw an exception + maxParts = -1; try { - maxParts = -1; partitions = client.listPartitions(dbName, tblName, maxParts); fail("should have thrown MetaException about partition limit"); } catch (MetaException e) { assertTrue(true); } + try { + client.listPartitions(dbName, tblName, partVal, maxParts); + fail("should have thrown MetaException about partition limit"); + } catch (MetaException e) { + assertTrue(true); + } + // Requesting more partitions than allowed should throw an exception + maxParts = DEFAULT_LIMIT_PARTITION_REQUEST + 1; try { - maxParts = DEFAULT_LIMIT_PARTITION_REQUEST + 1; partitions = client.listPartitions(dbName, tblName, maxParts); fail("should have thrown MetaException about partition limit"); } catch (MetaException e) { assertTrue(true); } + try { + client.listPartitions(dbName, tblName, partVal, maxParts); + fail("should have thrown MetaException about partition limit"); + } catch (MetaException e) { + assertTrue(true); + } + // Requesting less partitions than allowed should work maxParts = DEFAULT_LIMIT_PARTITION_REQUEST / 2; partitions = client.listPartitions(dbName, tblName, maxParts); assertNotNull("should have returned partitions", partitions); assertEquals(" should have returned 50 partitions", maxParts, partitions.size()); + + partitions = client.listPartitions(dbName, tblName, partVal, maxParts); + assertNotNull("should have returned partitions", partitions); + assertEquals(" should have returned 50 partitions", maxParts, partitions.size()); } @Test