diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index 7e0ce0734b..69fd6608fc 100644 --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -459,6 +459,12 @@ public int getNumPartitionsByExpr(String catName, String dbName, String tblName, return objectStore.getNumPartitionsByExpr(catName, dbName, tblName, expr); } + @Override + public int getNumPartitionsByPs(String catName, String dbName, String tblName, List partVals) + throws MetaException, NoSuchObjectException { + return objectStore.getNumPartitionsByPs(catName, dbName, tblName, partVals); + } + @Override public List getPartitionsByNames(String catName, String dbName, String tblName, List partNames) diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index d1db106270..ecea1d94e8 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -759,7 +759,9 @@ public static ConfVars getMetaConf(String name) { "get_partitions_with_auth, \n" + "get_partitions_by_filter, \n" + "get_partitions_spec_by_filter, \n" + - "get_partitions_by_expr.\n" + + "get_partitions_by_expr,\n" + + "get_partitions_ps,\n" + + "get_partitions_ps_with_auth.\n" + "The default value \"-1\" means no limit."), LOG4J_FILE("metastore.log4j.file", "hive.log4j.file", "", "Hive log4j configuration file.\n" + diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index f4f54581d1..ed308eb511 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -5216,6 +5216,15 @@ private void checkLimitNumberOfPartitionsByExpr(String catName, String dbName, S } } + private void checkLimitNumberOfPartitionsByPs(String catName, String dbName, String tblName, + List partVals, int maxParts) + throws TException { + if (isPartitionLimitEnabled()) { + checkLimitNumberOfPartitions(tblName, getNumPartitionsByPs(catName, dbName, tblName, + partVals), maxParts); + } + } + private boolean isPartitionLimitEnabled() { int partitionLimit = MetastoreConf.getIntVar(conf, ConfVars.LIMIT_PARTITION_REQUEST); return partitionLimit > -1; @@ -6245,6 +6254,8 @@ public boolean drop_partition_by_name_with_environment_context(final String db_n List ret = null; Exception ex = null; try { + checkLimitNumberOfPartitionsByPs(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], + tbl_name, part_vals, max_parts); authorizeTableForPartitionMetadata(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name); ret = getMS().listPartitionsPsWithAuth(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, part_vals, max_parts, userName, groupNames); @@ -6935,6 +6946,26 @@ private int get_num_partitions_by_expr(final String catName, final String dbName return ret; } + private int getNumPartitionsByPs(final String catName, final String dbName, + final String tblName, List partVals) + throws TException { + String[] parsedDbName = parseDbName(dbName, conf); + startTableFunction("getNumPartitionsByPs", parsedDbName[CAT_NAME], + parsedDbName[DB_NAME], tblName); + + int ret = -1; + Exception ex = null; + try { + ret = getMS().getNumPartitionsByPs(catName, dbName, tblName, partVals); + } catch (Exception e) { + ex = e; + rethrowException(e); + } finally { + endFunction("getNumPartitionsByPs", ret != -1, ex, tblName); + } + return ret; + } + @Override @Deprecated public List get_partitions_by_names(final String dbName, final String tblName, diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 8fd49827b9..5be974e070 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -3363,6 +3363,41 @@ private PartitionValuesResponse getDistinctValuesForPartitionsNoTxn( return pns; } + @Override + public int getNumPartitionsByPs(String catName, String dbName, String tblName, List partVals) + throws MetaException, NoSuchObjectException { + boolean success = false; + Long result; + QueryWrapper queryWrapper = new QueryWrapper(); + + try { + openTransaction(); + LOG.debug("executing getNumPartitionsByPs"); + catName = normalizeIdentifier(catName); + dbName = normalizeIdentifier(dbName); + tblName = normalizeIdentifier(tblName); + Table table = getTable(catName, dbName, tblName); + if (table == null) { + throw new NoSuchObjectException(TableName.getQualified(catName, dbName, tblName) + + " table not found"); + } + // size is known since it contains dbName, catName, tblName and partialRegex pattern + Map params = new HashMap<>(4); + String filter = getJDOFilterStrForPartitionVals(table, partVals, params); + queryWrapper.query = pm.newQuery( + "select count(partitionName) from org.apache.hadoop.hive.metastore.model.MPartition" + ); + Query query = queryWrapper.query; + query.setFilter(filter); + query.declareParameters(makeParameterDeclarationString(params)); + result = (Long) query.executeWithMap(params); + success = commitTransaction(); + } finally { + rollbackAndCleanup(success, queryWrapper); + } + return result.intValue(); + } + /** * Retrieves a Collection of partition-related results from the database that match * the partial specification given for a specific table. diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 8b66a46b6f..4a1273ed65 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -641,6 +641,20 @@ int getNumPartitionsByFilter(String catName, String dbName, String tblName, Stri int getNumPartitionsByExpr(String catName, String dbName, String tblName, byte[] expr) throws MetaException, NoSuchObjectException; + /** + * Get the number of partitions that match a given partial specification. + * @param catName catalog name. + * @param dbName database name. + * @param tblName table name. + * @param partVals A partial list of values for partitions in order of the table's partition keys. + * Entries can be empty if you need to specify latter partitions. + * @return number of matching partitions. + * @throws MetaException error accessing the RDBMS or working with the specification. + * @throws NoSuchObjectException no such table. + */ + int getNumPartitionsByPs(String catName, String dbName, String tblName, List partVals) + throws MetaException, NoSuchObjectException; + /** * Get partitions by name. * @param catName catalog name. diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 8d28c1fd5a..070275848c 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -1886,6 +1886,12 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, Str return partitionNames; } + @Override + public int getNumPartitionsByPs(String catName, String dbName, String tblName, List partSpecs) + throws MetaException, NoSuchObjectException { + return rawStore.getNumPartitionsByPs(catName, dbName, tblName, partSpecs); + } + @Override public List listPartitionsPsWithAuth(String catName, String dbName, String tblName, List partSpecs, short maxParts, String userName, List groupNames) throws MetaException, InvalidObjectException, NoSuchObjectException { diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index e31dc064c1..37ef4ea89d 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -415,6 +415,12 @@ public int getNumPartitionsByExpr(String catName, String dbName, String tblName, return objectStore.getNumPartitionsByExpr(catName, dbName, tblName, expr); } + @Override + public int getNumPartitionsByPs(String catName, String dbName, String tblName, List partVals) + throws MetaException, NoSuchObjectException { + return objectStore.getNumPartitionsByPs(catName, dbName, tblName, partVals); + } + @Override public List getPartitionsByNames(String catName, String dbName, String tblName, List partNames) throws MetaException, NoSuchObjectException { diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 89acdcc55b..97ba1877b3 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -431,6 +431,12 @@ public int getNumPartitionsByExpr(String catName, String dbName, String tblName, return -1; } + @Override + public int getNumPartitionsByPs(String catName, String dbName, String tblName, List partVals) + throws MetaException, NoSuchObjectException { + return -1; + } + @Override public Table markPartitionForEvent(String catName, String dbName, String tblName, Map partVals, PartitionEventType evtType) throws MetaException, UnknownTableException, diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index dc91b71db9..beeb851deb 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -556,43 +556,63 @@ public void testListPartitionsWihtLimitEnabled() throws Throwable { String dbName = "compdb"; String tblName = "comptbl"; String typeName = "Person"; + String ds = "2008-07-01 14:13:12"; cleanUp(dbName, tblName, typeName); // Create too many partitions, just enough to validate over limit requests List> values = new ArrayList<>(); for (int i=0; i partitions; + List partVal = Collections.singletonList(ds); short maxParts; // Requesting more partitions than allowed should throw an exception + maxParts = -1; try { - maxParts = -1; partitions = client.listPartitions(dbName, tblName, maxParts); fail("should have thrown MetaException about partition limit"); } catch (MetaException e) { assertTrue(true); } + try { + client.listPartitions(dbName, tblName, partVal, maxParts); + fail("should have thrown MetaException about partition limit"); + } catch (MetaException e) { + assertTrue(true); + } + // Requesting more partitions than allowed should throw an exception + maxParts = DEFAULT_LIMIT_PARTITION_REQUEST + 1; try { - maxParts = DEFAULT_LIMIT_PARTITION_REQUEST + 1; partitions = client.listPartitions(dbName, tblName, maxParts); fail("should have thrown MetaException about partition limit"); } catch (MetaException e) { assertTrue(true); } + try { + client.listPartitions(dbName, tblName, partVal, maxParts); + fail("should have thrown MetaException about partition limit"); + } catch (MetaException e) { + assertTrue(true); + } + // Requesting less partitions than allowed should work maxParts = DEFAULT_LIMIT_PARTITION_REQUEST / 2; partitions = client.listPartitions(dbName, tblName, maxParts); assertNotNull("should have returned partitions", partitions); assertEquals(" should have returned 50 partitions", maxParts, partitions.size()); + + partitions = client.listPartitions(dbName, tblName, partVal, maxParts); + assertNotNull("should have returned partitions", partitions); + assertEquals(" should have returned 50 partitions", maxParts, partitions.size()); } @Test diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index 3a3ce5af5c..8a6b5fee93 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -86,6 +86,7 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -390,11 +391,23 @@ public void testPartitionOps() throws Exception { } Assert.assertEquals(partitions.size(), numPartitions); + List partVal = Collections.singletonList(""); + try (AutoCloseable c = deadline()) { + numPartitions = objectStore.getNumPartitionsByPs(DEFAULT_CATALOG_NAME, DB1, TABLE1, partVal); + } + Assert.assertEquals(partitions.size(), numPartitions); + try (AutoCloseable c = deadline()) { numPartitions = objectStore.getNumPartitionsByFilter(DEFAULT_CATALOG_NAME, DB1, TABLE1, "country = \"US\""); } Assert.assertEquals(2, numPartitions); + partVal = Collections.singletonList("US"); + try (AutoCloseable c = deadline()) { + numPartitions = objectStore.getNumPartitionsByPs(DEFAULT_CATALOG_NAME, DB1, TABLE1, partVal); + } + Assert.assertEquals(2, numPartitions); + try (AutoCloseable c = deadline()) { objectStore.dropPartition(DEFAULT_CATALOG_NAME, DB1, TABLE1, value1); partitions = objectStore.getPartitions(DEFAULT_CATALOG_NAME, DB1, TABLE1, 10); diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java index b3003144f6..a2569e5d26 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/client/TestListPartitions.java @@ -712,13 +712,12 @@ public void testListPartitionsWithAuthByValuesTooManyVals() throws Exception { .newArrayList("0", "1", "2", "3"), (short)-1, "", Lists.newArrayList()); } - @Test + @Test(expected = MetaException.class) + @ConditionalIgnoreOnSessionHiveMetastoreClient public void testListPartitionsWithAuthByValuesHighMaxParts() throws Exception { - List> partValues = createTable4PartColsParts(client); - //This doesn't throw MetaException when setting to high max part count - List partitions = client.listPartitionsWithAuthInfo(DB_NAME, TABLE_NAME, Lists + createTable4PartColsParts(client); + client.listPartitionsWithAuthInfo(DB_NAME, TABLE_NAME, Lists .newArrayList("2017"), (short)101, "", Lists.newArrayList()); - assertPartitionsHaveCorrectValues(partitions, partValues.subList(2, 4)); } @Test(expected = MetaException.class)