diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index 0cc0ae5..40b3e4b 100644 --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -776,8 +776,9 @@ public void setMetaStoreSchemaVersion(String schemaVersion, String comment) thro @Override public boolean doesPartitionExist(String catName, String dbName, String tableName, - List partVals) throws MetaException, NoSuchObjectException { - return objectStore.doesPartitionExist(catName, dbName, tableName, partVals); + List partKeys, List partVals) + throws MetaException, NoSuchObjectException { + return objectStore.doesPartitionExist(catName, dbName, tableName, partKeys, partVals); } @Override diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index d8b8414..7f062e7 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -3206,7 +3206,7 @@ public boolean equals(Object obj) { throw new MetaException("Partition value cannot be null."); } - boolean shouldAdd = startAddPartition(ms, part, ifNotExists); + boolean shouldAdd = startAddPartition(ms, part, tbl.getPartitionKeys(), ifNotExists); if (!shouldAdd) { existingParts.add(part); LOG.info("Not adding partition {} as it already exists", part); @@ -3236,6 +3236,7 @@ public boolean equals(Object obj) { for (final Partition partition : partitionsToAdd) { initializePartitionParameters(table, partition); + newParts.add(partition); partFutures.add(threadPool.submit(() -> { if (failureOccurred.get()) { @@ -3472,7 +3473,7 @@ private int add_partitions_pspec_core(RawStore ms, String catName, String dbName throw new MetaException("The partition values cannot be null or empty."); } - boolean shouldAdd = startAddPartition(ms, part, ifNotExists); + boolean shouldAdd = startAddPartition(ms, part, tbl.getPartitionKeys(), ifNotExists); if (!shouldAdd) { LOG.info("Not adding partition {} as it already exists", part); continue; @@ -3580,11 +3581,12 @@ private int add_partitions_pspec_core(RawStore ms, String catName, String dbName } private boolean startAddPartition( - RawStore ms, Partition part, boolean ifNotExists) throws TException { + RawStore ms, Partition part, List partitionKeys, boolean ifNotExists) + throws TException { MetaStoreUtils.validatePartitionNameCharacters(part.getValues(), partitionValidationPattern); boolean doesExist = ms.doesPartitionExist(part.getCatName(), - part.getDbName(), part.getTableName(), part.getValues()); + part.getDbName(), part.getTableName(), partitionKeys, part.getValues()); if (doesExist && !ifNotExists) { throw new AlreadyExistsException("Partition already exists: " + part); } @@ -3707,7 +3709,7 @@ private Partition add_partition_core(final RawStore ms, if (part.getValues() == null || part.getValues().isEmpty()) { throw new MetaException("The partition values cannot be null or empty."); } - boolean shouldAdd = startAddPartition(ms, part, false); + boolean shouldAdd = startAddPartition(ms, part, tbl.getPartitionKeys(), false); assert shouldAdd; // start would throw if it already existed here boolean madeDir = createLocationForAddedPartition(tbl, part); try { diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index b15d89d..98ebfae 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -2239,7 +2239,7 @@ public boolean addPartitions(String catName, String dbName, String tblName, List throw new MetaException("Partition does not belong to target table " + dbName + "." + tblName + ": " + part); } - MPartition mpart = convertToMPart(part, true); + MPartition mpart = convertToMPart(part, table, true); toPersist.add(mpart); int now = (int)(System.currentTimeMillis()/1000); if (tabGrants != null) { @@ -2275,11 +2275,11 @@ public boolean addPartitions(String catName, String dbName, String tblName, List } private boolean isValidPartition( - Partition part, boolean ifNotExists) throws MetaException { + Partition part, List partitionKeys, boolean ifNotExists) throws MetaException { MetaStoreUtils.validatePartitionNameCharacters(part.getValues(), partitionValidationPattern); boolean doesExist = doesPartitionExist(part.getCatName(), - part.getDbName(), part.getTableName(), part.getValues()); + part.getDbName(), part.getTableName(), partitionKeys, part.getValues()); if (doesExist && !ifNotExists) { throw new MetaException("Partition already exists: " + part); } @@ -2310,11 +2310,12 @@ public boolean addPartitions(String catName, String dbName, String tblName, int now = (int)(System.currentTimeMillis()/1000); + List partitionKeys = convertToFieldSchemas(table.getPartitionKeys()); while (iterator.hasNext()) { Partition part = iterator.next(); - if (isValidPartition(part, ifNotExists)) { - MPartition mpart = convertToMPart(part, true); + if (isValidPartition(part, partitionKeys, ifNotExists)) { + MPartition mpart = convertToMPart(part, table, true); pm.makePersistent(mpart); if (tabGrants != null) { for (MTablePrivilege tab : tabGrants) { @@ -2416,24 +2417,31 @@ public Partition getPartition(String catName, String dbName, String tableName, private MPartition getMPartition(String catName, String dbName, String tableName, List part_vals) throws MetaException { + catName = normalizeIdentifier(catName); + dbName = normalizeIdentifier(dbName); + tableName = normalizeIdentifier(tableName); + MTable mtbl = getMTable(catName, dbName, tableName); + if (mtbl == null) { + return null; + } + // Change the query to use part_vals instead of the name which is + // redundant TODO: callers of this often get part_vals out of name for no reason... + String name = + Warehouse.makePartName(convertToFieldSchemas(mtbl.getPartitionKeys()), part_vals); + return getMPartition(catName, dbName, tableName, name); + } + + private MPartition getMPartition(String catName, String dbName, String tableName, + String name) throws MetaException { + catName = normalizeIdentifier(catName); + dbName = normalizeIdentifier(dbName); + tableName = normalizeIdentifier(tableName); List mparts = null; MPartition ret = null; boolean commited = false; Query query = null; try { openTransaction(); - catName = normalizeIdentifier(catName); - dbName = normalizeIdentifier(dbName); - tableName = normalizeIdentifier(tableName); - MTable mtbl = getMTable(catName, dbName, tableName); - if (mtbl == null) { - commited = commitTransaction(); - return null; - } - // Change the query to use part_vals instead of the name which is - // redundant TODO: callers of this often get part_vals out of name for no reason... - String name = - Warehouse.makePartName(convertToFieldSchemas(mtbl.getPartitionKeys()), part_vals); query = pm.newQuery(MPartition.class, "table.tableName == t1 && table.database.name == t2 && partitionName == t3 " + @@ -2484,6 +2492,26 @@ private MPartition convertToMPart(Partition part, boolean useTableCD) return null; } MTable mt = getMTable(part.getCatName(), part.getDbName(), part.getTableName()); + return convertToMPart(part, mt, useTableCD); + } + + /** + * Convert a Partition object into an MPartition, which is an object backed by the db + * If the Partition's set of columns is the same as the parent table's AND useTableCD + * is true, then this partition's storage descriptor's column descriptor will point + * to the same one as the table's storage descriptor. + * @param part the partition to convert + * @param mt the parent table object + * @param useTableCD whether to try to use the parent table's column descriptor. + * @return the model partition object + * @throws InvalidObjectException + * @throws MetaException + */ + private MPartition convertToMPart(Partition part, MTable mt, boolean useTableCD) + throws InvalidObjectException, MetaException { + if (part == null) { + return null; + } if (mt == null) { throw new InvalidObjectException( "Partition doesn't have a valid table or database name"); @@ -9113,14 +9141,11 @@ public void setMetaStoreSchemaVersion(String schemaVersion, String comment) thro } @Override - public boolean doesPartitionExist(String catName, String dbName, String tableName, List - partVals) + public boolean doesPartitionExist(String catName, String dbName, String tableName, + List partKeys, List partVals) throws MetaException { - try { - return this.getPartition(catName, dbName, tableName, partVals) != null; - } catch (NoSuchObjectException e) { - return false; - } + String name = Warehouse.makePartName(partKeys, partVals); + return this.getMPartition(catName, dbName, tableName, name) != null; } private void debugLog(String message) { diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 283798c..444200f 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -328,7 +328,8 @@ Partition getPartition(String catName, String dbName, String tableName, * @throws NoSuchObjectException this is never thrown. */ boolean doesPartitionExist(String catName, String dbName, String tableName, - List part_vals) throws MetaException, NoSuchObjectException; + List partKeys, List part_vals) + throws MetaException, NoSuchObjectException; /** * Drop a partition. diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 9da8d72..da5e4d7 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -969,17 +969,18 @@ public Partition getPartition(String catName, String dbName, String tblName, Lis @Override public boolean doesPartitionExist(String catName, String dbName, String tblName, - List part_vals) throws MetaException, NoSuchObjectException { + List partKeys, List part_vals) + throws MetaException, NoSuchObjectException { catName = normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); if (!shouldCacheTable(catName, dbName, tblName)) { - return rawStore.doesPartitionExist(catName, dbName, tblName, part_vals); + return rawStore.doesPartitionExist(catName, dbName, tblName, partKeys, part_vals); } Table tbl = sharedCache.getTableFromCache(catName, dbName, tblName); if (tbl == null) { // The table containing the partition is not yet loaded in cache - return rawStore.doesPartitionExist(catName, dbName, tblName, part_vals); + return rawStore.doesPartitionExist(catName, dbName, tblName, partKeys, part_vals); } return sharedCache.existPartitionFromCache(catName, dbName, tblName, part_vals); } diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 0461c4e..dfa7eb7 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -739,8 +739,9 @@ public void setMetaStoreSchemaVersion(String schemaVersion, String comment) thro @Override public boolean doesPartitionExist(String catName, String dbName, String tableName, - List partVals) throws MetaException, NoSuchObjectException { - return objectStore.doesPartitionExist(catName, dbName, tableName, partVals); + List partKeys, List partVals) + throws MetaException, NoSuchObjectException { + return objectStore.doesPartitionExist(catName, dbName, tableName, partKeys, partVals); } @Override diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index b71eda4..82c9c28 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -749,7 +749,8 @@ public void setMetaStoreSchemaVersion(String version, String comment) throws Met @Override public boolean doesPartitionExist(String catName, String dbName, String tableName, - List partVals) throws MetaException, NoSuchObjectException { + List partKeys, List partVals) + throws MetaException, NoSuchObjectException { return false; }