diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java index 81cc8c4..932ce8d 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java @@ -172,7 +172,7 @@ public void onDropPartition (DropPartitionEvent partitionEvent) throws MetaExce Table t = partitionEvent.getTable(); NotificationEvent event = new NotificationEvent(0, now(), HCatConstants.HCAT_DROP_PARTITION_EVENT, - msgFactory.buildDropPartitionMessage(t, partitionEvent.getPartition()).toString()); + msgFactory.buildDropPartitionMessage(t, partitionEvent.getPartitionIterator()).toString()); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); enqueue(event); diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java index da3d4da..f7c2f44 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java @@ -19,9 +19,6 @@ package org.apache.hive.hcatalog.listener; -import java.util.ArrayList; -import java.util.HashMap; - import javax.jms.Connection; import javax.jms.ConnectionFactory; import javax.jms.DeliveryMode; @@ -43,9 +40,7 @@ import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; @@ -165,21 +160,14 @@ public void onAlterPartition(AlterPartitionEvent ape) throws MetaException { @Override public void onDropPartition(DropPartitionEvent partitionEvent) throws MetaException { if (partitionEvent.getStatus()) { - Partition partition = partitionEvent.getPartition(); - StorageDescriptor sd = partition.getSd(); - sd.setBucketCols(new ArrayList()); - sd.setSortCols(new ArrayList()); - sd.setParameters(new HashMap()); - sd.getSerdeInfo().setParameters(new HashMap()); - sd.getSkewedInfo().setSkewedColNames(new ArrayList()); String topicName = getTopicName(partitionEvent.getTable()); if (topicName != null && !topicName.equals("")) { - send(messageFactory.buildDropPartitionMessage(partitionEvent.getTable(), partition), topicName); + send(messageFactory.buildDropPartitionMessage(partitionEvent.getTable(), partitionEvent.getPartitionIterator()), topicName); } else { LOG.info("Topic name not found in metastore. Suppressing HCatalog notification for " - + partition.getDbName() + + partitionEvent.getTable().getDbName() + "." - + partition.getTableName() + + partitionEvent.getTable().getTableName() + " To enable notifications for this table, please do alter table set properties (" + HCatConstants.HCAT_MSGBUS_TOPIC_NAME + "=.) or whatever you want topic name to be."); diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java index aec80c1..71dc048 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java @@ -157,10 +157,10 @@ public abstract AlterPartitionMessage buildAlterPartitionMessage(Table table, Pa /** * Factory method for DropPartitionMessage. * @param table The Table from which the partition is dropped. - * @param partition The Partition being dropped. + * @param partitions The set of partitions being dropped. * @return DropPartitionMessage instance. */ - public abstract DropPartitionMessage buildDropPartitionMessage(Table table, Partition partition); + public abstract DropPartitionMessage buildDropPartitionMessage(Table table, Iterator partitions); /** * Factory method for building insert message diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java index 954cd3a..2db05c6 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java @@ -115,9 +115,9 @@ public AlterPartitionMessage buildAlterPartitionMessage(Table table, Partition b } @Override - public DropPartitionMessage buildDropPartitionMessage(Table table, Partition partition) { - return new JSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, partition.getDbName(), - partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)), now()); + public DropPartitionMessage buildDropPartitionMessage(Table table, Iterator partitions) { + return new JSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), getPartitionKeyValues(table, partitions), now()); } @Override diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/exim/TestEximReplicationTasks.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/exim/TestEximReplicationTasks.java index 861ebc8..9682702 100644 --- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/exim/TestEximReplicationTasks.java +++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/exim/TestEximReplicationTasks.java @@ -53,6 +53,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -400,7 +401,8 @@ public void testDropPartition() throws HCatException { Partition p = createPtn(t, Arrays.asList("102", "lmn")); NotificationEvent event = new NotificationEvent(getEventId(), getTime(), - HCatConstants.HCAT_DROP_PARTITION_EVENT, msgFactory.buildDropPartitionMessage(t,p).toString()); + HCatConstants.HCAT_DROP_PARTITION_EVENT, msgFactory.buildDropPartitionMessage( + t, Collections.singletonList(p).iterator()).toString()); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java index ead5a19..2f3c04b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java @@ -20,7 +20,9 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -175,8 +177,12 @@ private void validateLoadPartitionDone(String expectedTableName, assertEquals(expectedTableName, actualTableName); } - private void validateDropPartition(Partition expectedPartition, Partition actualPartition) { - validatePartition(expectedPartition, actualPartition); + private void validateDropPartition(Iterator expectedPartitions, Iterator actualPartitions) { + while (expectedPartitions.hasNext()){ + assertTrue(actualPartitions.hasNext()); + validatePartition(expectedPartitions.next(), actualPartitions.next()); + } + assertFalse(actualPartitions.hasNext()); } private void validateTableInDropPartition(Table expectedTable, Table actualTable) { @@ -417,10 +423,10 @@ public void testListener() throws Exception { DropPartitionEvent dropPart = (DropPartitionEvent)notifyList.get(listSize - 1); assert dropPart.getStatus(); - validateDropPartition(part, dropPart.getPartition()); + validateDropPartition(Collections.singletonList(part).iterator(), dropPart.getPartitionIterator()); validateTableInDropPartition(tbl, dropPart.getTable()); - validateDropPartition(part, preDropPart.getPartition()); + validateDropPartition(Collections.singletonList(part).iterator(), preDropPart.getPartitionIterator()); validateTableInDropPartition(tbl, preDropPart.getTable()); driver.run("drop table " + tblName); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java index ed63bac..e12fb8d 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java @@ -22,28 +22,30 @@ import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; +import java.util.Collections; +import java.util.Iterator; + public class DropPartitionEvent extends ListenerEvent { private final Table table; - private final Partition partition; + private final Iterable partitions; private final boolean deleteData; public DropPartitionEvent (Table table, Partition partition, boolean status, boolean deleteData, HMSHandler handler) { super (status, handler); this.table = table; - this.partition = partition; + this.partitions = Collections.singletonList(partition); // In HiveMetaStore, the deleteData flag indicates whether DFS data should be // removed on a drop. this.deleteData = deleteData; } /** - * @return the partition + * @return the partitions */ - public Partition getPartition() { - - return partition; + public Iterator getPartitionIterator() { + return partitions.iterator(); } /** diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java index 658f4e2..ac391ee 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java @@ -22,35 +22,28 @@ import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; +import java.util.Collections; +import java.util.Iterator; + public class PreDropPartitionEvent extends PreEventContext { - private final Partition partition; + private final Iterable partitions; private final Table table; private final boolean deleteData; - public PreDropPartitionEvent (Partition partition, boolean deleteData, HMSHandler handler) { - super (PreEventType.DROP_PARTITION, handler); - this.partition = partition; - this.table = null; - // In HiveMetaStore, the deleteData flag indicates whether DFS data should be - // removed on a drop. - this.deleteData = false; - } - public PreDropPartitionEvent (Table table, Partition partition, boolean deleteData, HMSHandler handler) { super (PreEventType.DROP_PARTITION, handler); - this.partition = partition; + this.partitions = Collections.singletonList(partition); this.table = table; this.deleteData = false; } /** - * @return the partition + * @return the partitions */ - public Partition getPartition() { - - return partition; + public Iterator getPartitionIterator() { + return partitions.iterator(); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java index fc9d0bd..774a134 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.security.authorization; +import java.util.Iterator; import java.util.List; import org.apache.commons.logging.Log; @@ -33,6 +34,7 @@ import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.events.PreAddPartitionEvent; import org.apache.hadoop.hive.metastore.events.PreAlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent; @@ -314,9 +316,7 @@ private void authorizeAddPartition(PreAddPartitionEvent context) HiveOperation.ALTERTABLE_ADDPARTS.getOutputRequiredPrivileges()); } } - } catch (AuthorizationException e) { - throw invalidOperationException(e); - } catch (NoSuchObjectException e) { + } catch (AuthorizationException | NoSuchObjectException e) { throw invalidOperationException(e); } catch (HiveException e) { throw metaException(e); @@ -326,18 +326,20 @@ private void authorizeAddPartition(PreAddPartitionEvent context) private void authorizeDropPartition(PreDropPartitionEvent context) throws InvalidOperationException, MetaException { try { - org.apache.hadoop.hive.metastore.api.Partition mapiPart = context.getPartition(); - org.apache.hadoop.hive.ql.metadata.Partition wrappedPartition = new PartitionWrapper( - mapiPart, context); - for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) { - authorizer.authorize(wrappedPartition, - HiveOperation.ALTERTABLE_DROPPARTS.getInputRequiredPrivileges(), - HiveOperation.ALTERTABLE_DROPPARTS.getOutputRequiredPrivileges()); + TableWrapper tableWrapper = new TableWrapper(context.getTable()); + Iterator partitionIterator = context.getPartitionIterator(); + while (partitionIterator.hasNext()) { + org.apache.hadoop.hive.metastore.api.Partition mapiPart = partitionIterator.next(); + org.apache.hadoop.hive.ql.metadata.Partition wrappedPartition + = new PartitionWrapper(tableWrapper, mapiPart); + for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) { + authorizer.authorize(wrappedPartition, + HiveOperation.ALTERTABLE_DROPPARTS.getInputRequiredPrivileges(), + HiveOperation.ALTERTABLE_DROPPARTS.getOutputRequiredPrivileges()); + } } } catch (AuthorizationException e) { throw invalidOperationException(e); - } catch (NoSuchObjectException e) { - throw invalidOperationException(e); } catch (HiveException e) { throw metaException(e); } @@ -354,9 +356,7 @@ private void authorizeAlterPartition(PreAlterPartitionEvent context) null, new Privilege[]{Privilege.ALTER_METADATA}); } - } catch (AuthorizationException e) { - throw invalidOperationException(e); - } catch (NoSuchObjectException e) { + } catch (AuthorizationException | NoSuchObjectException e) { throw invalidOperationException(e); } catch (HiveException e) { throw metaException(e);