diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java index 358a882..0d5628e 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java @@ -172,7 +172,7 @@ public void onDropPartition (DropPartitionEvent partitionEvent) throws MetaExce Table t = partitionEvent.getTable(); NotificationEvent event = new NotificationEvent(0, now(), HCatConstants.HCAT_DROP_PARTITION_EVENT, - msgFactory.buildDropPartitionMessage(t, partitionEvent.getPartition()).toString()); + msgFactory.buildDropPartitionMessage(t, partitionEvent.getPartitionIterator()).toString()); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); enqueue(event); diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java index 1718d79..f3cf705 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java @@ -43,9 +43,7 @@ import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; @@ -55,11 +53,8 @@ import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; -import org.apache.hadoop.hive.metastore.events.InsertEvent; -import org.apache.hadoop.hive.metastore.events.ListenerEvent; import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; import org.apache.hive.hcatalog.common.HCatConstants; -import org.apache.hive.hcatalog.messaging.AlterTableMessage; import org.apache.hive.hcatalog.messaging.HCatEventMessage; import org.apache.hive.hcatalog.messaging.MessageFactory; import org.slf4j.Logger; @@ -168,21 +163,14 @@ public void onAlterPartition(AlterPartitionEvent ape) throws MetaException { @Override public void onDropPartition(DropPartitionEvent partitionEvent) throws MetaException { if (partitionEvent.getStatus()) { - Partition partition = partitionEvent.getPartition(); - StorageDescriptor sd = partition.getSd(); - sd.setBucketCols(new ArrayList()); - sd.setSortCols(new ArrayList()); - sd.setParameters(new HashMap()); - sd.getSerdeInfo().setParameters(new HashMap()); - sd.getSkewedInfo().setSkewedColNames(new ArrayList()); String topicName = getTopicName(partitionEvent.getTable()); if (topicName != null && !topicName.equals("")) { - send(messageFactory.buildDropPartitionMessage(partitionEvent.getTable(), partition), topicName); + send(messageFactory.buildDropPartitionMessage(partitionEvent.getTable(), partitionEvent.getPartitionIterator()), topicName); } else { LOG.info("Topic name not found in metastore. Suppressing HCatalog notification for " - + partition.getDbName() + + partitionEvent.getTable().getDbName() + "." - + partition.getTableName() + + partitionEvent.getTable().getTableName() + " To enable notifications for this table, please do alter table set properties (" + HCatConstants.HCAT_MSGBUS_TOPIC_NAME + "=.) or whatever you want topic name to be."); diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java index 2b16745..8e656e1 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java @@ -155,10 +155,10 @@ public abstract AlterPartitionMessage buildAlterPartitionMessage(Partition befor /** * Factory method for DropPartitionMessage. * @param table The Table from which the partition is dropped. - * @param partition The Partition being dropped. + * @param partitions The set of Partitions being dropped. * @return DropPartitionMessage instance. */ - public abstract DropPartitionMessage buildDropPartitionMessage(Table table, Partition partition); + public abstract DropPartitionMessage buildDropPartitionMessage(Table table, Iterator partitions); /** * Factory method for building insert message diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java index 06efb89..292d75c 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java @@ -115,9 +115,9 @@ public AlterPartitionMessage buildAlterPartitionMessage(Partition before, Partit } @Override - public DropPartitionMessage buildDropPartitionMessage(Table table, Partition partition) { - return new JSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, partition.getDbName(), - partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)), now()); + public DropPartitionMessage buildDropPartitionMessage(Table table, Iterator partitions) { + return new JSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), + table.getTableName(), getPartitionKeyValues(table, partitions), now()); } @Override diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java index ead5a19..ebad63b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -175,8 +176,10 @@ private void validateLoadPartitionDone(String expectedTableName, assertEquals(expectedTableName, actualTableName); } - private void validateDropPartition(Partition expectedPartition, Partition actualPartition) { - validatePartition(expectedPartition, actualPartition); + private void validateDropPartition(Partition expectedPartition, Iterator actualPartition) { + assertTrue(actualPartition.hasNext()); + validatePartition(expectedPartition, actualPartition.next()); + assertFalse(actualPartition.hasNext()); } private void validateTableInDropPartition(Table expectedTable, Table actualTable) { @@ -417,10 +420,10 @@ public void testListener() throws Exception { DropPartitionEvent dropPart = (DropPartitionEvent)notifyList.get(listSize - 1); assert dropPart.getStatus(); - validateDropPartition(part, dropPart.getPartition()); + validateDropPartition(part, dropPart.getPartitionIterator()); validateTableInDropPartition(tbl, dropPart.getTable()); - validateDropPartition(part, preDropPart.getPartition()); + validateDropPartition(part, preDropPart.getPartitionIterator()); validateTableInDropPartition(tbl, preDropPart.getTable()); driver.run("drop table " + tblName); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java index ed63bac..626c17e 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/DropPartitionEvent.java @@ -22,17 +22,20 @@ import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; +import java.util.Collections; +import java.util.Iterator; + public class DropPartitionEvent extends ListenerEvent { private final Table table; - private final Partition partition; + private final Iterable partitions; private final boolean deleteData; public DropPartitionEvent (Table table, Partition partition, boolean status, boolean deleteData, HMSHandler handler) { super (status, handler); this.table = table; - this.partition = partition; + this.partitions = Collections.singletonList(partition); // In HiveMetaStore, the deleteData flag indicates whether DFS data should be // removed on a drop. this.deleteData = deleteData; @@ -41,9 +44,8 @@ public DropPartitionEvent (Table table, /** * @return the partition */ - public Partition getPartition() { - - return partition; + public Iterator getPartitionIterator() { + return partitions.iterator(); } /** diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java index 658f4e2..3326af9 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/PreDropPartitionEvent.java @@ -22,25 +22,19 @@ import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; +import java.util.Collections; +import java.util.Iterator; + public class PreDropPartitionEvent extends PreEventContext { - private final Partition partition; + private final Iterable partitions; private final Table table; private final boolean deleteData; - public PreDropPartitionEvent (Partition partition, boolean deleteData, HMSHandler handler) { - super (PreEventType.DROP_PARTITION, handler); - this.partition = partition; - this.table = null; - // In HiveMetaStore, the deleteData flag indicates whether DFS data should be - // removed on a drop. - this.deleteData = false; - } - public PreDropPartitionEvent (Table table, Partition partition, boolean deleteData, HMSHandler handler) { super (PreEventType.DROP_PARTITION, handler); - this.partition = partition; + this.partitions = Collections.singletonList(partition); this.table = table; this.deleteData = false; } @@ -48,9 +42,8 @@ public PreDropPartitionEvent (Table table, /** * @return the partition */ - public Partition getPartition() { - - return partition; + public Iterator getPartitionIterator() { + return partitions.iterator(); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java index fc9d0bd..16ac021 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.security.authorization; +import java.util.Iterator; import java.util.List; import org.apache.commons.logging.Log; @@ -33,6 +34,7 @@ import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.events.PreAddPartitionEvent; import org.apache.hadoop.hive.metastore.events.PreAlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent; @@ -314,9 +316,7 @@ private void authorizeAddPartition(PreAddPartitionEvent context) HiveOperation.ALTERTABLE_ADDPARTS.getOutputRequiredPrivileges()); } } - } catch (AuthorizationException e) { - throw invalidOperationException(e); - } catch (NoSuchObjectException e) { + } catch (AuthorizationException|NoSuchObjectException e) { throw invalidOperationException(e); } catch (HiveException e) { throw metaException(e); @@ -326,18 +326,20 @@ private void authorizeAddPartition(PreAddPartitionEvent context) private void authorizeDropPartition(PreDropPartitionEvent context) throws InvalidOperationException, MetaException { try { - org.apache.hadoop.hive.metastore.api.Partition mapiPart = context.getPartition(); - org.apache.hadoop.hive.ql.metadata.Partition wrappedPartition = new PartitionWrapper( - mapiPart, context); - for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) { - authorizer.authorize(wrappedPartition, - HiveOperation.ALTERTABLE_DROPPARTS.getInputRequiredPrivileges(), - HiveOperation.ALTERTABLE_DROPPARTS.getOutputRequiredPrivileges()); + TableWrapper tableWrapper = new TableWrapper(context.getTable()); + Iterator partitionIterator = context.getPartitionIterator(); + while (partitionIterator.hasNext()) { + org.apache.hadoop.hive.metastore.api.Partition mapiPart = partitionIterator.next(); + org.apache.hadoop.hive.ql.metadata.Partition wrappedPartition + = new PartitionWrapper(tableWrapper, mapiPart); + for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get()) { + authorizer.authorize(wrappedPartition, + HiveOperation.ALTERTABLE_DROPPARTS.getInputRequiredPrivileges(), + HiveOperation.ALTERTABLE_DROPPARTS.getOutputRequiredPrivileges()); + } } } catch (AuthorizationException e) { throw invalidOperationException(e); - } catch (NoSuchObjectException e) { - throw invalidOperationException(e); } catch (HiveException e) { throw metaException(e); } @@ -354,9 +356,7 @@ private void authorizeAlterPartition(PreAlterPartitionEvent context) null, new Privilege[]{Privilege.ALTER_METADATA}); } - } catch (AuthorizationException e) { - throw invalidOperationException(e); - } catch (NoSuchObjectException e) { + } catch (AuthorizationException|NoSuchObjectException e) { throw invalidOperationException(e); } catch (HiveException e) { throw metaException(e);