diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java index 9b27912..cf18420 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java @@ -162,7 +162,7 @@ public void onAddPartition (AddPartitionEvent partitionEvent) Table t = partitionEvent.getTable(); NotificationEvent event = new NotificationEvent(0, now(), HCatConstants.HCAT_ADD_PARTITION_EVENT, - msgFactory.buildAddPartitionMessage(t, partitionEvent.getPartitions()).toString()); + msgFactory.buildAddPartitionMessage(t, partitionEvent.getPartitionIterator()).toString()); event.setDbName(t.getDbName()); event.setTableName(t.getTableName()); enqueue(event); diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java index 89278f1..0a98a10 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/NotificationListener.java @@ -126,15 +126,14 @@ public void onAddPartition(AddPartitionEvent partitionEvent) // and message selector string as "HCAT_EVENT = HCAT_ADD_PARTITION" if (partitionEvent.getStatus()) { Table table = partitionEvent.getTable(); - List partitions = partitionEvent.getPartitions(); String topicName = getTopicName(table); if (topicName != null && !topicName.equals("")) { - send(messageFactory.buildAddPartitionMessage(table, partitions), topicName); + send(messageFactory.buildAddPartitionMessage(table, partitionEvent.getPartitionIterator()), topicName); } else { LOG.info("Topic name not found in metastore. Suppressing HCatalog notification for " - + partitions.get(0).getDbName() + + partitionEvent.getTable().getDbName() + "." - + partitions.get(0).getTableName() + + partitionEvent.getTable().getTableName() + " To enable notifications for this table, please do alter table set properties (" + HCatConstants.HCAT_MSGBUS_TOPIC_NAME + "=.) or whatever you want topic name to be."); diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java index 68cbb9a..05423f4 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/MessageFactory.java @@ -29,6 +29,7 @@ import org.apache.hadoop.util.ReflectionUtils; import org.apache.hive.hcatalog.messaging.json.JSONMessageFactory; +import java.util.Iterator; import java.util.List; /** @@ -142,13 +143,15 @@ public static MessageDeserializer getDeserializer(String format, * @param partitions The set of Partitions being added. * @return AddPartitionMessage instance. */ - public abstract AddPartitionMessage buildAddPartitionMessage(Table table, List partitions); + public abstract AddPartitionMessage buildAddPartitionMessage(Table table, Iterator partitions); /** * Factory method for AddPartitionMessage. * @param table The Table to which the partitions are added. * @param partitionSpec The set of Partitions being added. * @return AddPartitionMessage instance. + * + * TODO : Should we remove this? Using Iterator in above method solves the base problem. */ @InterfaceAudience.LimitedPrivate({"Hive"}) @InterfaceStability.Evolving diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java index 742434f..015fb9d 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/messaging/json/JSONMessageFactory.java @@ -98,9 +98,9 @@ public DropTableMessage buildDropTableMessage(Table table) { } @Override - public AddPartitionMessage buildAddPartitionMessage(Table table, List partitions) { + public AddPartitionMessage buildAddPartitionMessage(Table table, Iterator partitionsIterator) { return new JSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), getPartitionKeyValues(table, partitions), now()); + table.getTableName(), getPartitionKeyValues(table, partitionsIterator), now()); } @Override @@ -108,7 +108,7 @@ public AddPartitionMessage buildAddPartitionMessage(Table table, List @InterfaceStability.Evolving public AddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec) { return new JSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(), - table.getTableName(), getPartitionKeyValues(table, partitionSpec), now()); + table.getTableName(), getPartitionKeyValues(table, partitionSpec.getPartitionIterator()), now()); } @Override @@ -142,18 +142,9 @@ private long now() { return partitionKeys; } - private static List> getPartitionKeyValues(Table table, List partitions) { - List> partitionList = new ArrayList>(partitions.size()); - for (Partition partition : partitions) - partitionList.add(getPartitionKeyValues(table, partition)); - return partitionList; - } - @InterfaceAudience.LimitedPrivate({"Hive"}) - @InterfaceStability.Evolving - private static List> getPartitionKeyValues(Table table, PartitionSpecProxy partitionSpec) { + private static List> getPartitionKeyValues(Table table, Iterator iterator) { List> partitionList = new ArrayList>(); - PartitionSpecProxy.PartitionIterator iterator = partitionSpec.getPartitionIterator(); while (iterator.hasNext()) { Partition partition = iterator.next(); partitionList.add(getPartitionKeyValues(table, partition)); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java index 5b9c350..2874470 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/events/AddPartitionEvent.java @@ -63,16 +63,32 @@ public Table getTable() { /** * @return List of partitions. + * + * Deprecated interface - use getPartitionIterator instead + * -> or, we should change this to return Iterable */ + @Deprecated public List getPartitions() { return partitions; + // TODO : We should change the signature of this function from returning a List to returning an Iterable +// if (partitions != null) { +// return partitions; +// } else if (partitionSpecProxy != null){ +// return ImmutableList.copyOf(partitionSpecProxy.getPartitionIterator()); +// } else { +// return null; +// } } /** * @return Iterator for partitions. */ public Iterator getPartitionIterator() { - return partitionSpecProxy == null ? null : partitionSpecProxy.getPartitionIterator(); + if (partitions != null){ + return partitions.iterator(); + } else { + return partitionSpecProxy == null ? null : partitionSpecProxy.getPartitionIterator(); + } } }