commit 27496e92a454c077c0eca98860f445dd69156d00 Author: Alice Fan Date: Tue Jul 24 22:38:08 2018 -0700 HIVE-20226 : HMS getNextNotification will throw exception when request maxEvents exceed table's max_rows diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 6f8276f4e9..beea054d89 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -9582,11 +9582,14 @@ public NotificationEventResponse getNextNotification(NotificationEventRequest rq try { openTransaction(); long lastEvent = rqst.getLastEvent(); - int maxEvents = rqst.getMaxEvents() > 0 ? rqst.getMaxEvents() : Integer.MAX_VALUE; query = pm.newQuery(MNotificationLog.class, "eventId > lastEvent"); query.declareParameters("java.lang.Long lastEvent"); query.setOrdering("eventId ascending"); - query.setRange(0, maxEvents); + int maxEventResponse = MetastoreConf.getIntVar(conf, ConfVars.METASTORE_MAX_EVENT_RESPONSE); + int maxEvents = rqst.getMaxEvents() < maxEventResponse ? rqst.getMaxEvents() : maxEventResponse; + if (maxEvents > 0) { + query.setRange(0, maxEvents); + } Collection events = (Collection) query.execute(lastEvent); commited = commitTransaction(); if (events == null) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index c2bbba5813..05e5c889b6 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -594,6 +594,8 @@ public static ConfVars getMetaConf(String name) { "When used in conjunction with the org.apache.hadoop.hive.ql.parse.MetaDataExportListener pre event listener, \n" + "it is the location to which the metadata will be exported. The default is an empty string, which results in the \n" + "metadata being exported to the current user's home directory on HDFS."), + METASTORE_MAX_EVENT_RESPONSE("metastore.max.event.response", "hive.metastore.max.event.response", 1000000, + "The parameter will decide the maximum number of events that HMS will respond."), MOVE_EXPORTED_METADATA_TO_TRASH("metastore.metadata.move.exported.metadata.to.trash", "hive.metadata.move.exported.metadata.to.trash", true, "When used in conjunction with the org.apache.hadoop.hive.ql.parse.MetaDataExportListener pre event listener, \n" + @@ -1366,6 +1368,17 @@ public static void setLongVar(Configuration conf, ConfVars var, long val) { conf.setLong(var.varname, val); } + /** + * Set the variable as a int + * @param conf configuration file to set it in + * @param var variable to set + * @param val value to set it to + */ + public static void setIntVar(Configuration conf, ConfVars var, int val) { + assert var.defaultVal.getClass() == int.class; + conf.setInt(var.varname, val); + } + /** * Get the variable as a boolean * @param conf configuration to retrieve it from diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index 833e2bdabf..a09ed3ad88 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -811,6 +811,22 @@ public void testNotificationOps() throws InterruptedException, MetaException { Assert.assertEquals(0, eventResponse.getEventsSize()); } + /** + * Test METASTORE_MAX_EVENT_RESPONSE + */ + @Test + public void testMaxEventResponse() throws InterruptedException, MetaException { + NotificationEvent event = + new NotificationEvent(0, 0, EventMessage.EventType.CREATE_DATABASE.toString(), ""); + // Verify if METASTORE_MAX_EVENT_RESPONSE will limit number of event been responded + for (int i = 0; i < 3; i++) { + objectStore.addNotificationEvent(event); + } + MetastoreConf.setIntVar(conf, MetastoreConf.ConfVars.METASTORE_MAX_EVENT_RESPONSE, 1); + NotificationEventResponse eventResponse = objectStore.getNextNotification(new NotificationEventRequest()); + Assert.assertEquals(1, eventResponse.getEventsSize()); + } + @Ignore( "This test is here to allow testing with other databases like mysql / postgres etc\n" + " with user changes to the code. This cannot be run on apache derby because of\n"