diff --git hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java index 351546c3bc9edb69a435f04795b5ea6c3421f5b0..1acac6c6c255b02f88ad5f75c0bfa22bcd710d28 100644 --- hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java +++ hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java @@ -485,12 +485,12 @@ private void process(NotificationEvent event, ListenerEvent listenerEvent) throw HMSHandler.getMSForConf(hiveConf).addNotificationEvent(event); } - // Set the DB_NOTIFICATION_EVENT_ID for future reference by other listeners. - if (event.isSetEventId()) { - listenerEvent.putParameter( - MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME, - Long.toString(event.getEventId())); - } + // Set the DB_NOTIFICATION_EVENT_ID for future reference by other listeners. + if (event.isSetEventId()) { + listenerEvent.putParameter( + MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME, + Long.toString(event.getEventId())); + } } private static class CleanerThread extends Thread { diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index 35989f5b09d016ebde1c9953893f145cae414e42..454b4cbeb52148c190a33b794a53e37508797651 100644 --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -208,7 +208,11 @@ public Table getTable(String dbName, String tableName) throws MetaException { @Override public boolean addPartition(Partition part) throws InvalidObjectException, MetaException { - return objectStore.addPartition(part); + if (shouldEventSucceed) { + return objectStore.addPartition(part); + } else { + throw new RuntimeException("Event failed."); + } } @Override diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java index c36b632fd17791f00c9b69247a520a1bed22a337..b8c4c8b740199ab8fbc999502ae03cf4ae580fba 100644 --- itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java @@ -28,20 +28,15 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Stack; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; -import org.apache.hadoop.hive.metastore.MetaStoreEventListener; -import org.apache.hadoop.hive.metastore.MetaStoreEventListenerConstants; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -51,7 +46,6 @@ import org.apache.hadoop.hive.metastore.api.FunctionType; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.InsertEventRequestData; -import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; import org.apache.hadoop.hive.metastore.api.Order; @@ -62,21 +56,6 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.events.AddIndexEvent; -import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; -import org.apache.hadoop.hive.metastore.events.AlterIndexEvent; -import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; -import org.apache.hadoop.hive.metastore.events.AlterTableEvent; -import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; -import org.apache.hadoop.hive.metastore.events.CreateFunctionEvent; -import org.apache.hadoop.hive.metastore.events.CreateTableEvent; -import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; -import org.apache.hadoop.hive.metastore.events.DropFunctionEvent; -import org.apache.hadoop.hive.metastore.events.DropIndexEvent; -import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; -import org.apache.hadoop.hive.metastore.events.DropTableEvent; -import org.apache.hadoop.hive.metastore.events.InsertEvent; -import org.apache.hadoop.hive.metastore.events.ListenerEvent; import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage; import org.apache.hadoop.hive.metastore.messaging.AlterIndexMessage; import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage; @@ -97,10 +76,7 @@ import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.hcatalog.api.repl.ReplicationV1CompatRule; -import org.apache.hive.hcatalog.data.Pair; -import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; @@ -108,26 +84,29 @@ import org.slf4j.LoggerFactory; /** - * Tests DbNotificationListener when used as a transactional event listener - * (hive.metastore.transactional.event.listeners) + * Abstract class to test DbNotificationListener. + * To test the listener when configured using + * HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, + * override TestNonTransactionalDbNotificationListener. + * To test the listener when configured using + * HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS, + * override TestTransactionalDbNotificationListener. */ -public class TestDbNotificationListener { +public abstract class TestDbNotificationListener { private static final Logger LOG = LoggerFactory.getLogger(TestDbNotificationListener.class .getName()); private static final int EVENTS_TTL = 30; private static final int CLEANUP_SLEEP_TIME = 10; - private static Map emptyParameters = new HashMap(); - private static IMetaStoreClient msClient; private static Driver driver; private static MessageDeserializer md = null; - private int startTime; - private long firstEventId; - private static List testsToSkipForReplV1BackwardCompatTesting = new ArrayList<>(Arrays.asList("cleanupNotifs", "sqlTempTable")); // Make sure we skip backward-compat checking for those tests that don't generate events - private static ReplicationV1CompatRule bcompat = null; + protected int startTime; + protected long firstEventId; + protected static Map emptyParameters = new HashMap(); + protected static IMetaStoreClient msClient; @Rule public TestRule replV1BackwardCompatibleRule = bcompat; @@ -137,105 +116,8 @@ // context. So, bcompat is initialzed in a static context, but this rule is initialized // before the tests run, and will pick up an initialized value of bcompat. - /* This class is used to verify that HiveMetaStore calls the non-transactional listeners with the - * current event ID set by the DbNotificationListener class */ - public static class MockMetaStoreEventListener extends MetaStoreEventListener { - private static Stack> eventsIds = new Stack<>(); - - private static void pushEventId(EventType eventType, final ListenerEvent event) { - if (event.getStatus()) { - Map parameters = event.getParameters(); - if (parameters.containsKey(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)) { - Pair pair = - new Pair<>(eventType, parameters.get(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)); - eventsIds.push(pair); - } - } - } - - public static void popAndVerifyLastEventId(EventType eventType, long id) { - if (!eventsIds.isEmpty()) { - Pair pair = eventsIds.pop(); - - assertEquals("Last event type does not match.", eventType, pair.first); - assertEquals("Last event ID does not match.", Long.toString(id), pair.second); - } else { - assertTrue("List of events is empty.",false); - } - } - - public static void clearEvents() { - eventsIds.clear(); - } - - public MockMetaStoreEventListener(Configuration config) { - super(config); - } - - public void onCreateTable (CreateTableEvent tableEvent) throws MetaException { - pushEventId(EventType.CREATE_TABLE, tableEvent); - } - - public void onDropTable (DropTableEvent tableEvent) throws MetaException { - pushEventId(EventType.DROP_TABLE, tableEvent); - } - - public void onAlterTable (AlterTableEvent tableEvent) throws MetaException { - pushEventId(EventType.ALTER_TABLE, tableEvent); - } - - public void onAddPartition (AddPartitionEvent partitionEvent) throws MetaException { - pushEventId(EventType.ADD_PARTITION, partitionEvent); - } - - public void onDropPartition (DropPartitionEvent partitionEvent) throws MetaException { - pushEventId(EventType.DROP_PARTITION, partitionEvent); - } - - public void onAlterPartition (AlterPartitionEvent partitionEvent) throws MetaException { - pushEventId(EventType.ALTER_PARTITION, partitionEvent); - } - - public void onCreateDatabase (CreateDatabaseEvent dbEvent) throws MetaException { - pushEventId(EventType.CREATE_DATABASE, dbEvent); - } - - public void onDropDatabase (DropDatabaseEvent dbEvent) throws MetaException { - pushEventId(EventType.DROP_DATABASE, dbEvent); - } - - public void onAddIndex(AddIndexEvent indexEvent) throws MetaException { - pushEventId(EventType.CREATE_INDEX, indexEvent); - } - - public void onDropIndex(DropIndexEvent indexEvent) throws MetaException { - pushEventId(EventType.DROP_INDEX, indexEvent); - } - - public void onAlterIndex(AlterIndexEvent indexEvent) throws MetaException { - pushEventId(EventType.ALTER_INDEX, indexEvent); - } - - public void onCreateFunction (CreateFunctionEvent fnEvent) throws MetaException { - pushEventId(EventType.CREATE_FUNCTION, fnEvent); - } - - public void onDropFunction (DropFunctionEvent fnEvent) throws MetaException { - pushEventId(EventType.DROP_FUNCTION, fnEvent); - } - - public void onInsert(InsertEvent insertEvent) throws MetaException { - pushEventId(EventType.INSERT, insertEvent); - } - } - - @SuppressWarnings("rawtypes") - @BeforeClass - public static void connectToMetastore() throws Exception { - HiveConf conf = new HiveConf(); - conf.setVar(HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS, - DbNotificationListener.class.getName()); - conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, MockMetaStoreEventListener.class.getName()); + public static void connectToMetastore(HiveConf listenerConfigs) throws Exception { + HiveConf conf = new HiveConf(listenerConfigs); conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_DB_LISTENER_TTL, String.valueOf(EVENTS_TTL) + "s"); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(HiveConf.ConfVars.FIRE_EVENTS_FOR_DML, true); @@ -257,7 +139,6 @@ public static void connectToMetastore() throws Exception { msClient = new HiveMetaStoreClient(conf); driver = new Driver(conf); md = MessageFactory.getInstance().getDeserializer(); - bcompat = new ReplicationV1CompatRule(msClient, conf, testsToSkipForReplV1BackwardCompatTesting ); } @@ -274,16 +155,10 @@ public void setup() throws Exception { DummyRawStoreFailEvent.setEventSucceed(true); } - @After - public void tearDown() { - MockMetaStoreEventListener.clearEvents(); - } - @Test public void createDatabase() throws Exception { String dbName = "createdb"; - String dbName2 = "createdb2"; String dbLocationUri = "file:/tmp"; String dbDescription = "no description"; Database db = new Database(dbName, dbDescription, dbLocationUri, emptyParameters); @@ -304,28 +179,11 @@ public void createDatabase() throws Exception { // Parse the message field CreateDatabaseMessage createDbMsg = md.getCreateDatabaseMessage(event.getMessage()); assertEquals(dbName, createDbMsg.getDB()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_DATABASE, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - DummyRawStoreFailEvent.setEventSucceed(false); - db = new Database(dbName2, dbDescription, dbLocationUri, emptyParameters); - try { - msClient.createDatabase(db); - fail("Error: create database should've failed"); - } catch (Exception ex) { - // expected - } - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(1, rsp.getEventsSize()); } @Test public void dropDatabase() throws Exception { String dbName = "dropdb"; - String dbName2 = "dropdb2"; String dbLocationUri = "file:/tmp"; String dbDescription = "no description"; Database db = new Database(dbName, dbDescription, dbLocationUri, emptyParameters); @@ -349,31 +207,12 @@ public void dropDatabase() throws Exception { // Parse the message field DropDatabaseMessage dropDbMsg = md.getDropDatabaseMessage(event.getMessage()); assertEquals(dbName, dropDbMsg.getDB()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_DATABASE, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_DATABASE, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - db = new Database(dbName2, dbDescription, dbLocationUri, emptyParameters); - msClient.createDatabase(db); - DummyRawStoreFailEvent.setEventSucceed(false); - try { - msClient.dropDatabase(dbName2); - fail("Error: drop database should've failed"); - } catch (Exception ex) { - // expected - } - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(3, rsp.getEventsSize()); } @Test public void createTable() throws Exception { String defaultDbName = "default"; String tblName = "createtable"; - String tblName2 = "createtable2"; String tblOwner = "me"; String serdeLocation = "file:/tmp"; FieldSchema col1 = new FieldSchema("col1", "int", "no comment"); @@ -404,24 +243,6 @@ public void createTable() throws Exception { assertEquals(tblName, createTblMsg.getTable()); assertEquals(table, createTblMsg.getTableObj()); assertEquals(TableType.MANAGED_TABLE.toString(), createTblMsg.getTableType()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - table = - new Table(tblName2, defaultDbName, tblOwner, startTime, startTime, 0, sd, null, - emptyParameters, null, null, null); - DummyRawStoreFailEvent.setEventSucceed(false); - try { - msClient.createTable(table); - fail("Error: create table should've failed"); - } catch (Exception ex) { - // expected - } - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(1, rsp.getEventsSize()); } @Test @@ -464,28 +285,12 @@ public void alterTable() throws Exception { AlterTableMessage alterTableMessage = md.getAlterTableMessage(event.getMessage()); assertEquals(table, alterTableMessage.getTableObjAfter()); assertEquals(TableType.MANAGED_TABLE.toString(), alterTableMessage.getTableType()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - DummyRawStoreFailEvent.setEventSucceed(false); - try { - msClient.alter_table(defaultDbName, tblName, table); - fail("Error: alter table should've failed"); - } catch (Exception ex) { - // expected - } - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(2, rsp.getEventsSize()); } @Test public void dropTable() throws Exception { String defaultDbName = "default"; String tblName = "droptbl"; - String tblName2 = "droptbl2"; String tblOwner = "me"; String serdeLocation = "file:/tmp"; FieldSchema col1 = new FieldSchema("col1", "int", "no comment"); @@ -519,33 +324,12 @@ public void dropTable() throws Exception { assertEquals(defaultDbName, dropTblMsg.getDB()); assertEquals(tblName, dropTblMsg.getTable()); assertEquals(TableType.MANAGED_TABLE.toString(), dropTblMsg.getTableType()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_TABLE, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - table = - new Table(tblName2, defaultDbName, tblOwner, startTime, startTime, 0, sd, null, - emptyParameters, null, null, null); - msClient.createTable(table); - DummyRawStoreFailEvent.setEventSucceed(false); - try { - msClient.dropTable(defaultDbName, tblName2); - fail("Error: drop table should've failed"); - } catch (Exception ex) { - // expected - } - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(3, rsp.getEventsSize()); } @Test public void addPartition() throws Exception { String defaultDbName = "default"; String tblName = "addptn"; - String tblName2 = "addptn2"; String tblOwner = "me"; String serdeLocation = "file:/tmp"; FieldSchema col1 = new FieldSchema("col1", "int", "no comment"); @@ -589,25 +373,6 @@ public void addPartition() throws Exception { assertTrue(ptnIter.hasNext()); assertEquals(partition, ptnIter.next()); assertEquals(TableType.MANAGED_TABLE.toString(), addPtnMsg.getTableType()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - partition = - new Partition(Arrays.asList("tomorrow"), defaultDbName, tblName2, startTime, startTime, sd, - emptyParameters); - DummyRawStoreFailEvent.setEventSucceed(false); - try { - msClient.add_partition(partition); - fail("Error: add partition should've failed"); - } catch (Exception ex) { - // expected - } - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(2, rsp.getEventsSize()); } @Test @@ -660,22 +425,6 @@ public void alterPartition() throws Exception { assertEquals(tblName, alterPtnMsg.getTable()); assertEquals(newPart, alterPtnMsg.getPtnObjAfter()); assertEquals(TableType.MANAGED_TABLE.toString(), alterPtnMsg.getTableType()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - DummyRawStoreFailEvent.setEventSucceed(false); - try { - msClient.alter_partition(defaultDbName, tblName, newPart, null); - fail("Error: alter partition should've failed"); - } catch (Exception ex) { - // expected - } - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(3, rsp.getEventsSize()); } @Test @@ -728,28 +477,6 @@ public void dropPartition() throws Exception { assertEquals(table.getTableName(), tableObj.getTableName()); assertEquals(table.getOwner(), tableObj.getOwner()); assertEquals(TableType.MANAGED_TABLE.toString(), dropPtnMsg.getTableType()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_PARTITION, firstEventId + 3); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - List newpartCol1Vals = Arrays.asList("tomorrow"); - partition = - new Partition(newpartCol1Vals, defaultDbName, tblName, startTime, startTime, sd, - emptyParameters); - msClient.add_partition(partition); - DummyRawStoreFailEvent.setEventSucceed(false); - try { - msClient.dropPartition(defaultDbName, tblName, newpartCol1Vals, false); - fail("Error: drop partition should've failed"); - } catch (Exception ex) { - // expected - } - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(4, rsp.getEventsSize()); } @Test @@ -833,13 +560,6 @@ public void exchangePartition() throws Exception { Iterator> parts = dropPtnMsg.getPartitions().iterator(); assertTrue(parts.hasNext()); assertEquals(part1.getValues(), Lists.newArrayList(parts.next().values())); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_PARTITION, firstEventId + 5); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 4); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 3); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); } @Test @@ -879,37 +599,15 @@ public void createFunction() throws Exception { assertEquals(1, funcObj.getResourceUrisSize()); assertEquals(ResourceType.JAR, funcObj.getResourceUris().get(0).getResourceType()); assertEquals(funcResource, funcObj.getResourceUris().get(0).getUri()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_FUNCTION, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - DummyRawStoreFailEvent.setEventSucceed(false); - func = - new Function(funcName2, defaultDbName, funcClass2, ownerName, PrincipalType.USER, - startTime, FunctionType.JAVA, Arrays.asList(new ResourceUri(ResourceType.JAR, - funcResource2))); - try { - msClient.createFunction(func); - fail("Error: create function should've failed"); - } catch (Exception ex) { - // expected - } - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(1, rsp.getEventsSize()); } @Test public void dropFunction() throws Exception { String defaultDbName = "default"; String funcName = "dropfunction"; - String funcName2 = "dropfunction2"; String ownerName = "me"; String funcClass = "o.a.h.h.dropfunction"; - String funcClass2 = "o.a.h.h.dropfunction2"; String funcResource = "file:/tmp/somewhere"; - String funcResource2 = "file:/tmp/somewhere2"; Function func = new Function(funcName, defaultDbName, funcClass, ownerName, PrincipalType.USER, startTime, FunctionType.JAVA, Arrays.asList(new ResourceUri(ResourceType.JAR, funcResource))); @@ -931,27 +629,6 @@ public void dropFunction() throws Exception { DropFunctionMessage dropFuncMsg = md.getDropFunctionMessage(event.getMessage()); assertEquals(defaultDbName, dropFuncMsg.getDB()); assertEquals(funcName, dropFuncMsg.getFunctionName()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_FUNCTION, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_FUNCTION, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - func = - new Function(funcName2, defaultDbName, funcClass2, ownerName, PrincipalType.USER, - startTime, FunctionType.JAVA, Arrays.asList(new ResourceUri(ResourceType.JAR, - funcResource2))); - msClient.createFunction(func); - DummyRawStoreFailEvent.setEventSucceed(false); - try { - msClient.dropFunction(defaultDbName, funcName2); - fail("Error: drop function should've failed"); - } catch (Exception ex) { - // expected - } - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(3, rsp.getEventsSize()); } @Test @@ -1000,29 +677,6 @@ public void createIndex() throws Exception { assertEquals(indexName, indexObj.getIndexName()); assertEquals(tableName, indexObj.getOrigTableName()); assertEquals(indexTableName, indexObj.getIndexTableName()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_INDEX, firstEventId + 3); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - DummyRawStoreFailEvent.setEventSucceed(false); - index = - new Index("createIndexTable2", null, "default", tableName, startTime, startTime, - "createIndexTable2__createIndexTable2__", sd, emptyParameters, false); - Table indexTable2 = - new Table("createIndexTable2__createIndexTable2__", dbName, "me", startTime, startTime, 0, - sd, null, emptyParameters, null, null, null); - try { - msClient.createIndex(index, indexTable2); - fail("Error: create index should've failed"); - } catch (Exception ex) { - // expected - } - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(3, rsp.getEventsSize()); } @Test @@ -1071,33 +725,6 @@ public void dropIndex() throws Exception { assertEquals(indexName.toLowerCase(), dropIdxMsg.getIndexName()); assertEquals(indexTableName.toLowerCase(), dropIdxMsg.getIndexTableName()); assertEquals(tableName.toLowerCase(), dropIdxMsg.getOrigTableName()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.DROP_INDEX, firstEventId + 4); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_INDEX, firstEventId + 3); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - index = - new Index("dropIndexTable2", null, "default", tableName, startTime, startTime, - "dropIndexTable__dropIndexTable2__", sd, emptyParameters, false); - Table indexTable2 = - new Table("dropIndexTable__dropIndexTable2__", dbName, "me", startTime, startTime, 0, sd, - null, emptyParameters, null, null, null); - msClient.createIndex(index, indexTable2); - DummyRawStoreFailEvent.setEventSucceed(false); - try { - // drops index and indexTable - msClient.dropIndex(dbName, tableName, "dropIndex2", true); - fail("Error: drop index should've failed"); - } catch (Exception ex) { - // expected - } - - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(6, rsp.getEventsSize()); } @Test @@ -1151,24 +778,6 @@ public void alterIndex() throws Exception { assertEquals(tableName, indexObj.getOrigTableName()); assertEquals(indexTableName, indexObj.getIndexTableName()); assertTrue(indexObj.getCreateTime() < indexObj.getLastAccessTime()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ALTER_INDEX, firstEventId + 4); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_INDEX, firstEventId + 3); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); - - // When hive.metastore.transactional.event.listeners is set, - // a failed event should not create a new notification - DummyRawStoreFailEvent.setEventSucceed(false); - try { - msClient.alter_index(dbName, tableName, indexName, newIndex); - fail("Error: alter index should've failed"); - } catch (Exception ex) { - // expected - } - rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(4, rsp.getEventsSize()); } @Test @@ -1220,10 +829,6 @@ public void insertTable() throws Exception { assertEquals(defaultDbName, insertMessage.getDB()); assertEquals(tblName, insertMessage.getTable()); assertEquals(TableType.MANAGED_TABLE.toString(), insertMessage.getTableType()); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.INSERT, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); } @Test @@ -1285,11 +890,6 @@ public void insertPartition() throws Exception { List ptnValues = insertMessage.getPtnObj().getValues(); assertEquals(partKeyVals, ptnValues); - - // Verify the eventID was passed to the non-transactional listener - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.INSERT, firstEventId + 3); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.ADD_PARTITION, firstEventId + 2); - MockMetaStoreEventListener.popAndVerifyLastEventId(EventType.CREATE_TABLE, firstEventId + 1); } diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestNonTransactionalDbNotificationListener.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestNonTransactionalDbNotificationListener.java new file mode 100644 index 0000000000000000000000000000000000000000..827176abed573d74908e4d73cc9775fef10d2198 --- /dev/null +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestNonTransactionalDbNotificationListener.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.hcatalog.listener; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.junit.BeforeClass; + +public class TestNonTransactionalDbNotificationListener extends TestDbNotificationListener { + + @SuppressWarnings("rawtypes") + @BeforeClass + public static void setListenerAndConnectToMetastore() throws Exception { + HiveConf conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, DbNotificationListener.class.getName()); + connectToMetastore(conf); + } +} diff --git itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestTransactionalDbNotificationListener.java itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestTransactionalDbNotificationListener.java new file mode 100644 index 0000000000000000000000000000000000000000..309f61843d22142939d737fe94b8baabe41c2d31 --- /dev/null +++ itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestTransactionalDbNotificationListener.java @@ -0,0 +1,723 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.hcatalog.listener; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.MetaStoreEventListener; +import org.apache.hadoop.hive.metastore.MetaStoreEventListenerConstants; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.Function; +import org.apache.hadoop.hive.metastore.api.FunctionType; +import org.apache.hadoop.hive.metastore.api.Index; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; +import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.api.ResourceType; +import org.apache.hadoop.hive.metastore.api.ResourceUri; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.events.AddIndexEvent; +import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterIndexEvent; +import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; +import org.apache.hadoop.hive.metastore.events.AlterTableEvent; +import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.CreateFunctionEvent; +import org.apache.hadoop.hive.metastore.events.CreateTableEvent; +import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; +import org.apache.hadoop.hive.metastore.events.DropFunctionEvent; +import org.apache.hadoop.hive.metastore.events.DropIndexEvent; +import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; +import org.apache.hadoop.hive.metastore.events.DropTableEvent; +import org.apache.hadoop.hive.metastore.events.InsertEvent; +import org.apache.hadoop.hive.metastore.events.ListenerEvent; +import org.apache.hadoop.hive.metastore.messaging.EventMessage; +import org.apache.hive.hcatalog.data.Pair; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Stack; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/* + * Test DbNotificationListener that runs in the same transaction as the notification event. + * There are two items of interest to verify currently: + * - Pass eventID from transactional to non-transactional listener. + * - When metadata event transactions fails, an event should not be generated. + */ +public class TestTransactionalDbNotificationListener extends TestDbNotificationListener { + + @SuppressWarnings("rawtypes") + @BeforeClass + public static void setListenerAndConnectToMetastore() throws Exception { + HiveConf conf = new HiveConf(); + conf.setVar(HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS, DbNotificationListener.class.getName()); + conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, MockMetaStoreEventListener.class.getName()); + connectToMetastore(conf); + } + + /* + * Class used to verify that HiveMetaStore calls the non-transactional listeners with the + * current event ID set by the DbNotificationListener class + */ + public static class MockMetaStoreEventListener extends MetaStoreEventListener { + private static Stack> eventsIds = new Stack<>(); + + private static void pushEventId(EventMessage.EventType eventType, final ListenerEvent event) { + if (event.getStatus()) { + Map parameters = event.getParameters(); + if (parameters.containsKey(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)) { + Pair pair = + new Pair<>(eventType, parameters.get(MetaStoreEventListenerConstants.DB_NOTIFICATION_EVENT_ID_KEY_NAME)); + eventsIds.push(pair); + } + } + } + + public static void popAndVerifyLastEventId(EventMessage.EventType eventType, long id) { + if (!eventsIds.isEmpty()) { + Pair pair = eventsIds.pop(); + assertEquals("Last event type does not match.", eventType, pair.first); + assertEquals("Last event ID does not match.", Long.toString(id), pair.second); + } else { + assertTrue("List of events is empty.",false); + } + } + + public static void clearEvents() { + eventsIds.clear(); + } + + public MockMetaStoreEventListener(Configuration config) { + super(config); + } + + public void onCreateTable (CreateTableEvent tableEvent) throws MetaException { + pushEventId(EventMessage.EventType.CREATE_TABLE, tableEvent); + } + + public void onDropTable (DropTableEvent tableEvent) throws MetaException { + pushEventId(EventMessage.EventType.DROP_TABLE, tableEvent); + } + + public void onAlterTable (AlterTableEvent tableEvent) throws MetaException { + pushEventId(EventMessage.EventType.ALTER_TABLE, tableEvent); + } + + public void onAddPartition (AddPartitionEvent partitionEvent) throws MetaException { + pushEventId(EventMessage.EventType.ADD_PARTITION, partitionEvent); + } + + public void onDropPartition (DropPartitionEvent partitionEvent) throws MetaException { + pushEventId(EventMessage.EventType.DROP_PARTITION, partitionEvent); + } + + public void onAlterPartition (AlterPartitionEvent partitionEvent) throws MetaException { + pushEventId(EventMessage.EventType.ALTER_PARTITION, partitionEvent); + } + + public void onCreateDatabase (CreateDatabaseEvent dbEvent) throws MetaException { + pushEventId(EventMessage.EventType.CREATE_DATABASE, dbEvent); + } + + public void onDropDatabase (DropDatabaseEvent dbEvent) throws MetaException { + pushEventId(EventMessage.EventType.DROP_DATABASE, dbEvent); + } + + public void onAddIndex(AddIndexEvent indexEvent) throws MetaException { + pushEventId(EventMessage.EventType.CREATE_INDEX, indexEvent); + } + + public void onDropIndex(DropIndexEvent indexEvent) throws MetaException { + pushEventId(EventMessage.EventType.DROP_INDEX, indexEvent); + } + + public void onAlterIndex(AlterIndexEvent indexEvent) throws MetaException { + pushEventId(EventMessage.EventType.ALTER_INDEX, indexEvent); + } + + public void onCreateFunction (CreateFunctionEvent fnEvent) throws MetaException { + pushEventId(EventMessage.EventType.CREATE_FUNCTION, fnEvent); + } + + public void onDropFunction (DropFunctionEvent fnEvent) throws MetaException { + pushEventId(EventMessage.EventType.DROP_FUNCTION, fnEvent); + } + + public void onInsert(InsertEvent insertEvent) throws MetaException { + pushEventId(EventMessage.EventType.INSERT, insertEvent); + } + } + + @After + public void tearDown() { + MockMetaStoreEventListener.clearEvents(); + } + + @Test + public void createDatabase() throws Exception { + super.createDatabase(); + + MockMetaStoreEventListener.popAndVerifyLastEventId( + EventMessage.EventType.CREATE_DATABASE, firstEventId + 1); + + // When hive.metastore.transactional.event.listeners is set, + // a failed event should not create a new notification + String dbName2 = "createdb2Transactional"; + String dbLocationUri = "file:/tmp"; + String dbDescription = "no description"; + DummyRawStoreFailEvent.setEventSucceed(false); + Database db = new Database(dbName2, dbDescription, dbLocationUri, emptyParameters); + try { + msClient.createDatabase(db); + fail("Error: create database should've failed"); + } catch (Exception ex) { + // expected + } + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(1, rsp.getEventsSize()); + } + + @Test + public void dropDatabase() throws Exception { + super.dropDatabase(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId( + EventMessage.EventType.DROP_DATABASE, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId( + EventMessage.EventType.CREATE_DATABASE, firstEventId + 1); + + String dbName2 = "dropdb2Txn"; + String dbLocationUri = "file:/tmp"; + String dbDescription = "no description"; + // When hive.metastore.transactional.event.listeners is set, + // a failed event should not create a new notification + Database db = new Database(dbName2, dbDescription, dbLocationUri, emptyParameters); + msClient.createDatabase(db); + DummyRawStoreFailEvent.setEventSucceed(false); + try { + msClient.dropDatabase(dbName2); + fail("Error: drop database should've failed"); + } catch (Exception ex) { + // expected + } + + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(3, rsp.getEventsSize()); + } + + @Test + public void createTable() throws Exception { + super.createTable(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId( + EventMessage.EventType.CREATE_TABLE, firstEventId + 1); + + String defaultDbName = "default"; + String tblName2 = "createtable2createtableTransactional"; + String tblOwner = "me"; + String serdeLocation = "file:/tmp"; + FieldSchema col1 = new FieldSchema("col1", "int", "no comment"); + List cols = new ArrayList(); + cols.add(col1); + SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); + StorageDescriptor sd = + new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, + emptyParameters); + // When hive.metastore.transactional.event.listeners is set, + // a failed event should not create a new notification + Table table = + new Table(tblName2, defaultDbName, tblOwner, startTime, startTime, 0, sd, null, + emptyParameters, null, null, null); + DummyRawStoreFailEvent.setEventSucceed(false); + try { + msClient.createTable(table); + fail("Error: create table should've failed"); + } catch (Exception ex) { + // expected + } + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(1, rsp.getEventsSize()); + } + + @Test + public void alterTable() throws Exception { + super.alterTable(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 1); + + String defaultDbName = "default"; + String tblName = "altertabletblTxn"; + String tblOwner = "me"; + String serdeLocation = "file:/tmp"; + FieldSchema col1 = new FieldSchema("col1", "int", "no comment"); + FieldSchema col2 = new FieldSchema("col2", "int", "no comment"); + List cols = new ArrayList(); + cols.add(col1); + SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); + StorageDescriptor sd = + new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, + emptyParameters); + Table table = + new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, + new ArrayList(), emptyParameters, null, null, null); + + // Event 3 + msClient.createTable(table); + cols.add(col2); + + // When hive.metastore.transactional.event.listeners is set, + // a failed event should not create a new notification + DummyRawStoreFailEvent.setEventSucceed(false); + try { + msClient.alter_table(defaultDbName, tblName, table); + super.alterTable(); + fail("Error: alter table should've failed"); + } catch (Exception ex) { + // expected + } + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(3, rsp.getEventsSize()); + } + + @Test + public void dropTable() throws Exception { + super.dropTable(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.DROP_TABLE, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 1); + + String defaultDbName = "default"; + String tblName = "droptbl2Txn"; + String tblOwner = "me"; + String serdeLocation = "file:/tmp"; + FieldSchema col1 = new FieldSchema("col1", "int", "no comment"); + List cols = new ArrayList(); + cols.add(col1); + SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); + StorageDescriptor sd = + new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, + emptyParameters); + + // When hive.metastore.transactional.event.listeners is set, + // a failed event should not create a new notification + Table table = + new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, null, + emptyParameters, null, null, null); + msClient.createTable(table); + DummyRawStoreFailEvent.setEventSucceed(false); + try { + msClient.dropTable(defaultDbName, tblName); + fail("Error: drop table should've failed"); + } catch (Exception ex) { + // expected + } + + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(3, rsp.getEventsSize()); + } + + @Test + public void addPartition() throws Exception { + super.addPartition(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.ADD_PARTITION, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 1); + + String defaultDbName = "default"; + String tblName = "addptn2Txn"; + String tblOwner = "me"; + String serdeLocation = "file:/tmp"; + FieldSchema col1 = new FieldSchema("col1", "int", "no comment"); + List cols = new ArrayList(); + cols.add(col1); + SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); + StorageDescriptor sd = + new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, + emptyParameters); + FieldSchema partCol1 = new FieldSchema("ds", "string", "no comment"); + List partCols = new ArrayList(); + partCols.add(partCol1); + Table table = + new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, partCols, + emptyParameters, null, null, null); + msClient.createTable(table); + // When hive.metastore.transactional.event.listeners is set, + // a failed event should not create a new notification + Partition partition = + new Partition(Arrays.asList("tomorrow"), defaultDbName, tblName, startTime, startTime, sd, + emptyParameters); + DummyRawStoreFailEvent.setEventSucceed(false); + try { + msClient.add_partition(partition); + fail("Error: add partition should've failed"); + } catch (Exception ex) { + // expected + } + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(3, rsp.getEventsSize()); + } + + @Test + public void alterPartition() throws Exception { + super.alterPartition(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.ADD_PARTITION, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 1); + + String defaultDbName = "default"; + String tblName = "alterptnTxn"; + String tblOwner = "me"; + String serdeLocation = "file:/tmp"; + FieldSchema col1 = new FieldSchema("col1", "int", "no comment"); + List cols = new ArrayList(); + cols.add(col1); + SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); + StorageDescriptor sd = + new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, + emptyParameters); + FieldSchema partCol1 = new FieldSchema("ds", "string", "no comment"); + List partCols = new ArrayList(); + partCols.add(partCol1); + Table table = + new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, partCols, + emptyParameters, null, null, null); + + // Event 4 + msClient.createTable(table); + Partition newPart = + new Partition(Arrays.asList("today"), defaultDbName, tblName, startTime, startTime + 1, sd, + emptyParameters); + + // When hive.metastore.transactional.event.listeners is set, + // a failed event should not create a new notification + DummyRawStoreFailEvent.setEventSucceed(false); + try { + msClient.alter_partition(defaultDbName, tblName, newPart, null); + fail("Error: alter partition should've failed"); + } catch (Exception ex) { + // expected + } + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(4, rsp.getEventsSize()); + } + + @Test + public void dropPartition() throws Exception { + super.dropPartition(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.DROP_PARTITION, firstEventId + 3); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.ADD_PARTITION, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 1); + + String defaultDbName = "default"; + String tblName = "dropptnTxn"; + String tblOwner = "me"; + String serdeLocation = "file:/tmp"; + FieldSchema col1 = new FieldSchema("col1", "int", "no comment"); + List cols = new ArrayList(); + cols.add(col1); + SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); + StorageDescriptor sd = + new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serde, null, null, + emptyParameters); + FieldSchema partCol1 = new FieldSchema("ds", "string", "no comment"); + List partCols = new ArrayList(); + partCols.add(partCol1); + Table table = + new Table(tblName, defaultDbName, tblOwner, startTime, startTime, 0, sd, partCols, + emptyParameters, null, null, null); + // Event 4 + msClient.createTable(table); + // When hive.metastore.transactional.event.listeners is set, + // a failed event should not create a new notification + List newpartCol1Vals = Arrays.asList("tomorrow"); + Partition partition = + new Partition(newpartCol1Vals, defaultDbName, tblName, startTime, startTime, sd, + emptyParameters); + msClient.add_partition(partition); + DummyRawStoreFailEvent.setEventSucceed(false); + try { + msClient.dropPartition(defaultDbName, tblName, newpartCol1Vals, false); + fail("Error: drop partition should've failed"); + } catch (Exception ex) { + // expected + } + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(5, rsp.getEventsSize()); + } + + @Test + public void exchangePartition() throws Exception { + super.exchangePartition(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.DROP_PARTITION, firstEventId + 5); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.ADD_PARTITION, firstEventId + 4); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.ADD_PARTITION, firstEventId + 3); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 1); + } + + @Test + public void createFunction() throws Exception { + super.createFunction(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_FUNCTION, firstEventId + 1); + + String defaultDbName = "default"; + String funcName2 = "createfunction2Txn"; + String ownerName = "me"; + String funcClass2 = "o.a.h.h.createfunc2"; + String funcResource2 = "file:/tmp/somewhere2"; + // When hive.metastore.transactional.event.listeners is set, + // a failed event should not create a new notification + DummyRawStoreFailEvent.setEventSucceed(false); + Function func = + new Function(funcName2, defaultDbName, funcClass2, ownerName, PrincipalType.USER, + startTime, FunctionType.JAVA, Arrays.asList(new ResourceUri(ResourceType.JAR, + funcResource2))); + try { + msClient.createFunction(func); + fail("Error: create function should've failed"); + } catch (Exception ex) { + // expected + } + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(1, rsp.getEventsSize()); + } + + @Test + public void dropFunction() throws Exception { + super.dropFunction(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.DROP_FUNCTION, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_FUNCTION, firstEventId + 1); + + String defaultDbName = "default"; + String funcName2 = "dropfunction2Txn"; + String ownerName = "me"; + String funcClass2 = "o.a.h.h.dropfunctionTxn2"; + String funcResource2 = "file:/tmp/somewhere2"; + + // When hive.metastore.transactional.event.listeners is set, + // a failed event should not create a new notification + Function func = + new Function(funcName2, defaultDbName, funcClass2, ownerName, PrincipalType.USER, + startTime, FunctionType.JAVA, Arrays.asList(new ResourceUri(ResourceType.JAR, + funcResource2))); + msClient.createFunction(func); + DummyRawStoreFailEvent.setEventSucceed(false); + try { + msClient.dropFunction(defaultDbName, funcName2); + fail("Error: drop function should've failed"); + } catch (Exception ex) { + // expected + } + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(3, rsp.getEventsSize()); + } + + @Test + public void createIndex() throws Exception { + super.createIndex(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_INDEX, firstEventId + 3); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 1); + + String dbName = "default"; + String tableName = "createIndexTableTxn"; + int startTime = (int) (System.currentTimeMillis() / 1000); + List cols = new ArrayList(); + cols.add(new FieldSchema("col1", "int", "")); + SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); + Map params = new HashMap(); + params.put("key", "value"); + StorageDescriptor sd = + new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17, serde, + Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params); + Table table = + new Table(tableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, + null, null, null); + // Event 4 + msClient.createTable(table); + // When hive.metastore.transactional.event.listeners is set, + // a failed event should not create a new notification + DummyRawStoreFailEvent.setEventSucceed(false); + Index index = + new Index("createIndexTable2", null, "default", tableName, startTime, startTime, + "createIndexTable2__createIndexTable2__", sd, emptyParameters, false); + Table indexTable2 = + new Table("createIndexTable2__createIndexTable2__", dbName, "me", startTime, startTime, 0, + sd, null, emptyParameters, null, null, null); + try { + msClient.createIndex(index, indexTable2); + fail("Error: create index should've failed"); + } catch (Exception ex) { + // expected + } + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(4, rsp.getEventsSize()); + } + + @Test + public void dropIndex() throws Exception { + super.dropIndex(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.DROP_INDEX, firstEventId + 4); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_INDEX, firstEventId + 3); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 1); + + String dbName = "default"; + String tableName = "dropIndexTableTxn"; + int startTime = (int) (System.currentTimeMillis() / 1000); + List cols = new ArrayList(); + cols.add(new FieldSchema("col1", "int", "")); + SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); + Map params = new HashMap(); + params.put("key", "value"); + StorageDescriptor sd = + new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17, serde, + Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params); + Table table = + new Table(tableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, + null, null, null); + // Event 5 + msClient.createTable(table); + // When hive.metastore.transactional.event.listeners is set, + // a failed event should not create a new notification + Index index = + new Index("dropIndexTable2", null, "default", tableName, startTime, startTime, + "dropIndexTable__dropIndexTable2__", sd, emptyParameters, false); + Table indexTable2 = + new Table("dropIndexTable__dropIndexTable2__", dbName, "me", startTime, startTime, 0, sd, + null, emptyParameters, null, null, null); + // Event 6, 7 + msClient.createIndex(index, indexTable2); + DummyRawStoreFailEvent.setEventSucceed(false); + try { + // drops index and indexTable + msClient.dropIndex(dbName, tableName, "dropIndex2", true); + fail("Error: drop index should've failed"); + } catch (Exception ex) { + // expected + } + + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(7, rsp.getEventsSize()); + } + + @Test + public void alterIndex() throws Exception { + super.alterIndex(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.ALTER_INDEX, firstEventId + 4); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_INDEX, firstEventId + 3); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 1); + + String indexName = "alterIndexTxn"; + String dbName = "default"; + String tableName = "alterIndexTableTxn"; + String indexTableName = tableName + "__" + indexName + "__"; + int startTime = (int) (System.currentTimeMillis() / 1000); + List cols = new ArrayList(); + cols.add(new FieldSchema("col1", "int", "")); + SerDeInfo serde = new SerDeInfo("serde", "seriallib", null); + Map params = new HashMap(); + params.put("key", "value"); + StorageDescriptor sd = + new StorageDescriptor(cols, "file:/tmp", "input", "output", false, 17, serde, + Arrays.asList("bucketcol"), Arrays.asList(new Order("sortcol", 1)), params); + Table table = + new Table(tableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, + null, null, null); + // Event 5 + msClient.createTable(table); + Index oldIndex = + new Index(indexName, null, "default", tableName, startTime, startTime, indexTableName, sd, + emptyParameters, false); + Table oldIndexTable = + new Table(indexTableName, dbName, "me", startTime, startTime, 0, sd, null, emptyParameters, + null, null, null); + // Event 6, 7 + msClient.createIndex(oldIndex, oldIndexTable); // creates index and index table + Index newIndex = + new Index(indexName, null, "default", tableName, startTime, startTime + 1, indexTableName, + sd, emptyParameters, false); + // Event 4 // When hive.metastore.transactional.event.listeners is set, + // a failed event should not create a new notification + DummyRawStoreFailEvent.setEventSucceed(false); + try { + msClient.alter_index(dbName, tableName, indexName, newIndex); + fail("Error: alter index should've failed"); + } catch (Exception ex) { + // expected + } + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(7, rsp.getEventsSize()); + } + + @Test + public void insertTable() throws Exception { + super.insertTable(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.INSERT, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 1); + } + + @Test + public void insertPartition() throws Exception { + super.insertPartition(); + + // Verify the eventID was passed to the non-transactional listener + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.INSERT, firstEventId + 3); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.ADD_PARTITION, firstEventId + 2); + MockMetaStoreEventListener.popAndVerifyLastEventId(EventMessage.EventType.CREATE_TABLE, firstEventId + 1); + } +}