diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java index fe101d3049..fa7ab25160 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.events.AddForeignKeyEvent; @@ -79,6 +80,10 @@ import org.apache.hadoop.hive.metastore.events.AllocWriteIdEvent; import org.apache.hadoop.hive.metastore.events.ListenerEvent; import org.apache.hadoop.hive.metastore.events.AcidWriteEvent; +import org.apache.hadoop.hive.metastore.events.UpdateTableColumnStatEvent; +import org.apache.hadoop.hive.metastore.events.DeleteTableColumnStatEvent; +import org.apache.hadoop.hive.metastore.events.UpdatePartitionColumnStatEvent; +import org.apache.hadoop.hive.metastore.events.DeletePartitionColumnStatEvent; import org.apache.hadoop.hive.metastore.messaging.AbortTxnMessage; import org.apache.hadoop.hive.metastore.messaging.AcidWriteMessage; import org.apache.hadoop.hive.metastore.messaging.AddForeignKeyMessage; @@ -107,6 +112,10 @@ import org.apache.hadoop.hive.metastore.messaging.MessageSerializer; import org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage; import org.apache.hadoop.hive.metastore.messaging.PartitionFiles; +import org.apache.hadoop.hive.metastore.messaging.UpdateTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.DeleteTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.UpdatePartitionColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.DeletePartitionColumnStatMessage; import org.apache.hadoop.hive.metastore.tools.SQLGenerator; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; @@ -744,6 +753,69 @@ public void onAcidWrite(AcidWriteEvent acidWriteEvent, Connection dbConn, SQLGen } } + @Override + public void onUpdateTableColumnStat(UpdateTableColumnStatEvent updateTableColumnStatEvent) throws MetaException { + UpdateTableColumnStatMessage msg = MessageBuilder.getInstance() + .buildUpdateTableColumnStatMessage(updateTableColumnStatEvent.getColStats(), + updateTableColumnStatEvent.getTableParameters(), + updateTableColumnStatEvent.getValidWriteIds(), updateTableColumnStatEvent.getWriteId()); + NotificationEvent event = new NotificationEvent(0, now(), EventType.UPDATE_TABLE_COLUMN_STAT.toString(), + msgEncoder.getSerializer().serialize(msg)); + ColumnStatisticsDesc statDesc = updateTableColumnStatEvent.getColStats().getStatsDesc(); + event.setCatName(statDesc.isSetCatName() ? statDesc.getCatName() : DEFAULT_CATALOG_NAME); + event.setDbName(statDesc.getDbName()); + event.setTableName(statDesc.getTableName()); + process(event, updateTableColumnStatEvent); + } + + @Override + public void onDeleteTableColumnStat(DeleteTableColumnStatEvent deleteTableColumnStatEvent) throws MetaException { + DeleteTableColumnStatMessage msg = MessageBuilder.getInstance() + .buildDeleteTableColumnStatMessage(deleteTableColumnStatEvent.getDBName(), + deleteTableColumnStatEvent.getColName()); + NotificationEvent event = new NotificationEvent(0, now(), EventType.DELETE_TABLE_COLUMN_STAT.toString(), + msgEncoder.getSerializer().serialize(msg)); + event.setCatName(deleteTableColumnStatEvent.getCatName()); + event.setDbName(deleteTableColumnStatEvent.getDBName()); + event.setTableName(deleteTableColumnStatEvent.getTableName()); + process(event, deleteTableColumnStatEvent); + } + + @Override + public void onUpdatePartitionColumnStat(UpdatePartitionColumnStatEvent updatePartColStatEvent) throws MetaException { + UpdatePartitionColumnStatMessage msg = MessageBuilder.getInstance() + .buildUpdatePartitionColumnStatMessage(updatePartColStatEvent.getPartColStats(), + updatePartColStatEvent.getPartVals(), + updatePartColStatEvent.getPartParameters(), + updatePartColStatEvent.getValidWriteIds(), updatePartColStatEvent.getWriteId()); + NotificationEvent event = new NotificationEvent(0, now(), EventType.UPDATE_PARTITION_COLUMN_STAT.toString(), + msgEncoder.getSerializer().serialize(msg)); + ColumnStatisticsDesc statDesc = updatePartColStatEvent.getPartColStats().getStatsDesc(); + event.setCatName(statDesc.isSetCatName() ? statDesc.getCatName() : DEFAULT_CATALOG_NAME); + event.setDbName(statDesc.getDbName()); + event.setTableName(statDesc.getTableName()); + process(event, updatePartColStatEvent); + } + + @Override + public void onDeletePartitionColumnStat(DeletePartitionColumnStatEvent deletePartColStatEvent) throws MetaException { + DeletePartitionColumnStatMessage msg = MessageBuilder.getInstance() + .buildDeletePartitionColumnStatMessage(deletePartColStatEvent.getDBName(), + deletePartColStatEvent.getColName(), deletePartColStatEvent.getPartName(), + deletePartColStatEvent.getPartVals()); + NotificationEvent event = new NotificationEvent(0, now(), EventType.DELETE_PARTITION_COLUMN_STAT.toString(), + msgEncoder.getSerializer().serialize(msg)); + event.setCatName(deletePartColStatEvent.getCatName()); + event.setDbName(deletePartColStatEvent.getDBName()); + event.setTableName(deletePartColStatEvent.getTableName()); + process(event, deletePartColStatEvent); + } + + @Override + public boolean doesAddEventsToNotificationLogTable() { + return true; + } + private int now() { long millis = System.currentTimeMillis(); millis /= 1000; diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java index d4b7b02139..be4f9ae62a 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java @@ -1250,7 +1250,7 @@ public void sqlInsertTable() throws Exception { // Get notifications from metastore NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(6, rsp.getEventsSize()); + assertEquals(7, rsp.getEventsSize()); NotificationEvent event = rsp.getEvents().get(0); assertEquals(firstEventId + 1, event.getEventId()); assertEquals(EventType.CREATE_TABLE.toString(), event.getEventType()); @@ -1261,14 +1261,14 @@ public void sqlInsertTable() throws Exception { // Parse the message field verifyInsert(event, defaultDbName, tblName); - event = rsp.getEvents().get(4); - assertEquals(firstEventId + 5, event.getEventId()); - assertEquals(EventType.ALTER_TABLE.toString(), event.getEventType()); - event = rsp.getEvents().get(5); assertEquals(firstEventId + 6, event.getEventId()); + assertEquals(EventType.ALTER_TABLE.toString(), event.getEventType()); + + event = rsp.getEvents().get(6); + assertEquals(firstEventId + 7, event.getEventId()); assertEquals(EventType.DROP_TABLE.toString(), event.getEventType()); - testEventCounts(defaultDbName, firstEventId, null, null, 6); + testEventCounts(defaultDbName, firstEventId, null, null, 7); } @Test @@ -1285,7 +1285,7 @@ public void sqlCTAS() throws Exception { // Get notifications from metastore NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(6, rsp.getEventsSize()); + assertEquals(7, rsp.getEventsSize()); NotificationEvent event = rsp.getEvents().get(0); assertEquals(firstEventId + 1, event.getEventId()); assertEquals(EventType.CREATE_TABLE.toString(), event.getEventType()); @@ -1296,10 +1296,10 @@ public void sqlCTAS() throws Exception { // Parse the message field verifyInsert(event, null, sourceTblName); - event = rsp.getEvents().get(4); - assertEquals(firstEventId + 5, event.getEventId()); + event = rsp.getEvents().get(5); + assertEquals(firstEventId + 6, event.getEventId()); assertEquals(EventType.CREATE_TABLE.toString(), event.getEventType()); - testEventCounts(defaultDbName, firstEventId, null, null, 6); + testEventCounts(defaultDbName, firstEventId, null, null, 7); } @Test @@ -1349,9 +1349,9 @@ public void sqlInsertPartition() throws Exception { // Event 9, 10 driver.run("alter table " + tblName + " add partition (ds = 'yesterday')"); - testEventCounts(defaultDbName, firstEventId, null, null, 10); + testEventCounts(defaultDbName, firstEventId, null, null, 13); // Test a limit higher than available events - testEventCounts(defaultDbName, firstEventId, null, 100, 10); + testEventCounts(defaultDbName, firstEventId, null, 100, 13); // Test toEventId lower than current eventId testEventCounts(defaultDbName, firstEventId, (long) firstEventId + 5, null, 5); @@ -1371,84 +1371,89 @@ public void sqlInsertPartition() throws Exception { // Get notifications from metastore NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); - assertEquals(24, rsp.getEventsSize()); + assertEquals(31, rsp.getEventsSize()); + NotificationEvent event = rsp.getEvents().get(1); assertEquals(firstEventId + 2, event.getEventId()); assertEquals(EventType.ADD_PARTITION.toString(), event.getEventType()); event = rsp.getEvents().get(3); assertEquals(firstEventId + 4, event.getEventId()); + assertEquals(EventType.UPDATE_PARTITION_COLUMN_STAT.toString(), event.getEventType()); + + event = rsp.getEvents().get(4); + assertEquals(firstEventId + 5, event.getEventId()); assertEquals(EventType.INSERT.toString(), event.getEventType()); // Parse the message field verifyInsert(event, null, tblName); - event = rsp.getEvents().get(6); - assertEquals(firstEventId + 7, event.getEventId()); + event = rsp.getEvents().get(8); + assertEquals(firstEventId + 9, event.getEventId()); assertEquals(EventType.INSERT.toString(), event.getEventType()); // Parse the message field verifyInsert(event, null, tblName); - event = rsp.getEvents().get(9); - assertEquals(firstEventId + 10, event.getEventId()); + event = rsp.getEvents().get(12); + assertEquals(firstEventId + 13, event.getEventId()); assertEquals(EventType.ADD_PARTITION.toString(), event.getEventType()); - event = rsp.getEvents().get(10); - assertEquals(firstEventId + 11, event.getEventId()); + event = rsp.getEvents().get(13); + assertEquals(firstEventId + 14, event.getEventId()); assertEquals(EventType.INSERT.toString(), event.getEventType()); // Parse the message field verifyInsert(event, null, tblName); - event = rsp.getEvents().get(13); - assertEquals(firstEventId + 14, event.getEventId()); + event = rsp.getEvents().get(17); + assertEquals(firstEventId + 18, event.getEventId()); assertEquals(EventType.INSERT.toString(), event.getEventType()); // Parse the message field verifyInsert(event, null, tblName); - event = rsp.getEvents().get(16); - assertEquals(firstEventId + 17, event.getEventId()); + event = rsp.getEvents().get(21); + assertEquals(firstEventId + 22, event.getEventId()); assertEquals(EventType.ADD_PARTITION.toString(), event.getEventType()); - event = rsp.getEvents().get(18); - assertEquals(firstEventId + 19, event.getEventId()); + event = rsp.getEvents().get(24); + assertEquals(firstEventId + 25, event.getEventId()); assertEquals(EventType.DROP_PARTITION.toString(), event.getEventType()); - event = rsp.getEvents().get(19); - assertEquals(firstEventId + 20, event.getEventId()); + event = rsp.getEvents().get(25); + assertEquals(firstEventId + 26, event.getEventId()); assertEquals(EventType.ADD_PARTITION.toString(), event.getEventType()); - event = rsp.getEvents().get(20); - assertEquals(firstEventId + 21, event.getEventId()); + event = rsp.getEvents().get(26); + assertEquals(firstEventId + 27, event.getEventId()); assertEquals(EventType.ALTER_PARTITION.toString(), event.getEventType()); assertTrue(event.getMessage().matches(".*\"ds\":\"todaytwo\".*")); // Test fromEventId different from the very first - testEventCounts(defaultDbName, event.getEventId(), null, null, 3); + testEventCounts(defaultDbName, event.getEventId(), null, null, 4); - event = rsp.getEvents().get(21); - assertEquals(firstEventId + 22, event.getEventId()); + event = rsp.getEvents().get(28); + assertEquals(firstEventId + 29, event.getEventId()); assertEquals(EventType.INSERT.toString(), event.getEventType()); // replace-overwrite introduces no new files assertTrue(event.getMessage().matches(".*\"files\":\\[\\].*")); - event = rsp.getEvents().get(22); - assertEquals(firstEventId + 23, event.getEventId()); + event = rsp.getEvents().get(29); + assertEquals(firstEventId + 30, event.getEventId()); assertEquals(EventType.ALTER_PARTITION.toString(), event.getEventType()); assertTrue(event.getMessage().matches(".*\"ds\":\"todaytwo\".*")); - event = rsp.getEvents().get(23); - assertEquals(firstEventId + 24, event.getEventId()); + event = rsp.getEvents().get(30); + assertEquals(firstEventId + 31, event.getEventId()); assertEquals(EventType.ALTER_PARTITION.toString(), event.getEventType()); assertTrue(event.getMessage().matches(".*\"ds\":\"todaytwo\".*")); - testEventCounts(defaultDbName, firstEventId, null, null, 24); + testEventCounts(defaultDbName, firstEventId, null, null, 31); // Test a limit within the available events testEventCounts(defaultDbName, firstEventId, null, 10, 10); // Test toEventId greater than current eventId - testEventCounts(defaultDbName, firstEventId, (long) firstEventId + 100, null, 24); + testEventCounts(defaultDbName, firstEventId, (long) firstEventId + 100, null, 31); // Test toEventId greater than current eventId with some limit within available events testEventCounts(defaultDbName, firstEventId, (long) firstEventId + 100, 10, 10); // Test toEventId greater than current eventId with some limit beyond available events - testEventCounts(defaultDbName, firstEventId, (long) firstEventId + 100, 50, 24); + testEventCounts(defaultDbName, firstEventId, (long) firstEventId + 100, 50, 31); } private void verifyInsert(NotificationEvent event, String dbName, String tblName) throws Exception { diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestCachedStoreUpdateUsingEvents.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestCachedStoreUpdateUsingEvents.java new file mode 100644 index 0000000000..83f12a5fd9 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestCachedStoreUpdateUsingEvents.java @@ -0,0 +1,535 @@ +package org.apache.hadoop.hive.metastore.cache; + +import java.util.*; + +import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.*; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; +import org.apache.hadoop.hive.metastore.api.*; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.utils.FileUtils; +import org.apache.hadoop.util.StringUtils; +import org.apache.hive.hcatalog.listener.DbNotificationListener; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import jline.internal.Log; + +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; + +public class TestCachedStoreUpdateUsingEvents { + + private RawStore rawStore; + private SharedCache sharedCache; + private Configuration conf; + private HiveMetaStore.HMSHandler hmsHandler; + + @Before + public void setUp() throws Exception { + conf = MetastoreConf.newMetastoreConf(); + MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST, true); + // Disable memory estimation for this test class + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CACHED_RAW_STORE_MAX_CACHE_MEMORY, "-1Kb"); + MetastoreConf.setVar(conf, ConfVars.TRANSACTIONAL_EVENT_LISTENERS, DbNotificationListener.class.getName()); + MetastoreConf.setVar(conf, ConfVars.RAW_STORE_IMPL, "org.apache.hadoop.hive.metastore.cache.CachedStore"); + MetastoreConf.setBoolVar(conf, ConfVars.METASTORE_CACHE_CAN_USE_EVENT, true); + MetaStoreTestUtils.setConfForStandloneMode(conf); + + hmsHandler = new HiveMetaStore.HMSHandler("testCachedStore", conf, true); + + rawStore = hmsHandler.getMS(); + sharedCache = CachedStore.getSharedCache(); + + // Stop the CachedStore cache update service. We'll start it explicitly to control the test + CachedStore.stopCacheUpdateService(1); + + // Create the 'hive' catalog with new warehouse directory + HiveMetaStore.HMSHandler.createDefaultCatalog(rawStore, new Warehouse(conf)); + } + + private Database createTestDb(String dbName, String dbOwner) { + String dbDescription = dbName; + String dbLocation = "file:/tmp"; + Map dbParams = new HashMap<>(); + Database db = new Database(dbName, dbDescription, dbLocation, dbParams); + db.setOwnerName(dbOwner); + db.setOwnerType(PrincipalType.USER); + db.setCatalogName(DEFAULT_CATALOG_NAME); + return db; + } + + private Table createTestTblParam(String dbName, String tblName, String tblOwner, + List cols, List ptnCols, Map tblParams) { + String serdeLocation = "file:/tmp"; + Map serdeParams = new HashMap<>(); + SerDeInfo serdeInfo = new SerDeInfo("serde", "seriallib", new HashMap<>()); + StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, + serdeInfo, null, null, serdeParams); + sd.setStoredAsSubDirectories(false); + Table tbl = new Table(tblName, dbName, tblOwner, 0, 0, 0, sd, ptnCols, tblParams, null, null, + TableType.MANAGED_TABLE.toString()); + tbl.setCatName(DEFAULT_CATALOG_NAME); + return tbl; + } + + private Table createTestTbl(String dbName, String tblName, String tblOwner, + List cols, List ptnCols) { + return createTestTblParam(dbName, tblName, tblOwner, cols, ptnCols, new HashMap<>()); + } + + private void compareTables(Table tbl1, Table tbl2) { + Assert.assertEquals(tbl1.getDbName(), tbl2.getDbName()); + Assert.assertEquals(tbl1.getSd(), tbl2.getSd()); + Assert.assertEquals(tbl1.getParameters(), tbl2.getParameters()); + Assert.assertEquals(tbl1.getTableName(), tbl2.getTableName()); + Assert.assertEquals(tbl1.getCatName(), tbl2.getCatName()); + Assert.assertEquals(tbl1.getCreateTime(), tbl2.getCreateTime()); + Assert.assertEquals(tbl1.getCreationMetadata(), tbl2.getCreationMetadata()); + Assert.assertEquals(tbl1.getId(), tbl2.getId()); + } + + private void comparePartitions(Partition part1, Partition part2) { + Assert.assertEquals(part1.getParameters(), part2.getParameters()); + Assert.assertEquals(part1.getCatName(), part2.getCatName()); + Assert.assertEquals(part1.getCreateTime(), part2.getCreateTime()); + Assert.assertEquals(part1.getTableName(), part2.getTableName()); + Assert.assertEquals(part1.getDbName().toLowerCase(), part2.getDbName().toLowerCase()); + Assert.assertEquals(part1.getLastAccessTime(), part2.getLastAccessTime()); + } + + @Test + public void testDatabaseOpsForUpdateUsingEvents() throws Exception { + RawStore rawStore = hmsHandler.getMS(); + + // Prewarm CachedStore + CachedStore.setCachePrewarmedState(false); + CachedStore.prewarm(rawStore); + + // Add a db via rawStore + String dbName = "testDatabaseOps"; + String dbOwner = "user1"; + Database db = createTestDb(dbName, dbOwner); + + hmsHandler.create_database(db); + db = rawStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); + + // Read database via CachedStore + Database dbRead = sharedCache.getDatabaseFromCache(DEFAULT_CATALOG_NAME, dbName); + Assert.assertEquals(db, dbRead); + + // Add another db via rawStore + final String dbName1 = "testDatabaseOps1"; + Database db1 = createTestDb(dbName1, dbOwner); + hmsHandler.create_database(db1); + db1 = rawStore.getDatabase(DEFAULT_CATALOG_NAME, dbName1); + + // Read database via CachedStore + dbRead = sharedCache.getDatabaseFromCache(DEFAULT_CATALOG_NAME, dbName1); + Assert.assertEquals(db1, dbRead); + + // Alter the db via rawStore (can only alter owner or parameters) + dbOwner = "user2"; + Database newdb = new Database(db); + newdb.setOwnerName(dbOwner); + hmsHandler.alter_database(dbName, newdb); + newdb = rawStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); + + // Read db via cachedStore + dbRead = sharedCache.getDatabaseFromCache(DEFAULT_CATALOG_NAME, dbName); + Assert.assertEquals(newdb, dbRead); + + // Add another db via rawStore + final String dbName2 = "testDatabaseOps2"; + Database db2 = createTestDb(dbName2, dbOwner); + hmsHandler.create_database(db2); + db2 = rawStore.getDatabase(DEFAULT_CATALOG_NAME, dbName2); + + // Alter db "testDatabaseOps" via rawStore + dbOwner = "user1"; + newdb = new Database(db); + newdb.setOwnerName(dbOwner); + hmsHandler.alter_database(dbName, newdb); + newdb = rawStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); + + // Drop db "testDatabaseOps1" via rawStore + Database dropDb = rawStore.getDatabase(DEFAULT_CATALOG_NAME, dbName1); + hmsHandler.drop_database(dbName1, true, true); + + // Read the newly added db via CachedStore + dbRead = sharedCache.getDatabaseFromCache(DEFAULT_CATALOG_NAME, dbName2); + Assert.assertEquals(db2, dbRead); + + // Read the altered db via CachedStore (altered user from "user2" to "user1") + dbRead = sharedCache.getDatabaseFromCache(DEFAULT_CATALOG_NAME, dbName); + Assert.assertEquals(newdb, dbRead); + + // Try to read the dropped db after cache update + dbRead = sharedCache.getDatabaseFromCache(DEFAULT_CATALOG_NAME, dbName1); + Assert.assertEquals(null, dbRead); + + // Clean up + hmsHandler.drop_database(dbName, true, true); + hmsHandler.drop_database(dbName2, true, true); + sharedCache.getDatabaseCache().clear(); + sharedCache.getTableCache().clear(); + sharedCache.getSdCache().clear(); + } + + @Test + public void testTableOpsForUpdateUsingEvents() throws Exception { + long lastEventId = -1; + RawStore rawStore = hmsHandler.getMS(); + + // Add a db via rawStore + String dbName = "test_table_ops"; + String dbOwner = "user1"; + Database db = createTestDb(dbName, dbOwner); + hmsHandler.create_database(db); + db = rawStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); + + // Prewarm CachedStore + CachedStore.setCachePrewarmedState(false); + CachedStore.prewarm(rawStore); + + // Add a table via rawStore + String tblName = "tbl"; + String tblOwner = "user1"; + FieldSchema col1 = new FieldSchema("col1", "int", "integer column"); + FieldSchema col2 = new FieldSchema("col2", "string", "string column"); + List cols = new ArrayList(); + cols.add(col1); + cols.add(col2); + List ptnCols = new ArrayList(); + Table tbl = createTestTbl(dbName, tblName, tblOwner, cols, ptnCols); + hmsHandler.create_table(tbl); + tbl = rawStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName); + + // Read database, table via CachedStore + Database dbRead= sharedCache.getDatabaseFromCache(DEFAULT_CATALOG_NAME, dbName); + Assert.assertEquals(db, dbRead); + Table tblRead = sharedCache.getTableFromCache(DEFAULT_CATALOG_NAME, dbName, tblName); + compareTables(tblRead, tbl); + + // Add a new table via rawStore + String tblName2 = "tbl2"; + Table tbl2 = createTestTbl(dbName, tblName2, tblOwner, cols, ptnCols); + hmsHandler.create_table(tbl2); + tbl2 = rawStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName2); + + // Alter table "tbl" via rawStore + tblOwner = "role1"; + Table newTable = new Table(tbl); + newTable.setOwner(tblOwner); + newTable.setOwnerType(PrincipalType.ROLE); + hmsHandler.alter_table(dbName, tblName, newTable); + newTable = rawStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName); + + Assert.assertEquals("Owner of the table did not change.", tblOwner, newTable.getOwner()); + Assert.assertEquals("Owner type of the table did not change", PrincipalType.ROLE, newTable.getOwnerType()); + + // Drop table "tbl2" via rawStore + hmsHandler.drop_table(dbName, tblName2, true); + + // Read the altered "tbl" via CachedStore + tblRead = sharedCache.getTableFromCache(DEFAULT_CATALOG_NAME, dbName, tblName); + compareTables(tblRead, newTable); + + // Try to read the dropped "tbl2" via CachedStore (should throw exception) + tblRead = sharedCache.getTableFromCache(DEFAULT_CATALOG_NAME, dbName, tblName2); + Assert.assertNull(tblRead); + + // Clean up + hmsHandler.drop_database(dbName, true, true); + + tblRead = sharedCache.getTableFromCache(DEFAULT_CATALOG_NAME, dbName, tblName2); + Assert.assertNull(tblRead); + + tblRead = sharedCache.getTableFromCache(DEFAULT_CATALOG_NAME, dbName, tblName); + Assert.assertNull(tblRead); + + sharedCache.getDatabaseCache().clear(); + sharedCache.getTableCache().clear(); + sharedCache.getSdCache().clear(); + } + + @Test + public void testPartitionOpsForUpdateUsingEvents() throws Exception { + long lastEventId = -1; + RawStore rawStore = hmsHandler.getMS(); + + // Add a db via rawStore + String dbName = "test_partition_ops"; + String dbOwner = "user1"; + Database db = createTestDb(dbName, dbOwner); + hmsHandler.create_database(db); + db = rawStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); + + // Add a table via rawStore + String tblName = "tbl"; + String tblOwner = "user1"; + FieldSchema col1 = new FieldSchema("col1", "int", "integer column"); + FieldSchema col2 = new FieldSchema("col2", "string", "string column"); + List cols = new ArrayList(); + cols.add(col1); + cols.add(col2); + FieldSchema ptnCol1 = new FieldSchema("part1", "string", "string partition column"); + List ptnCols = new ArrayList(); + ptnCols.add(ptnCol1); + Table tbl = createTestTbl(dbName, tblName, tblOwner, cols, ptnCols); + hmsHandler.create_table(tbl); + tbl = rawStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName); + + // Prewarm CachedStore + CachedStore.setCachePrewarmedState(false); + CachedStore.prewarm(rawStore); + + final String ptnColVal1 = "aaa"; + Map partParams = new HashMap(); + Partition ptn1 = + new Partition(Arrays.asList(ptnColVal1), dbName, tblName, 0, 0, tbl.getSd(), partParams); + ptn1.setCatName(DEFAULT_CATALOG_NAME); + hmsHandler.add_partition(ptn1); + ptn1 = rawStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal1)); + + final String ptnColVal2 = "bbb"; + Partition ptn2 = + new Partition(Arrays.asList(ptnColVal2), dbName, tblName, 0, 0, tbl.getSd(), partParams); + ptn2.setCatName(DEFAULT_CATALOG_NAME); + hmsHandler.add_partition(ptn2); + ptn2 = rawStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal2)); + + // Read database, table, partition via CachedStore + Database dbRead = sharedCache.getDatabaseFromCache(DEFAULT_CATALOG_NAME.toLowerCase(), dbName.toLowerCase()); + Assert.assertEquals(db, dbRead); + Table tblRead = sharedCache.getTableFromCache(DEFAULT_CATALOG_NAME.toLowerCase(), dbName.toLowerCase(), tblName.toLowerCase()); + compareTables(tbl, tblRead); + Partition ptn1Read = sharedCache.getPartitionFromCache(DEFAULT_CATALOG_NAME.toLowerCase(), dbName.toLowerCase(), tblName.toLowerCase(), Arrays.asList(ptnColVal1)); + comparePartitions(ptn1, ptn1Read); + Partition ptn2Read = sharedCache.getPartitionFromCache(DEFAULT_CATALOG_NAME.toLowerCase(), dbName.toLowerCase(), tblName.toLowerCase(), Arrays.asList(ptnColVal2)); + comparePartitions(ptn2, ptn2Read); + + // Add a new partition via rawStore + final String ptnColVal3 = "ccc"; + Partition ptn3 = + new Partition(Arrays.asList(ptnColVal3), dbName, tblName, 0, 0, tbl.getSd(), partParams); + ptn3.setCatName(DEFAULT_CATALOG_NAME); + hmsHandler.add_partition(ptn3); + ptn3 = rawStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal3)); + + // Alter an existing partition ("aaa") via rawStore + ptn1 = rawStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal1)); + final String ptnColVal1Alt = "aaa"; + Partition ptn1Atl = + new Partition(Arrays.asList(ptnColVal1Alt), dbName, tblName, 0, 0, tbl.getSd(), partParams); + ptn1Atl.setCatName(DEFAULT_CATALOG_NAME); + hmsHandler.alter_partitions(dbName, tblName, Arrays.asList(ptn1Atl)); + ptn1Atl = rawStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal1Alt)); + + // Drop an existing partition ("bbb") via rawStore + Partition ptnDrop = rawStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal2)); + hmsHandler.drop_partition(dbName, tblName, Arrays.asList(ptnColVal2), false); + + // Read the newly added partition via CachedStore + Partition ptnRead = sharedCache.getPartitionFromCache(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal3)); + comparePartitions(ptn3, ptnRead); + + // Read the altered partition via CachedStore + ptnRead = sharedCache.getPartitionFromCache(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal1Alt)); + Assert.assertEquals(ptn1Atl.getParameters(), ptnRead.getParameters()); + + ptnRead = sharedCache.getPartitionFromCache(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal2)); + Assert.assertEquals(null, ptnRead); + + // Drop table "tbl" via rawStore, it should remove the partition also + hmsHandler.drop_table(dbName, tblName, true); + + ptnRead = sharedCache.getPartitionFromCache(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal1Alt)); + Assert.assertEquals(null, ptnRead); + + ptnRead = sharedCache.getPartitionFromCache(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal3)); + Assert.assertEquals(null, ptnRead); + + // Clean up + rawStore.dropDatabase(DEFAULT_CATALOG_NAME, dbName); + sharedCache.getDatabaseCache().clear(); + sharedCache.getTableCache().clear(); + sharedCache.getSdCache().clear(); + } + + @Test + public void testTableColumnStatistics() throws Throwable { + String dbName = "column_stats_test_db"; + String tblName = "tbl"; + String typeName = "person"; + String tblOwner = "testowner"; + int lastAccessed = 6796; + String dbOwner = "user1"; + + // Add a db via rawStore + Database db = createTestDb(dbName, dbOwner); + hmsHandler.create_database(db); + db = rawStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); + + Map tableParams = new HashMap<>(); + tableParams.put("test_param_1", "hi"); + tableParams.put("test_param_2", "50"); + + // Add a table via rawStore + List cols = new ArrayList(); + cols.add(new FieldSchema("income", "int", "integer column")); + cols.add(new FieldSchema("name", "string", "string column")); + + List ptnCols = new ArrayList(); + ptnCols.add(new FieldSchema("ds", "string", "string partition column")); + ptnCols.add(new FieldSchema("hr", "int", "integer partition column")); + + Table tbl = createTestTblParam(dbName, tblName, tblOwner, cols, null, tableParams); + hmsHandler.create_table(tbl); + + // Prewarm CachedStore + CachedStore.setCachePrewarmedState(false); + CachedStore.prewarm(rawStore); + + // Create a ColumnStatistics Obj + String[] colName = new String[]{"income", "name"}; + double lowValue = 50000.21; + double highValue = 1200000.4525; + long numNulls = 3; + long numDVs = 22; + double avgColLen = 50.30; + long maxColLen = 102; + String[] colType = new String[] {"double", "string"}; + boolean isTblLevel = true; + String partName = null; + List statsObjs = new ArrayList<>(); + + ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(); + statsDesc.setDbName(dbName); + statsDesc.setTableName(tblName); + statsDesc.setIsTblLevel(isTblLevel); + statsDesc.setPartName(partName); + + ColumnStatisticsObj statsObj = new ColumnStatisticsObj(); + statsObj.setColName(colName[0]); + statsObj.setColType(colType[0]); + + ColumnStatisticsData statsData = new ColumnStatisticsData(); + DoubleColumnStatsData numericStats = new DoubleColumnStatsData(); + statsData.setDoubleStats(numericStats); + + statsData.getDoubleStats().setHighValue(highValue); + statsData.getDoubleStats().setLowValue(lowValue); + statsData.getDoubleStats().setNumDVs(numDVs); + statsData.getDoubleStats().setNumNulls(numNulls); + + statsObj.setStatsData(statsData); + statsObjs.add(statsObj); + + statsObj = new ColumnStatisticsObj(); + statsObj.setColName(colName[1]); + statsObj.setColType(colType[1]); + + statsData = new ColumnStatisticsData(); + StringColumnStatsData stringStats = new StringColumnStatsData(); + statsData.setStringStats(stringStats); + statsData.getStringStats().setAvgColLen(avgColLen); + statsData.getStringStats().setMaxColLen(maxColLen); + statsData.getStringStats().setNumDVs(numDVs); + statsData.getStringStats().setNumNulls(numNulls); + + statsObj.setStatsData(statsData); + statsObjs.add(statsObj); + + ColumnStatistics colStats = new ColumnStatistics(); + colStats.setStatsDesc(statsDesc); + colStats.setStatsObj(statsObjs); + + // write stats objs persistently + hmsHandler.update_table_column_statistics(colStats); + + ColumnStatisticsObj colStatsCache = sharedCache.getTableColStatsFromCache(DEFAULT_CATALOG_NAME, + dbName, tblName, Lists.newArrayList(colName[0])).get(0); + Assert.assertEquals(colStatsCache.getColName(), colName[0]); + Assert.assertEquals(colStatsCache.getStatsData().getDoubleStats().getLowValue(), lowValue, 0.01); + Assert.assertEquals(colStatsCache.getStatsData().getDoubleStats().getHighValue(), highValue, 0.01); + Assert.assertEquals(colStatsCache.getStatsData().getDoubleStats().getNumNulls(), numNulls); + Assert.assertEquals(colStatsCache.getStatsData().getDoubleStats().getNumDVs(), numDVs); + + // test delete column stats; if no col name is passed all column stats associated with the + // table is deleted + boolean status = hmsHandler.delete_table_column_statistics(dbName, tblName, null); + Assert.assertEquals(status, true); + + Assert.assertEquals(sharedCache.getTableColStatsFromCache(DEFAULT_CATALOG_NAME, + dbName, tblName, Lists.newArrayList(colName[0])).isEmpty(), true); + + tblName = "tbl_part"; + cols = new ArrayList<>(); + cols.add(new FieldSchema(colName[0], "int", null)); + List partCols = new ArrayList<>(); + partCols.add(new FieldSchema("col", "int", null)); + StorageDescriptor sd = + new StorageDescriptor(cols, null, "input", "output", false, + 0, new SerDeInfo("serde", "seriallib", new HashMap<>()), + null, null, null); + + tbl = new Table(tblName, dbName, null, 0, 0, 0, sd, partCols, new HashMap<>(), + null, null, TableType.MANAGED_TABLE.toString()); + tbl.setCatName(DEFAULT_CATALOG_NAME); + + hmsHandler.create_table(tbl); + + List partVals1 = new ArrayList<>(); + partVals1.add("1"); + List partVals2 = new ArrayList<>(); + partVals2.add("2"); + + Partition ptn1 = + new Partition(partVals1, dbName, tblName, 0, 0, sd, new HashMap<>()); + ptn1.setCatName(DEFAULT_CATALOG_NAME); + hmsHandler.add_partition(ptn1); + Partition ptn2 = + new Partition(partVals2, dbName, tblName, 0, 0, sd, new HashMap<>()); + ptn2.setCatName(DEFAULT_CATALOG_NAME); + hmsHandler.add_partition(ptn2); + + List partitions = hmsHandler.get_partition_names(dbName, tblName, (short)-1); + partName = partitions.get(0); + isTblLevel = false; + + // create a new columnstatistics desc to represent partition level column stats + statsDesc = new ColumnStatisticsDesc(); + statsDesc.setDbName(dbName); + statsDesc.setTableName(tblName); + statsDesc.setPartName(partName); + statsDesc.setIsTblLevel(isTblLevel); + + colStats = new ColumnStatistics(); + colStats.setStatsDesc(statsDesc); + colStats.setStatsObj(statsObjs); + + hmsHandler.update_partition_column_statistics(colStats); + ColumnStatisticsObj colStats2 = sharedCache.getPartitionColStatsFromCache(DEFAULT_CATALOG_NAME, dbName, tblName, + CachedStore.partNameToVals(partName), colName[1]); + // compare stats obj to ensure what we get is what we wrote + Assert.assertEquals(colStats.getStatsDesc().getPartName(), partName); + Assert.assertEquals(colStats2.getColName(), colName[1]); + Assert.assertEquals(colStats2.getStatsData().getStringStats().getMaxColLen(), maxColLen); + Assert.assertEquals(colStats2.getStatsData().getStringStats().getAvgColLen(), avgColLen, 0.01); + Assert.assertEquals(colStats2.getStatsData().getStringStats().getNumNulls(), numNulls); + Assert.assertEquals(colStats2.getStatsData().getStringStats().getNumDVs(), numDVs); + + // test stats deletion at partition level + hmsHandler.delete_partition_column_statistics(dbName, tblName, partName, colName[1]); + + colStats2 = sharedCache.getPartitionColStatsFromCache(DEFAULT_CATALOG_NAME, dbName, tblName, + CachedStore.partNameToVals(partName), colName[1]); + Assert.assertEquals(colStats2, null); + } +} diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java index 822532cf92..4472a61a0e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcidTables.java @@ -496,7 +496,7 @@ public void testOpenTxnEvent() throws Throwable { primary.dump(primaryDbName, bootStrapDump.lastReplicationId); long lastReplId = Long.parseLong(bootStrapDump.lastReplicationId); - primary.testEventCounts(primaryDbName, lastReplId, null, null, 21); + primary.testEventCounts(primaryDbName, lastReplId, null, null, 22); // Test load replica.load(replicatedDbName, incrementalDump.dumpLocation) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/DumpType.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/DumpType.java index 2e42267e82..4ec1bac6ee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/DumpType.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/DumpType.java @@ -41,6 +41,10 @@ import org.apache.hadoop.hive.ql.parse.repl.load.message.CommitTxnHandler; import org.apache.hadoop.hive.ql.parse.repl.load.message.AbortTxnHandler; import org.apache.hadoop.hive.ql.parse.repl.load.message.AllocWriteIdHandler; +import org.apache.hadoop.hive.ql.parse.repl.load.message.UpdateTableColStatHandler; +import org.apache.hadoop.hive.ql.parse.repl.load.message.DeleteTableColStatHandler; +import org.apache.hadoop.hive.ql.parse.repl.load.message.UpdatePartColStatHandler; +import org.apache.hadoop.hive.ql.parse.repl.load.message.DeletePartColStatHandler; public enum DumpType { @@ -211,6 +215,30 @@ public MessageHandler handler() { public MessageHandler handler() { return new AllocWriteIdHandler(); } + }, + EVENT_UPDATE_TABLE_COL_STAT("EVENT_UPDATE_TABLE_COL_STAT") { + @Override + public MessageHandler handler() { + return new UpdateTableColStatHandler(); + } + }, + EVENT_DELETE_TABLE_COL_STAT("EVENT_DELETE_TABLE_COL_STAT") { + @Override + public MessageHandler handler() { + return new DeleteTableColStatHandler(); + } + }, + EVENT_UPDATE_PART_COL_STAT("EVENT_UPDATE_PART_COL_STAT") { + @Override + public MessageHandler handler() { + return new UpdatePartColStatHandler(); + } + }, + EVENT_DELETE_PART_COL_STAT("EVENT_DELETE_PART_COL_STAT") { + @Override + public MessageHandler handler() { + return new DeletePartColStatHandler(); + } }; String type = null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/DeletePartColStatHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/DeletePartColStatHandler.java new file mode 100644 index 0000000000..7edbcff43c --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/DeletePartColStatHandler.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.dump.events; + +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.metastore.messaging.DeletePartitionColumnStatMessage; +import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; + +class DeletePartColStatHandler extends AbstractEventHandler { + + DeletePartColStatHandler(NotificationEvent event) { + super(event); + } + + @Override + DeletePartitionColumnStatMessage eventMessage(String stringRepresentation) { + return deserializer.getDeletePartitionColumnStatMessage(stringRepresentation); + } + + @Override + public void handle(Context withinContext) throws Exception { + LOG.info("Processing#{} DeletePartitionColumnStatMessage message : {}", fromEventId(), eventMessageAsJSON); + DumpMetaData dmd = withinContext.createDmd(this); + dmd.setPayload(eventMessageAsJSON); + dmd.write(); + } + + @Override + public DumpType dumpType() { + return DumpType.EVENT_DELETE_PART_COL_STAT; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/DeleteTableColStatHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/DeleteTableColStatHandler.java new file mode 100644 index 0000000000..8e94cea594 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/DeleteTableColStatHandler.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.dump.events; + +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.metastore.messaging.DeleteTableColumnStatMessage; +import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; + +class DeleteTableColStatHandler extends AbstractEventHandler { + + DeleteTableColStatHandler(NotificationEvent event) { + super(event); + } + + @Override + DeleteTableColumnStatMessage eventMessage(String stringRepresentation) { + return deserializer.getDeleteTableColumnStatMessage(stringRepresentation); + } + + @Override + public void handle(Context withinContext) throws Exception { + LOG.info("Processing#{} DeleteTableColumnStat message : {}", fromEventId(), eventMessageAsJSON); + DumpMetaData dmd = withinContext.createDmd(this); + dmd.setPayload(eventMessageAsJSON); + dmd.write(); + } + + @Override + public DumpType dumpType() { + return DumpType.EVENT_DELETE_TABLE_COL_STAT; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/EventHandlerFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/EventHandlerFactory.java index 2a0379e942..91f5503849 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/EventHandlerFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/EventHandlerFactory.java @@ -54,6 +54,10 @@ private EventHandlerFactory() { register(MessageBuilder.COMMIT_TXN_EVENT, CommitTxnHandler.class); register(MessageBuilder.ABORT_TXN_EVENT, AbortTxnHandler.class); register(MessageBuilder.ALLOC_WRITE_ID_EVENT, AllocWriteIdHandler.class); + register(MessageBuilder.UPDATE_TBL_COL_STAT_EVENT, UpdateTableColStatHandler.class); + register(MessageBuilder.DELETE_TBL_COL_STAT_EVENT, DeleteTableColStatHandler.class); + register(MessageBuilder.UPDATE_PART_COL_STAT_EVENT, UpdatePartColStatHandler.class); + register(MessageBuilder.DELETE_PART_COL_STAT_EVENT, DeletePartColStatHandler.class); } static void register(String event, Class handlerClazz) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/UpdatePartColStatHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/UpdatePartColStatHandler.java new file mode 100644 index 0000000000..332005bd56 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/UpdatePartColStatHandler.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.dump.events; + +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.metastore.messaging.UpdatePartitionColumnStatMessage; +import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; + +class UpdatePartColStatHandler extends AbstractEventHandler { + + UpdatePartColStatHandler(NotificationEvent event) { + super(event); + } + + @Override + UpdatePartitionColumnStatMessage eventMessage(String stringRepresentation) { + return deserializer.getUpdatePartitionColumnStatMessage(stringRepresentation); + } + + @Override + public void handle(Context withinContext) throws Exception { + LOG.info("Processing#{} UpdateTableColumnStat message : {}", fromEventId(), eventMessageAsJSON); + DumpMetaData dmd = withinContext.createDmd(this); + dmd.setPayload(eventMessageAsJSON); + dmd.write(); + } + + @Override + public DumpType dumpType() { + return DumpType.EVENT_UPDATE_PART_COL_STAT; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/UpdateTableColStatHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/UpdateTableColStatHandler.java new file mode 100644 index 0000000000..a3aecde766 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/UpdateTableColStatHandler.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.dump.events; + +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.metastore.messaging.UpdateTableColumnStatMessage; +import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; + +class UpdateTableColStatHandler extends AbstractEventHandler { + + UpdateTableColStatHandler(NotificationEvent event) { + super(event); + } + + @Override + UpdateTableColumnStatMessage eventMessage(String stringRepresentation) { + return deserializer.getUpdateTableColumnStatMessage(stringRepresentation); + } + + @Override + public void handle(Context withinContext) throws Exception { + LOG.info("Processing#{} UpdateTableColumnStat message : {}", fromEventId(), eventMessageAsJSON); + DumpMetaData dmd = withinContext.createDmd(this); + dmd.setPayload(eventMessageAsJSON); + dmd.write(); + } + + @Override + public DumpType dumpType() { + return DumpType.EVENT_UPDATE_TABLE_COL_STAT; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DeletePartColStatHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DeletePartColStatHandler.java new file mode 100644 index 0000000000..095d377ac6 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DeletePartColStatHandler.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.load.message; + +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; + +/** + * DeletePartColStatHandler + * Target(Load) side handler for partition stat delete event + */ +public class DeletePartColStatHandler extends AbstractMessageHandler { + @Override + public List> handle(Context context) + throws SemanticException { + context.log.info("Replication of partition stat delete event is not supported yet"); + if (!context.isDbNameEmpty()) { + updatedMetadata.set(context.dmd.getEventTo().toString(), context.dbName, context.tableName, null); + } + return Collections.singletonList(TaskFactory.get(new DependencyCollectionWork(), context.hiveConf)); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DeleteTableColStatHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DeleteTableColStatHandler.java new file mode 100644 index 0000000000..488f89dc49 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DeleteTableColStatHandler.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.load.message; + +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; + +/** + * DeleteTableColStatHandler + * Target(Load) side handler for table stat delete event + */ +public class DeleteTableColStatHandler extends AbstractMessageHandler { + @Override + public List> handle(Context context) + throws SemanticException { + context.log.info("Replication of table stat delete event is not supported yet"); + if (!context.isDbNameEmpty()) { + updatedMetadata.set(context.dmd.getEventTo().toString(), context.dbName, context.tableName, null); + } + return Collections.singletonList(TaskFactory.get(new DependencyCollectionWork(), context.hiveConf)); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/UpdatePartColStatHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/UpdatePartColStatHandler.java new file mode 100644 index 0000000000..4ba2ac49fb --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/UpdatePartColStatHandler.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.load.message; + +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; + +/** + * UpdatePartColStatHandler + * Target(Load) side handler for partition stat update event + */ +public class UpdatePartColStatHandler extends AbstractMessageHandler { + @Override + public List> handle(Context context) + throws SemanticException { + context.log.info("Replication of partition stat update event is not supported yet"); + if (!context.isDbNameEmpty()) { + updatedMetadata.set(context.dmd.getEventTo().toString(), context.dbName, context.tableName, null); + } + return Collections.singletonList(TaskFactory.get(new DependencyCollectionWork(), context.hiveConf)); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/UpdateTableColStatHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/UpdateTableColStatHandler.java new file mode 100644 index 0000000000..eb3d18a30e --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/UpdateTableColStatHandler.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.load.message; + +import org.apache.hadoop.hive.ql.exec.Task; +import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; + +/** + * UpdateTableColStatHandler + * Target(Load) side handler for table stat update event + */ +public class UpdateTableColStatHandler extends AbstractMessageHandler { + @Override + public List> handle(Context context) + throws SemanticException { + context.log.info("Replication of table stat update event is not supported yet"); + if (!context.isDbNameEmpty()) { + updatedMetadata.set(context.dmd.getEventTo().toString(), context.dbName, context.tableName, null); + } + return Collections.singletonList(TaskFactory.get(new DependencyCollectionWork(), context.hiveConf)); + } +} diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventRequest.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventRequest.java index f01620443b..77e57718ef 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventRequest.java +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEventRequest.java @@ -40,6 +40,7 @@ private static final org.apache.thrift.protocol.TField LAST_EVENT_FIELD_DESC = new org.apache.thrift.protocol.TField("lastEvent", org.apache.thrift.protocol.TType.I64, (short)1); private static final org.apache.thrift.protocol.TField MAX_EVENTS_FIELD_DESC = new org.apache.thrift.protocol.TField("maxEvents", org.apache.thrift.protocol.TType.I32, (short)2); + private static final org.apache.thrift.protocol.TField EVENT_TYPE_SKIP_LIST_FIELD_DESC = new org.apache.thrift.protocol.TField("eventTypeSkipList", org.apache.thrift.protocol.TType.LIST, (short)3); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { @@ -49,11 +50,13 @@ private long lastEvent; // required private int maxEvents; // optional + private List eventTypeSkipList; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { LAST_EVENT((short)1, "lastEvent"), - MAX_EVENTS((short)2, "maxEvents"); + MAX_EVENTS((short)2, "maxEvents"), + EVENT_TYPE_SKIP_LIST((short)3, "eventTypeSkipList"); private static final Map byName = new HashMap(); @@ -72,6 +75,8 @@ public static _Fields findByThriftId(int fieldId) { return LAST_EVENT; case 2: // MAX_EVENTS return MAX_EVENTS; + case 3: // EVENT_TYPE_SKIP_LIST + return EVENT_TYPE_SKIP_LIST; default: return null; } @@ -115,7 +120,7 @@ public String getFieldName() { private static final int __LASTEVENT_ISSET_ID = 0; private static final int __MAXEVENTS_ISSET_ID = 1; private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.MAX_EVENTS}; + private static final _Fields optionals[] = {_Fields.MAX_EVENTS,_Fields.EVENT_TYPE_SKIP_LIST}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -123,6 +128,9 @@ public String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); tmpMap.put(_Fields.MAX_EVENTS, new org.apache.thrift.meta_data.FieldMetaData("maxEvents", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.EVENT_TYPE_SKIP_LIST, new org.apache.thrift.meta_data.FieldMetaData("eventTypeSkipList", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(NotificationEventRequest.class, metaDataMap); } @@ -145,6 +153,10 @@ public NotificationEventRequest(NotificationEventRequest other) { __isset_bitfield = other.__isset_bitfield; this.lastEvent = other.lastEvent; this.maxEvents = other.maxEvents; + if (other.isSetEventTypeSkipList()) { + List __this__eventTypeSkipList = new ArrayList(other.eventTypeSkipList); + this.eventTypeSkipList = __this__eventTypeSkipList; + } } public NotificationEventRequest deepCopy() { @@ -157,6 +169,7 @@ public void clear() { this.lastEvent = 0; setMaxEventsIsSet(false); this.maxEvents = 0; + this.eventTypeSkipList = null; } public long getLastEvent() { @@ -203,6 +216,44 @@ public void setMaxEventsIsSet(boolean value) { __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MAXEVENTS_ISSET_ID, value); } + public int getEventTypeSkipListSize() { + return (this.eventTypeSkipList == null) ? 0 : this.eventTypeSkipList.size(); + } + + public java.util.Iterator getEventTypeSkipListIterator() { + return (this.eventTypeSkipList == null) ? null : this.eventTypeSkipList.iterator(); + } + + public void addToEventTypeSkipList(String elem) { + if (this.eventTypeSkipList == null) { + this.eventTypeSkipList = new ArrayList(); + } + this.eventTypeSkipList.add(elem); + } + + public List getEventTypeSkipList() { + return this.eventTypeSkipList; + } + + public void setEventTypeSkipList(List eventTypeSkipList) { + this.eventTypeSkipList = eventTypeSkipList; + } + + public void unsetEventTypeSkipList() { + this.eventTypeSkipList = null; + } + + /** Returns true if field eventTypeSkipList is set (has been assigned a value) and false otherwise */ + public boolean isSetEventTypeSkipList() { + return this.eventTypeSkipList != null; + } + + public void setEventTypeSkipListIsSet(boolean value) { + if (!value) { + this.eventTypeSkipList = null; + } + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case LAST_EVENT: @@ -221,6 +272,14 @@ public void setFieldValue(_Fields field, Object value) { } break; + case EVENT_TYPE_SKIP_LIST: + if (value == null) { + unsetEventTypeSkipList(); + } else { + setEventTypeSkipList((List)value); + } + break; + } } @@ -232,6 +291,9 @@ public Object getFieldValue(_Fields field) { case MAX_EVENTS: return getMaxEvents(); + case EVENT_TYPE_SKIP_LIST: + return getEventTypeSkipList(); + } throw new IllegalStateException(); } @@ -247,6 +309,8 @@ public boolean isSet(_Fields field) { return isSetLastEvent(); case MAX_EVENTS: return isSetMaxEvents(); + case EVENT_TYPE_SKIP_LIST: + return isSetEventTypeSkipList(); } throw new IllegalStateException(); } @@ -282,6 +346,15 @@ public boolean equals(NotificationEventRequest that) { return false; } + boolean this_present_eventTypeSkipList = true && this.isSetEventTypeSkipList(); + boolean that_present_eventTypeSkipList = true && that.isSetEventTypeSkipList(); + if (this_present_eventTypeSkipList || that_present_eventTypeSkipList) { + if (!(this_present_eventTypeSkipList && that_present_eventTypeSkipList)) + return false; + if (!this.eventTypeSkipList.equals(that.eventTypeSkipList)) + return false; + } + return true; } @@ -299,6 +372,11 @@ public int hashCode() { if (present_maxEvents) list.add(maxEvents); + boolean present_eventTypeSkipList = true && (isSetEventTypeSkipList()); + list.add(present_eventTypeSkipList); + if (present_eventTypeSkipList) + list.add(eventTypeSkipList); + return list.hashCode(); } @@ -330,6 +408,16 @@ public int compareTo(NotificationEventRequest other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetEventTypeSkipList()).compareTo(other.isSetEventTypeSkipList()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetEventTypeSkipList()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.eventTypeSkipList, other.eventTypeSkipList); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -359,6 +447,16 @@ public String toString() { sb.append(this.maxEvents); first = false; } + if (isSetEventTypeSkipList()) { + if (!first) sb.append(", "); + sb.append("eventTypeSkipList:"); + if (this.eventTypeSkipList == null) { + sb.append("null"); + } else { + sb.append(this.eventTypeSkipList); + } + first = false; + } sb.append(")"); return sb.toString(); } @@ -424,6 +522,24 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, NotificationEventRe org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 3: // EVENT_TYPE_SKIP_LIST + if (schemeField.type == org.apache.thrift.protocol.TType.LIST) { + { + org.apache.thrift.protocol.TList _list732 = iprot.readListBegin(); + struct.eventTypeSkipList = new ArrayList(_list732.size); + String _elem733; + for (int _i734 = 0; _i734 < _list732.size; ++_i734) + { + _elem733 = iprot.readString(); + struct.eventTypeSkipList.add(_elem733); + } + iprot.readListEnd(); + } + struct.setEventTypeSkipListIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -445,6 +561,20 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, NotificationEventR oprot.writeI32(struct.maxEvents); oprot.writeFieldEnd(); } + if (struct.eventTypeSkipList != null) { + if (struct.isSetEventTypeSkipList()) { + oprot.writeFieldBegin(EVENT_TYPE_SKIP_LIST_FIELD_DESC); + { + oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.eventTypeSkipList.size())); + for (String _iter735 : struct.eventTypeSkipList) + { + oprot.writeString(_iter735); + } + oprot.writeListEnd(); + } + oprot.writeFieldEnd(); + } + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -467,10 +597,22 @@ public void write(org.apache.thrift.protocol.TProtocol prot, NotificationEventRe if (struct.isSetMaxEvents()) { optionals.set(0); } - oprot.writeBitSet(optionals, 1); + if (struct.isSetEventTypeSkipList()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); if (struct.isSetMaxEvents()) { oprot.writeI32(struct.maxEvents); } + if (struct.isSetEventTypeSkipList()) { + { + oprot.writeI32(struct.eventTypeSkipList.size()); + for (String _iter736 : struct.eventTypeSkipList) + { + oprot.writeString(_iter736); + } + } + } } @Override @@ -478,11 +620,24 @@ public void read(org.apache.thrift.protocol.TProtocol prot, NotificationEventReq TTupleProtocol iprot = (TTupleProtocol) prot; struct.lastEvent = iprot.readI64(); struct.setLastEventIsSet(true); - BitSet incoming = iprot.readBitSet(1); + BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { struct.maxEvents = iprot.readI32(); struct.setMaxEventsIsSet(true); } + if (incoming.get(1)) { + { + org.apache.thrift.protocol.TList _list737 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32()); + struct.eventTypeSkipList = new ArrayList(_list737.size); + String _elem738; + for (int _i739 = 0; _i739 < _list737.size; ++_i739) + { + _elem738 = iprot.readString(); + struct.eventTypeSkipList.add(_elem738); + } + } + struct.setEventTypeSkipListIsSet(true); + } } } diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php index 39f8b1f05a..48396eaca8 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php @@ -22347,6 +22347,10 @@ class NotificationEventRequest { * @var int */ public $maxEvents = null; + /** + * @var string[] + */ + public $eventTypeSkipList = null; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -22359,6 +22363,14 @@ class NotificationEventRequest { 'var' => 'maxEvents', 'type' => TType::I32, ), + 3 => array( + 'var' => 'eventTypeSkipList', + 'type' => TType::LST, + 'etype' => TType::STRING, + 'elem' => array( + 'type' => TType::STRING, + ), + ), ); } if (is_array($vals)) { @@ -22368,6 +22380,9 @@ class NotificationEventRequest { if (isset($vals['maxEvents'])) { $this->maxEvents = $vals['maxEvents']; } + if (isset($vals['eventTypeSkipList'])) { + $this->eventTypeSkipList = $vals['eventTypeSkipList']; + } } } @@ -22404,6 +22419,23 @@ class NotificationEventRequest { $xfer += $input->skip($ftype); } break; + case 3: + if ($ftype == TType::LST) { + $this->eventTypeSkipList = array(); + $_size647 = 0; + $_etype650 = 0; + $xfer += $input->readListBegin($_etype650, $_size647); + for ($_i651 = 0; $_i651 < $_size647; ++$_i651) + { + $elem652 = null; + $xfer += $input->readString($elem652); + $this->eventTypeSkipList []= $elem652; + } + $xfer += $input->readListEnd(); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -22427,6 +22459,23 @@ class NotificationEventRequest { $xfer += $output->writeI32($this->maxEvents); $xfer += $output->writeFieldEnd(); } + if ($this->eventTypeSkipList !== null) { + if (!is_array($this->eventTypeSkipList)) { + throw new TProtocolException('Bad type in structure.', TProtocolException::INVALID_DATA); + } + $xfer += $output->writeFieldBegin('eventTypeSkipList', TType::LST, 3); + { + $output->writeListBegin(TType::STRING, count($this->eventTypeSkipList)); + { + foreach ($this->eventTypeSkipList as $iter653) + { + $xfer += $output->writeString($iter653); + } + } + $output->writeListEnd(); + } + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py index 4ef4aadfee..b85731e56b 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -15582,17 +15582,20 @@ class NotificationEventRequest: Attributes: - lastEvent - maxEvents + - eventTypeSkipList """ thrift_spec = ( None, # 0 (1, TType.I64, 'lastEvent', None, None, ), # 1 (2, TType.I32, 'maxEvents', None, None, ), # 2 + (3, TType.LIST, 'eventTypeSkipList', (TType.STRING,None), None, ), # 3 ) - def __init__(self, lastEvent=None, maxEvents=None,): + def __init__(self, lastEvent=None, maxEvents=None, eventTypeSkipList=None,): self.lastEvent = lastEvent self.maxEvents = maxEvents + self.eventTypeSkipList = eventTypeSkipList def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -15613,6 +15616,16 @@ def read(self, iprot): self.maxEvents = iprot.readI32() else: iprot.skip(ftype) + elif fid == 3: + if ftype == TType.LIST: + self.eventTypeSkipList = [] + (_etype647, _size644) = iprot.readListBegin() + for _i648 in xrange(_size644): + _elem649 = iprot.readString() + self.eventTypeSkipList.append(_elem649) + iprot.readListEnd() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -15631,6 +15644,13 @@ def write(self, oprot): oprot.writeFieldBegin('maxEvents', TType.I32, 2) oprot.writeI32(self.maxEvents) oprot.writeFieldEnd() + if self.eventTypeSkipList is not None: + oprot.writeFieldBegin('eventTypeSkipList', TType.LIST, 3) + oprot.writeListBegin(TType.STRING, len(self.eventTypeSkipList)) + for iter650 in self.eventTypeSkipList: + oprot.writeString(iter650) + oprot.writeListEnd() + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -15644,6 +15664,7 @@ def __hash__(self): value = 17 value = (value * 31) ^ hash(self.lastEvent) value = (value * 31) ^ hash(self.maxEvents) + value = (value * 31) ^ hash(self.eventTypeSkipList) return value def __repr__(self): diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb index 97dc0696b7..62154796d2 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -3460,10 +3460,12 @@ class NotificationEventRequest include ::Thrift::Struct, ::Thrift::Struct_Union LASTEVENT = 1 MAXEVENTS = 2 + EVENTTYPESKIPLIST = 3 FIELDS = { LASTEVENT => {:type => ::Thrift::Types::I64, :name => 'lastEvent'}, - MAXEVENTS => {:type => ::Thrift::Types::I32, :name => 'maxEvents', :optional => true} + MAXEVENTS => {:type => ::Thrift::Types::I32, :name => 'maxEvents', :optional => true}, + EVENTTYPESKIPLIST => {:type => ::Thrift::Types::LIST, :name => 'eventTypeSkipList', :element => {:type => ::Thrift::Types::STRING}, :optional => true} } def struct_fields; FIELDS; end diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index e25a8cf9a1..6e2a48b5aa 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -934,6 +934,8 @@ public static ConfVars getMetaConf(String name) { "Time interval describing how often the reaper runs"), TOKEN_SIGNATURE("metastore.token.signature", "hive.metastore.token.signature", "", "The delegation token service name to match when selecting a token from the current user's tokens."), + METASTORE_CACHE_CAN_USE_EVENT("metastore.cache.can.use.event", "hive.metastore.cache.can.use.event", false, + "Can notification events from notification log table be used for updating the metastore cache."), TRANSACTIONAL_EVENT_LISTENERS("metastore.transactional.event.listeners", "hive.metastore.transactional.event.listeners", "", "A comma separated list of Java classes that implement the org.apache.riven.MetaStoreEventListener" + diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index cb899d791f..9e4f3c244a 100644 --- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -1135,6 +1135,7 @@ struct CreationMetadata { struct NotificationEventRequest { 1: required i64 lastEvent, 2: optional i32 maxEvents, + 3: optional list eventTypeSkipList, } struct NotificationEvent { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 0ea46f8408..617c7bc012 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -274,7 +274,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam part.setDbName(newDbName); part.setTableName(newTblName); ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, catName, dbname, name, - part.getValues(), part.getSd().getCols(), oldt, part, null); + part.getValues(), part.getSd().getCols(), oldt, part, null, null); if (colStats != null) { columnStatsNeedUpdated.put(part, colStats); } @@ -312,7 +312,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam } } else { alterTableUpdateTableColumnStats( - msdb, oldt, newt, environmentContext, writeIdList); + msdb, oldt, newt, environmentContext, writeIdList, conf, null); } } else { // operations other than table rename @@ -332,7 +332,7 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam List oldCols = part.getSd().getCols(); part.getSd().setCols(newt.getSd().getCols()); ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, catName, dbname, name, - part.getValues(), oldCols, oldt, part, null); + part.getValues(), oldCols, oldt, part, null, null); assert(colStats == null); if (cascade) { msdb.alterPartition( @@ -349,11 +349,11 @@ public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbnam } else { LOG.warn("Alter table not cascaded to partitions."); alterTableUpdateTableColumnStats( - msdb, oldt, newt, environmentContext, writeIdList); + msdb, oldt, newt, environmentContext, writeIdList, conf, null); } } else { alterTableUpdateTableColumnStats( - msdb, oldt, newt, environmentContext, writeIdList); + msdb, oldt, newt, environmentContext, writeIdList, conf, null); } } @@ -481,7 +481,7 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str // PartitionView does not have SD. We do not need update its column stats if (oldPart.getSd() != null) { updateOrGetPartitionColumnStats(msdb, catName, dbname, name, new_part.getValues(), - oldPart.getSd().getCols(), tbl, new_part, null); + oldPart.getSd().getCols(), tbl, new_part, null, null); } msdb.alterPartition( catName, dbname, name, new_part.getValues(), new_part, validWriteIds); @@ -620,7 +620,7 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str String newPartName = Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues()); ColumnStatistics cs = updateOrGetPartitionColumnStats(msdb, catName, dbname, name, oldPart.getValues(), - oldPart.getSd().getCols(), tbl, new_part, null); + oldPart.getSd().getCols(), tbl, new_part, null, null); msdb.alterPartition(catName, dbname, name, part_vals, new_part, validWriteIds); if (cs != null) { cs.getStatsDesc().setPartName(newPartName); @@ -727,7 +727,7 @@ public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, Str // PartitionView does not have SD and we do not need to update its column stats if (oldTmpPart.getSd() != null) { updateOrGetPartitionColumnStats(msdb, catName, dbname, name, oldTmpPart.getValues(), - oldTmpPart.getSd().getCols(), tbl, tmpPart, null); + oldTmpPart.getSd().getCols(), tbl, tmpPart, null, null); } } @@ -799,8 +799,8 @@ private Path constructRenamedPath(Path defaultNewPath, Path currentPath) { } @VisibleForTesting - void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTable, - EnvironmentContext ec, String validWriteIds) + public static List alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTable, + EnvironmentContext ec, String validWriteIds, Configuration conf, List deletedCols) throws MetaException, InvalidObjectException { String catName = normalizeIdentifier(oldTable.isSetCatName() ? oldTable.getCatName() : getDefaultCatalog(conf)); @@ -808,11 +808,13 @@ void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTa String tableName = normalizeIdentifier(oldTable.getTableName()); String newDbName = newTable.getDbName().toLowerCase(); String newTableName = normalizeIdentifier(newTable.getTableName()); + List newStatsObjs = new ArrayList<>(); + //if its not called from cahced store then update the table + boolean doAlterTable = deletedCols == null; try { List oldCols = oldTable.getSd().getCols(); List newCols = newTable.getSd().getCols(); - List newStatsObjs = new ArrayList<>(); ColumnStatistics colStats = null; boolean updateColumnStats = !newDbName.equals(dbName) || !newTableName.equals(tableName) || !MetaStoreServerUtils.columnsIncludedByNameType(oldCols, newCols); @@ -834,7 +836,10 @@ void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTa } else { List statsObjs = colStats.getStatsObj(); if (statsObjs != null) { - List deletedCols = new ArrayList<>(); + // for out para, this value is initialized by caller. + if (deletedCols == null) { + deletedCols = new ArrayList<>(); + } for (ColumnStatisticsObj statsObj : statsObjs) { boolean found = false; for (FieldSchema newCol : newCols) { @@ -847,28 +852,36 @@ void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTa if (found) { if (!newDbName.equals(dbName) || !newTableName.equals(tableName)) { - msdb.deleteTableColumnStatistics(catName, dbName, tableName, statsObj.getColName()); + if (doAlterTable) { + msdb.deleteTableColumnStatistics(catName, dbName, tableName, statsObj.getColName()); + } newStatsObjs.add(statsObj); deletedCols.add(statsObj.getColName()); } } else { - msdb.deleteTableColumnStatistics(catName, dbName, tableName, statsObj.getColName()); + if (doAlterTable) { + msdb.deleteTableColumnStatistics(catName, dbName, tableName, statsObj.getColName()); + } deletedCols.add(statsObj.getColName()); } } - StatsSetupConst.removeColumnStatsState(newTable.getParameters(), deletedCols); + if (doAlterTable) { + StatsSetupConst.removeColumnStatsState(newTable.getParameters(), deletedCols); + } } } } - // Change to new table and append stats for the new table - msdb.alterTable(catName, dbName, tableName, newTable, validWriteIds); - if (updateColumnStats && !newStatsObjs.isEmpty()) { - ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); - statsDesc.setDbName(newDbName); - statsDesc.setTableName(newTableName); - colStats.setStatsObj(newStatsObjs); - msdb.updateTableColumnStatistics(colStats, validWriteIds, newTable.getWriteId()); + if (doAlterTable) { + // Change to new table and append stats for the new table + msdb.alterTable(catName, dbName, tableName, newTable, validWriteIds); + if (updateColumnStats && !newStatsObjs.isEmpty()) { + ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); + statsDesc.setDbName(newDbName); + statsDesc.setTableName(newTableName); + colStats.setStatsObj(newStatsObjs); + msdb.updateTableColumnStatistics(colStats, validWriteIds, newTable.getWriteId()); + } } } catch (NoSuchObjectException nsoe) { LOG.debug("Could not find db entry." + nsoe); @@ -876,13 +889,15 @@ void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTa //should not happen since the input were verified before passed in throw new InvalidObjectException("Invalid inputs to update table column stats: " + e); } + return newStatsObjs; } - private ColumnStatistics updateOrGetPartitionColumnStats( + public static ColumnStatistics updateOrGetPartitionColumnStats( RawStore msdb, String catName, String dbname, String tblname, List partVals, - List oldCols, Table table, Partition part, List newCols) + List oldCols, Table table, Partition part, List newCols, List deletedCols) throws MetaException, InvalidObjectException { ColumnStatistics newPartsColStats = null; + boolean updateColumnStats = true; try { // if newCols are not specified, use default ones. if (newCols == null) { @@ -906,10 +921,17 @@ private ColumnStatistics updateOrGetPartitionColumnStats( List partsColStats = msdb.getPartitionColumnStatistics(catName, dbname, tblname, oldPartNames, oldColNames); assert (partsColStats.size() <= 1); + + // for out para, this value is initialized by caller. + if (deletedCols == null) { + deletedCols = new ArrayList<>(); + } else { + // in case deletedCols is provided by caller, stats will be updated by caller. + updateColumnStats = false; + } for (ColumnStatistics partColStats : partsColStats) { //actually only at most one loop List newStatsObjs = new ArrayList<>(); List statsObjs = partColStats.getStatsObj(); - List deletedCols = new ArrayList<>(); for (ColumnStatisticsObj statsObj : statsObjs) { boolean found =false; for (FieldSchema newCol : newCols) { @@ -921,17 +943,25 @@ private ColumnStatistics updateOrGetPartitionColumnStats( } if (found) { if (rename) { - msdb.deletePartitionColumnStatistics(catName, dbname, tblname, partColStats.getStatsDesc().getPartName(), - partVals, statsObj.getColName()); + if (updateColumnStats) { + msdb.deletePartitionColumnStatistics(catName, dbname, tblname, + partColStats.getStatsDesc().getPartName(), partVals, statsObj.getColName()); + } else { + deletedCols.add(statsObj.getColName()); + } newStatsObjs.add(statsObj); } } else { - msdb.deletePartitionColumnStatistics(catName, dbname, tblname, partColStats.getStatsDesc().getPartName(), - partVals, statsObj.getColName()); + if (updateColumnStats) { + msdb.deletePartitionColumnStatistics(catName, dbname, tblname, partColStats.getStatsDesc().getPartName(), + partVals, statsObj.getColName()); + } deletedCols.add(statsObj.getColName()); } } - StatsSetupConst.removeColumnStatsState(part.getParameters(), deletedCols); + if (updateColumnStats) { + StatsSetupConst.removeColumnStatsState(part.getParameters(), deletedCols); + } if (!newStatsObjs.isEmpty()) { partColStats.setStatsObj(newStatsObjs); newPartsColStats = partColStats; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 598847df03..0a1b96dcf6 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -114,6 +114,7 @@ import org.apache.hadoop.hive.metastore.events.InsertEvent; import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; import org.apache.hadoop.hive.metastore.events.OpenTxnEvent; +import org.apache.hadoop.hive.metastore.events.UpdateTableColumnStatEvent; import org.apache.hadoop.hive.metastore.events.PreAddPartitionEvent; import org.apache.hadoop.hive.metastore.events.PreAlterCatalogEvent; import org.apache.hadoop.hive.metastore.events.PreAlterDatabaseEvent; @@ -140,6 +141,9 @@ import org.apache.hadoop.hive.metastore.events.PreReadISchemaEvent; import org.apache.hadoop.hive.metastore.events.PreReadTableEvent; import org.apache.hadoop.hive.metastore.events.PreReadhSchemaVersionEvent; +import org.apache.hadoop.hive.metastore.events.DeletePartitionColumnStatEvent; +import org.apache.hadoop.hive.metastore.events.DeleteTableColumnStatEvent; +import org.apache.hadoop.hive.metastore.events.UpdatePartitionColumnStatEvent; import org.apache.hadoop.hive.metastore.messaging.EventMessage.EventType; import org.apache.hadoop.hive.metastore.metrics.JvmPauseMonitor; import org.apache.hadoop.hive.metastore.metrics.Metrics; @@ -568,6 +572,19 @@ public void init() throws MetaException { listeners.add(new HMSMetricsListener(conf)); } + boolean canCachedStoreCanUseEvent = false; + for (MetaStoreEventListener listener : transactionalListeners) { + if (listener.doesAddEventsToNotificationLogTable()) { + canCachedStoreCanUseEvent = true; + break; + } + } + if (conf.getBoolean(ConfVars.METASTORE_CACHE_CAN_USE_EVENT.getVarname(), false) && + !canCachedStoreCanUseEvent) { + throw new MetaException("CahcedStore can not use events for invalidation as there is no " + + " TransactionalMetaStoreEventListener to add events to notification table"); + } + endFunctionListeners = MetaStoreServerUtils.getMetaStoreListeners( MetaStoreEndFunctionListener.class, conf, MetastoreConf.getVar(conf, ConfVars.END_FUNCTION_LISTENERS)); @@ -5786,14 +5803,33 @@ private boolean updateTableColumnStatsInternal(ColumnStatistics colStats, colStats.getStatsDesc().getCatName(), colStats.getStatsDesc().getDbName(), colStats.getStatsDesc().getTableName())); - boolean ret = false; + Map parameters = null; + getMS().openTransaction(); + boolean committed = false; try { - ret = getMS().updateTableColumnStatistics(colStats, validWriteIds, writeId) != null; + parameters = getMS().updateTableColumnStatistics(colStats, validWriteIds, writeId); + if (parameters != null) { + if (transactionalListeners != null && !transactionalListeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventType.UPDATE_TABLE_COLUMN_STAT, + new UpdateTableColumnStatEvent(colStats, parameters, validWriteIds, writeId, this)); + } + if (!listeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(listeners, + EventType.UPDATE_TABLE_COLUMN_STAT, + new UpdateTableColumnStatEvent(colStats, parameters, validWriteIds, writeId, this)); + } + } + committed = getMS().commitTransaction(); } finally { - endFunction("write_column_statistics", ret != false, null, + if (!committed) { + getMS().rollbackTransaction(); + } + endFunction("write_column_statistics", parameters != null, null, colStats.getStatsDesc().getTableName()); } - return ret; + + return parameters != null; } private void normalizeColStatsInput(ColumnStatistics colStats) throws MetaException { @@ -5826,16 +5862,37 @@ private boolean updatePartitonColStatsInternal(Table tbl, ColumnStatistics colSt boolean ret = false; + Map parameters; + List partVals; + boolean committed = false; + getMS().openTransaction(); try { if (tbl == null) { tbl = getTable(catName, dbName, tableName); } - List partVals = getPartValsFromName(tbl, csd.getPartName()); - return getMS().updatePartitionColumnStatistics( - colStats, partVals, validWriteIds, writeId) != null; + partVals = getPartValsFromName(tbl, csd.getPartName()); + parameters = getMS().updatePartitionColumnStatistics(colStats, partVals, validWriteIds, writeId); + if (parameters != null) { + if (transactionalListeners != null && !transactionalListeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventType.UPDATE_PARTITION_COLUMN_STAT, + new UpdatePartitionColumnStatEvent(colStats, partVals, parameters, validWriteIds, writeId, this)); + } + if (!listeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(listeners, + EventType.UPDATE_PARTITION_COLUMN_STAT, + new UpdatePartitionColumnStatEvent(colStats, partVals, parameters, validWriteIds, writeId, this)); + } + } + committed = getMS().commitTransaction(); } finally { + if (!committed) { + getMS().rollbackTransaction(); + } endFunction("write_partition_column_statistics", ret != false, null, tableName); } + + return parameters != null; } @Override @@ -5889,6 +5946,20 @@ public boolean delete_partition_column_statistics(String dbName, String tableNam ret = getMS().deletePartitionColumnStatistics(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, convertedPartName, partVals, colName); + if (ret) { + if (transactionalListeners != null && !transactionalListeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventType.DELETE_PARTITION_COLUMN_STAT, + new DeletePartitionColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, + convertedPartName, partVals, colName, this)); + } + if (!listeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(listeners, + EventType.DELETE_PARTITION_COLUMN_STAT, + new DeletePartitionColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, + convertedPartName, partVals, colName, this)); + } + } committed = getMS().commitTransaction(); } finally { if (!committed) { @@ -5926,6 +5997,20 @@ public boolean delete_table_column_statistics(String dbName, String tableName, S } ret = getMS().deleteTableColumnStatistics(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName, colName); + if (ret) { + if (transactionalListeners != null && !transactionalListeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventType.DELETE_TABLE_COLUMN_STAT, + new DeleteTableColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], + tableName, colName, this)); + } + if (!listeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(listeners, + EventType.DELETE_TABLE_COLUMN_STAT, + new DeleteTableColumnStatEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], + tableName, colName, this)); + } + } committed = getMS().commitTransaction(); } finally { if (!committed) { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java index de226bf327..0d6d10f796 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java @@ -56,6 +56,10 @@ import org.apache.hadoop.hive.metastore.events.AbortTxnEvent; import org.apache.hadoop.hive.metastore.events.AllocWriteIdEvent; import org.apache.hadoop.hive.metastore.events.AcidWriteEvent; +import org.apache.hadoop.hive.metastore.events.UpdateTableColumnStatEvent; +import org.apache.hadoop.hive.metastore.events.DeleteTableColumnStatEvent; +import org.apache.hadoop.hive.metastore.events.UpdatePartitionColumnStatEvent; +import org.apache.hadoop.hive.metastore.events.DeletePartitionColumnStatEvent; import org.apache.hadoop.hive.metastore.tools.SQLGenerator; import java.sql.Connection; @@ -294,6 +298,49 @@ public void onAcidWrite(AcidWriteEvent acidWriteEvent, Connection dbConn, SQLGen throws MetaException { } + /** + * This will be called to update table column stats + * @param updateTableColumnStatEvent event to be processed + * @throws MetaException + */ + public void onUpdateTableColumnStat(UpdateTableColumnStatEvent updateTableColumnStatEvent) + throws MetaException { + } + + /** + * This will be called to delete table column stats + * @param deleteTableColumnStatEvent event to be processed + * @throws MetaException + */ + public void onDeleteTableColumnStat(DeleteTableColumnStatEvent deleteTableColumnStatEvent) + throws MetaException { + } + + /** + * This will be called to update partition column stats + * @param updatePartColStatEvent event to be processed + * @throws MetaException + */ + public void onUpdatePartitionColumnStat(UpdatePartitionColumnStatEvent updatePartColStatEvent) + throws MetaException { + } + + /** + * This will be called to delete partition column stats + * @param deletePartColStatEvent event to be processed + * @throws MetaException + */ + public void onDeletePartitionColumnStat(DeletePartitionColumnStatEvent deletePartColStatEvent) + throws MetaException { + } + + /** + * This is to check if the listener adds the event info to notification log table. + */ + public boolean doesAddEventsToNotificationLogTable() { + return false; + } + @Override public Configuration getConf() { return this.conf; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java index c296f57559..dd82c4b03a 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java @@ -55,6 +55,10 @@ import org.apache.hadoop.hive.metastore.events.AbortTxnEvent; import org.apache.hadoop.hive.metastore.events.AllocWriteIdEvent; import org.apache.hadoop.hive.metastore.events.AcidWriteEvent; +import org.apache.hadoop.hive.metastore.events.UpdateTableColumnStatEvent; +import org.apache.hadoop.hive.metastore.events.DeleteTableColumnStatEvent; +import org.apache.hadoop.hive.metastore.events.UpdatePartitionColumnStatEvent; +import org.apache.hadoop.hive.metastore.events.DeletePartitionColumnStatEvent; import org.apache.hadoop.hive.metastore.tools.SQLGenerator; import java.sql.Connection; import java.util.List; @@ -224,6 +228,14 @@ public void notify(MetaStoreEventListener listener, ListenerEvent event) throws (listener, event) -> listener.onAllocWriteId((AllocWriteIdEvent) event, null, null)) .put(EventType.ACID_WRITE, (listener, event) -> listener.onAcidWrite((AcidWriteEvent) event, null, null)) + .put(EventType.UPDATE_TABLE_COLUMN_STAT, + (listener, event) -> listener.onUpdateTableColumnStat((UpdateTableColumnStatEvent) event)) + .put(EventType.DELETE_TABLE_COLUMN_STAT, + (listener, event) -> listener.onDeleteTableColumnStat((DeleteTableColumnStatEvent) event)) + .put(EventType.UPDATE_PARTITION_COLUMN_STAT, + (listener, event) -> listener.onUpdatePartitionColumnStat((UpdatePartitionColumnStatEvent) event)) + .put(EventType.DELETE_PARTITION_COLUMN_STAT, + (listener, event) -> listener.onDeletePartitionColumnStat((DeletePartitionColumnStatEvent) event)) .build() ); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index e598a43e4d..3ef7e734ef 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -4154,7 +4154,7 @@ public Table alterTable(String catName, String dbname, String name, Table newTab * Verifies that the stats JSON string is unchanged for alter table (txn stats). * @return Error message with the details of the change, or null if the value has not changed. */ - private static String verifyStatsChangeCtx(Map oldP, Map newP, + public static String verifyStatsChangeCtx(Map oldP, Map newP, long writeId, String validWriteIds, boolean isColStatsChange) { if (validWriteIds != null && writeId > 0) return null; // We have txn context. String oldVal = oldP == null ? null : oldP.get(StatsSetupConst.COLUMN_STATS_ACCURATE); @@ -9777,13 +9777,25 @@ public NotificationEventResponse getNextNotification(NotificationEventRequest rq try { openTransaction(); long lastEvent = rqst.getLastEvent(); - query = pm.newQuery(MNotificationLog.class, "eventId > lastEvent"); - query.declareParameters("java.lang.Long lastEvent"); + List parameterVals = new ArrayList<>(); + parameterVals.add(lastEvent); + StringBuilder filterBuilder = new StringBuilder("eventId > para" + parameterVals.size()); + StringBuilder parameterBuilder = new StringBuilder("java.lang.Long para" + parameterVals.size()); + if (rqst.isSetEventTypeSkipList()) { + for (String eventType : rqst.getEventTypeSkipList()) { + parameterVals.add(eventType); + parameterBuilder.append(", java.lang.String para" + parameterVals.size()); + filterBuilder.append(" && eventType != para" + parameterVals.size()); + } + } + query = pm.newQuery(MNotificationLog.class, filterBuilder.toString()); + query.declareParameters(parameterBuilder.toString()); query.setOrdering("eventId ascending"); int maxEventResponse = MetastoreConf.getIntVar(conf, ConfVars.METASTORE_MAX_EVENT_RESPONSE); int maxEvents = (rqst.getMaxEvents() < maxEventResponse && rqst.getMaxEvents() > 0) ? rqst.getMaxEvents() : maxEventResponse; query.setRange(0, maxEvents); - Collection events = (Collection) query.execute(lastEvent); + Collection events = + (Collection) query.executeWithArray(parameterVals.toArray(new Object[parameterVals.size()])); commited = commitTransaction(); if (events == null) { return result; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index e4ef46fdb4..bb504b0399 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.DatabaseName; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; @@ -49,13 +50,29 @@ import org.apache.hadoop.hive.metastore.RawStore; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.HiveAlterHandler; import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.metastore.cache.SharedCache.StatsType; import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregator; import org.apache.hadoop.hive.metastore.columnstats.aggr.ColumnStatsAggregatorFactory; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.messaging.AlterDatabaseMessage; +import org.apache.hadoop.hive.metastore.messaging.CreateDatabaseMessage; +import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage; +import org.apache.hadoop.hive.metastore.messaging.DropTableMessage; +import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage; +import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage; +import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage; +import org.apache.hadoop.hive.metastore.messaging.DropPartitionMessage; +import org.apache.hadoop.hive.metastore.messaging.UpdateTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.DeleteTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.UpdatePartitionColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.DeletePartitionColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.MessageBuilder; +import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; +import org.apache.hadoop.hive.metastore.messaging.MessageFactory; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.hadoop.hive.metastore.utils.JavaUtils; @@ -94,9 +111,11 @@ private static TablesPendingPrewarm tblsPendingPrewarm = new TablesPendingPrewarm(); private RawStore rawStore = null; private Configuration conf; - private boolean areTxnStatsSupported; + private static boolean areTxnStatsSupported; private PartitionExpressionProxy expressionProxy = null; private static final SharedCache sharedCache = new SharedCache(); + private static boolean canUseEvents = false; + private static long lastEventId; static final private Logger LOG = LoggerFactory.getLogger(CachedStore.class.getName()); @@ -119,7 +138,38 @@ void setConfForTest(Configuration conf) { initSharedCache(conf); } + synchronized private static void triggerUpdateUsingEvent(RawStore rawStore) { + if (!isCachePrewarmed.get()) { + LOG.error("cache update should be done only after prewarm"); + throw new RuntimeException("cache update should be done only after prewarm"); + } + long startTime = System.nanoTime(); + long preEventId = lastEventId; + try { + lastEventId = updateUsingNotificationEvents(rawStore, lastEventId); + } catch (Exception e) { + LOG.error(" cache update failed for start event id " + lastEventId + " with error ", e); + throw new RuntimeException(e.getMessage()); + } finally { + long endTime = System.nanoTime(); + LOG.info("Time taken in updateUsingNotificationEvents for num events : " + (lastEventId - preEventId) + " = " + + (endTime - startTime) / 1000000 + "ms"); + } + } + + synchronized private static void triggerPreWarm(RawStore rawStore) { + lastEventId = rawStore.getCurrentNotificationEventId().getEventId(); + prewarm(rawStore); + } + private void setConfInternal(Configuration conf) { + if (MetastoreConf.getBoolVar(conf, ConfVars.METASTORE_CACHE_CAN_USE_EVENT)) { + canUseEvents = true; + } else { + canUseEvents = false; + } + LOG.info("canUseEvents is set to " + canUseEvents + " in cached Store"); + String rawStoreClassName = MetastoreConf.getVar(conf, ConfVars.CACHED_RAW_STORE_IMPL, ObjectStore.class.getName()); if (rawStore == null) { @@ -150,6 +200,201 @@ private void initSharedCache(Configuration conf) { } } + @VisibleForTesting + public static SharedCache getSharedCache() { + return sharedCache; + } + + static private ColumnStatistics updateStatsForPart(RawStore rawStore, Table before, String catalogName, + String dbName, String tableName, Partition part) throws Exception { + ColumnStatistics colStats; + List deletedCols = new ArrayList<>(); + colStats = HiveAlterHandler.updateOrGetPartitionColumnStats(rawStore, catalogName, dbName, tableName, + part.getValues(), part.getSd().getCols(), before, part, null, deletedCols); + for (String column : deletedCols) { + sharedCache.removePartitionColStatsFromCache(catalogName, dbName, tableName, part.getValues(), column); + } + if (colStats != null) { + sharedCache.updatePartitionColStatsInCache(catalogName, dbName, tableName, part.getValues(), colStats.getStatsObj()); + } + return colStats; + } + + static private void updateStatsForTable(RawStore rawStore, Table before, Table after, String catalogName, + String dbName, String tableName) throws Exception { + ColumnStatistics colStats = null; + List deletedCols = new ArrayList<>(); + if (before.isSetPartitionKeys()) { + List parts = sharedCache.listCachedPartitions(catalogName, dbName, tableName, -1); + for (Partition part : parts) { + colStats = updateStatsForPart(rawStore, before, catalogName, dbName, tableName, part); + } + } + + boolean needUpdateAggrStat = false; + List statisticsObjs = HiveAlterHandler.alterTableUpdateTableColumnStats(rawStore, before, + after,null, null, rawStore.getConf(), deletedCols); + if (colStats != null) { + sharedCache.updateTableColStatsInCache(catalogName, dbName, tableName, statisticsObjs); + needUpdateAggrStat = true; + } + for (String column : deletedCols) { + sharedCache.removeTableColStatsFromCache(catalogName, dbName, tableName, column); + needUpdateAggrStat = true; + } + } + + @VisibleForTesting + public static long updateUsingNotificationEvents(RawStore rawStore, long lastEventId) throws Exception { + LOG.debug("updating cache using notification events starting from event id " + lastEventId); + NotificationEventRequest rqst = new NotificationEventRequest(lastEventId); + + //Add the events which are not related to metadata update + rqst.addToEventTypeSkipList(MessageBuilder.INSERT_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.OPEN_TXN_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.COMMIT_TXN_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.ABORT_TXN_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.ALLOC_WRITE_ID_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.ACID_WRITE_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.CREATE_FUNCTION_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.DROP_FUNCTION_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.ADD_PRIMARYKEY_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.ADD_FOREIGNKEY_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.ADD_UNIQUECONSTRAINT_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.ADD_NOTNULLCONSTRAINT_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.DROP_CONSTRAINT_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.CREATE_ISCHEMA_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.ALTER_ISCHEMA_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.DROP_ISCHEMA_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.ADD_SCHEMA_VERSION_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.ALTER_SCHEMA_VERSION_EVENT); + rqst.addToEventTypeSkipList(MessageBuilder.DROP_SCHEMA_VERSION_EVENT); + + Deadline.startTimer("getNextNotification"); + NotificationEventResponse resp = rawStore.getNextNotification(rqst); + Deadline.stopTimer(); + + if (resp == null || resp.getEvents() == null) { + LOG.debug("no events to process"); + return lastEventId; + } + + List eventList = resp.getEvents(); + LOG.debug("num events to process" + eventList.size()); + + for (NotificationEvent event : eventList) { + long eventId = event.getEventId(); + if (eventId <= lastEventId) { + LOG.error("Event id is not valid " + lastEventId + " : " + eventId); + throw new RuntimeException(" event id is not valid " + lastEventId + " : " + eventId); + } + lastEventId = eventId; + String message = event.getMessage(); + LOG.debug("Event to process " + event); + MessageDeserializer deserializer = MessageFactory.getInstance(event.getMessageFormat()).getDeserializer(); + String catalogName = event.getCatName() == null ? "" : event.getCatName().toLowerCase(); + String dbName = event.getDbName() == null ? "" : event.getDbName().toLowerCase(); + String tableName = event.getTableName() == null ? "" : event.getTableName().toLowerCase(); + if (!shouldCacheTable(catalogName, dbName, tableName)) { + continue; + } + switch (event.getEventType()) { + case MessageBuilder.ADD_PARTITION_EVENT: + AddPartitionMessage addPartMessage = deserializer.getAddPartitionMessage(message); + sharedCache.addPartitionsToCache(catalogName, + dbName, tableName, addPartMessage.getPartitionObjs()); + break; + case MessageBuilder.ALTER_PARTITION_EVENT: + AlterPartitionMessage alterPartitionMessage = deserializer.getAlterPartitionMessage(message); + sharedCache.alterPartitionInCache(catalogName, dbName, tableName, + alterPartitionMessage.getPtnObjBefore().getValues(), alterPartitionMessage.getPtnObjAfter()); + //TODO : Use the stat object stored in the alter table message to update the stats in cache. + if (updateStatsForPart(rawStore, alterPartitionMessage.getTableObj(), + catalogName, dbName, tableName, alterPartitionMessage.getPtnObjAfter()) != null) { + CacheUpdateMasterWork.updateTableAggregatePartitionColStats(rawStore, catalogName, dbName, tableName); + } + break; + case MessageBuilder.DROP_PARTITION_EVENT: + DropPartitionMessage dropPartitionMessage = deserializer.getDropPartitionMessage(message); + for (Map partMap : dropPartitionMessage.getPartitions()) { + sharedCache.removePartitionFromCache(catalogName, dbName, tableName, new ArrayList<>(partMap.values())); + } + break; + case MessageBuilder.CREATE_TABLE_EVENT: + CreateTableMessage createTableMessage = deserializer.getCreateTableMessage(message); + sharedCache.addTableToCache(catalogName, dbName, + tableName, createTableMessage.getTableObj()); + break; + case MessageBuilder.ALTER_TABLE_EVENT: + AlterTableMessage alterTableMessage = deserializer.getAlterTableMessage(message); + sharedCache.alterTableInCache(catalogName, dbName, tableName, alterTableMessage.getTableObjAfter()); + //TODO : Use the stat object stored in the alter table message to update the stats in cache. + updateStatsForTable(rawStore, alterTableMessage.getTableObjBefore(), alterTableMessage.getTableObjAfter(), + catalogName, dbName, tableName); + break; + case MessageBuilder.DROP_TABLE_EVENT: + DropTableMessage dropTableMessage = deserializer.getDropTableMessage(message); + int batchSize = MetastoreConf.getIntVar(rawStore.getConf(), ConfVars.BATCH_RETRIEVE_OBJECTS_MAX); + String tableDnsPath = null; + Path tablePath = new Path(dropTableMessage.getTableObj().getSd().getLocation()); + if (tablePath != null) { + tableDnsPath = new Warehouse(rawStore.getConf()).getDnsPath(tablePath).toString(); + } + + while (true) { + Map partitionLocations = rawStore.getPartitionLocations(catalogName, dbName, tableName, + tableDnsPath, batchSize); + if (partitionLocations == null || partitionLocations.isEmpty()) { + break; + } + sharedCache.removePartitionFromCache(catalogName, dbName, tableName, + new ArrayList<>(partitionLocations.values())); + } + sharedCache.removeTableFromCache(catalogName, dbName, tableName); + break; + case MessageBuilder.CREATE_DATABASE_EVENT: + CreateDatabaseMessage createDatabaseMessage = deserializer.getCreateDatabaseMessage(message); + sharedCache.addDatabaseToCache(createDatabaseMessage.getDatabaseObject()); + break; + case MessageBuilder.ALTER_DATABASE_EVENT: + AlterDatabaseMessage alterDatabaseMessage = deserializer.getAlterDatabaseMessage(message); + sharedCache.alterDatabaseInCache(catalogName, dbName, alterDatabaseMessage.getDbObjAfter()); + break; + case MessageBuilder.DROP_DATABASE_EVENT: + sharedCache.removeDatabaseFromCache(catalogName, dbName); + break; + case MessageBuilder.CREATE_CATALOG_EVENT: + case MessageBuilder.DROP_CATALOG_EVENT: + case MessageBuilder.ALTER_CATALOG_EVENT: + // TODO : Need to add cache invalidation for catalog events + LOG.error("catalog Events are not supported for cache invalidation : " + event.getEventType()); + break; + case MessageBuilder.UPDATE_TBL_COL_STAT_EVENT: + UpdateTableColumnStatMessage msg = deserializer.getUpdateTableColumnStatMessage(message); + updateTableColumnsStatsInternal(rawStore.getConf(), msg.getColumnStatistics(), msg.getParameters(), + msg.getValidWriteIds(), msg.getWriteId()); + break; + case MessageBuilder.DELETE_TBL_COL_STAT_EVENT: + DeleteTableColumnStatMessage msgDel = deserializer.getDeleteTableColumnStatMessage(message); + sharedCache.removeTableColStatsFromCache(catalogName, dbName, tableName, msgDel.getColName()); + break; + case MessageBuilder.UPDATE_PART_COL_STAT_EVENT: + UpdatePartitionColumnStatMessage msgPartUpdate = deserializer.getUpdatePartitionColumnStatMessage(message); + sharedCache.updatePartitionColStatsInCache(catalogName, dbName, tableName, msgPartUpdate.getPartVals(), + msgPartUpdate.getColumnStatistics().getStatsObj()); + break; + case MessageBuilder.DELETE_PART_COL_STAT_EVENT: + DeletePartitionColumnStatMessage msgPart = deserializer.getDeletePartitionColumnStatMessage(message); + sharedCache.removePartitionColStatsFromCache(catalogName, dbName, tableName, + msgPart.getPartValues(), msgPart.getColName()); + break; + default: + LOG.error("Event is not supported for cache invalidation : " + event.getEventType()); + } + } + return lastEventId; + } + @VisibleForTesting /** * This initializes the caches in SharedCache by getting the objects from Metastore DB via @@ -161,6 +406,7 @@ static void prewarm(RawStore rawStore) { } long startTime = System.nanoTime(); LOG.info("Prewarming CachedStore"); + long sleepTime = 100; while (!isCachePrewarmed.get()) { // Prevents throwing exceptions in our raw store calls since we're not using RawStoreProxy Deadline.registerIfNot(1000000); @@ -176,6 +422,12 @@ static void prewarm(RawStore rawStore) { sharedCache.populateCatalogsInCache(catalogs); } catch (MetaException | NoSuchObjectException e) { LOG.warn("Failed to populate catalogs in cache, going to try again", e); + try { + Thread.sleep(sleepTime); + sleepTime = sleepTime * 2; + } catch (InterruptedException timerEx) { + LOG.info("sleep interrupted", timerEx.getMessage()); + } // try again continue; } @@ -407,8 +659,7 @@ public Thread newThread(Runnable r) { } if (runOnlyOnce) { // Some tests control the execution of the background update thread - cacheUpdateMaster.schedule(new CacheUpdateMasterWork(conf, shouldRunPrewarm), 0, - TimeUnit.MILLISECONDS); + cacheUpdateMaster.schedule(new CacheUpdateMasterWork(conf, shouldRunPrewarm), 0, TimeUnit.MILLISECONDS); } } @@ -439,6 +690,7 @@ static void setCacheRefreshPeriod(long time) { private boolean shouldRunPrewarm = true; private final RawStore rawStore; + CacheUpdateMasterWork(Configuration conf, boolean shouldRunPrewarm) { this.shouldRunPrewarm = shouldRunPrewarm; String rawStoreClassName = @@ -456,11 +708,19 @@ static void setCacheRefreshPeriod(long time) { @Override public void run() { if (!shouldRunPrewarm) { - // TODO: prewarm and update can probably be merged. - update(); + if (canUseEvents) { + try { + triggerUpdateUsingEvent(rawStore); + } catch (Exception e) { + LOG.error("failed to update cache using events ", e); + } + } else { + // TODO: prewarm and update can probably be merged. + update(); + } } else { try { - prewarm(rawStore); + triggerPreWarm(rawStore); } catch (Exception e) { LOG.error("Prewarm failure", e); return; @@ -619,7 +879,7 @@ private void updateTablePartitionColStats(RawStore rawStore, String catName, Str // Update cached aggregate stats for all partitions of a table and for all // but default partition - private void updateTableAggregatePartitionColStats(RawStore rawStore, String catName, String dbName, + private static void updateTableAggregatePartitionColStats(RawStore rawStore, String catName, String dbName, String tblName) { try { Table table = rawStore.getTable(catName, dbName, tblName); @@ -675,7 +935,22 @@ public boolean openTransaction() { @Override public boolean commitTransaction() { - return rawStore.commitTransaction(); + if (!rawStore.commitTransaction()) { + return false; + } + + // In case of event based update, shared cache is not updated directly to avoid inconsistency. + // For example, if metastore B add a partition, then metastore A drop a partition later. However, on metastore A, + // it first get drop partition request, then from notification, create the partition. If there's no tombstone + // entry in partition cache to tell drop is after creation, we end up consumes the creation request. Though + // eventually there's drop partition notification, but during the interim, later event takes precedence. + // So we will not update the cache during raw store operation but wait during commit transaction to make sure that + // the event related to the current transactions are updated in the cache and thus we can support strong + // consistency in case there is only one metastore. + if (canUseEvents) { + triggerUpdateUsingEvent(rawStore); + } + return true; } @Override @@ -691,19 +966,26 @@ public void rollbackTransaction() { @Override public void createCatalog(Catalog cat) throws MetaException { rawStore.createCatalog(cat); - sharedCache.addCatalogToCache(cat); + // in case of event based cache update, cache will not be updated for catalog. + if (!canUseEvents) { + sharedCache.addCatalogToCache(cat); + } } @Override public void alterCatalog(String catName, Catalog cat) throws MetaException, InvalidOperationException { rawStore.alterCatalog(catName, cat); - sharedCache.alterCatalogInCache(StringUtils.normalizeIdentifier(catName), cat); + // in case of event based cache update, cache will not be updated for catalog. + if (!canUseEvents) { + sharedCache.alterCatalogInCache(StringUtils.normalizeIdentifier(catName), cat); + } } @Override public Catalog getCatalog(String catalogName) throws NoSuchObjectException, MetaException { - if (!sharedCache.isCatalogCachePrewarmed()) { + // in case of event based cache update, cache will not be updated for catalog. + if (!sharedCache.isCatalogCachePrewarmed() || canUseEvents) { return rawStore.getCatalog(catalogName); } Catalog cat = sharedCache.getCatalogFromCache(normalizeIdentifier(catalogName)); @@ -715,7 +997,8 @@ public Catalog getCatalog(String catalogName) throws NoSuchObjectException, Meta @Override public List getCatalogs() throws MetaException { - if (!sharedCache.isCatalogCachePrewarmed()) { + // in case of event based cache update, cache will not be updated for catalog. + if (!sharedCache.isCatalogCachePrewarmed() || canUseEvents) { return rawStore.getCatalogs(); } return sharedCache.listCachedCatalogs(); @@ -724,19 +1007,29 @@ public Catalog getCatalog(String catalogName) throws NoSuchObjectException, Meta @Override public void dropCatalog(String catalogName) throws NoSuchObjectException, MetaException { rawStore.dropCatalog(catalogName); - catalogName = catalogName.toLowerCase(); - sharedCache.removeCatalogFromCache(catalogName); + + // in case of event based cache update, cache will not be updated for catalog. + if (!canUseEvents) { + catalogName = catalogName.toLowerCase(); + sharedCache.removeCatalogFromCache(catalogName); + } } @Override public void createDatabase(Database db) throws InvalidObjectException, MetaException { rawStore.createDatabase(db); - sharedCache.addDatabaseToCache(db); + // in case of event based cache update, cache will be updated during commit. + if (!canUseEvents) { + sharedCache.addDatabaseToCache(db); + } } @Override public Database getDatabase(String catName, String dbName) throws NoSuchObjectException { - if (!sharedCache.isDatabaseCachePrewarmed()) { + // in case of event based cache update, cache will be updated during commit. So within active transaction, read + // directly from rawStore to avoid reading stale data as the data updated during same transaction will not be + // updated in the cache. + if (!sharedCache.isDatabaseCachePrewarmed() || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getDatabase(catName, dbName); } dbName = dbName.toLowerCase(); @@ -751,7 +1044,8 @@ public Database getDatabase(String catName, String dbName) throws NoSuchObjectEx @Override public boolean dropDatabase(String catName, String dbName) throws NoSuchObjectException, MetaException { boolean succ = rawStore.dropDatabase(catName, dbName); - if (succ) { + if (succ && !canUseEvents) { + // in case of event based cache update, cache will be updated during commit. sharedCache.removeDatabaseFromCache(StringUtils.normalizeIdentifier(catName), StringUtils.normalizeIdentifier(dbName)); } @@ -762,7 +1056,8 @@ public boolean dropDatabase(String catName, String dbName) throws NoSuchObjectEx public boolean alterDatabase(String catName, String dbName, Database db) throws NoSuchObjectException, MetaException { boolean succ = rawStore.alterDatabase(catName, dbName, db); - if (succ) { + if (succ && !canUseEvents) { + // in case of event based cache update, cache will be updated during commit. sharedCache.alterDatabaseInCache(StringUtils.normalizeIdentifier(catName), StringUtils.normalizeIdentifier(dbName), db); } @@ -771,7 +1066,7 @@ public boolean alterDatabase(String catName, String dbName, Database db) @Override public List getDatabases(String catName, String pattern) throws MetaException { - if (!sharedCache.isDatabaseCachePrewarmed()) { + if (!sharedCache.isDatabaseCachePrewarmed() || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getDatabases(catName, pattern); } return sharedCache.listCachedDatabases(catName, pattern); @@ -779,7 +1074,7 @@ public boolean alterDatabase(String catName, String dbName, Database db) @Override public List getAllDatabases(String catName) throws MetaException { - if (!sharedCache.isDatabaseCachePrewarmed()) { + if (!sharedCache.isDatabaseCachePrewarmed() || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getAllDatabases(catName); } return sharedCache.listCachedDatabases(catName); @@ -821,6 +1116,10 @@ private void validateTableType(Table tbl) { @Override public void createTable(Table tbl) throws InvalidObjectException, MetaException { rawStore.createTable(tbl); + // in case of event based cache update, cache will be updated during commit. + if (canUseEvents) { + return; + } String catName = normalizeIdentifier(tbl.getCatName()); String dbName = normalizeIdentifier(tbl.getDbName()); String tblName = normalizeIdentifier(tbl.getTableName()); @@ -835,7 +1134,8 @@ public void createTable(Table tbl) throws InvalidObjectException, MetaException public boolean dropTable(String catName, String dbName, String tblName) throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { boolean succ = rawStore.dropTable(catName, dbName, tblName); - if (succ) { + // in case of event based cache update, cache will be updated during commit. + if (succ && !canUseEvents) { catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); tblName = normalizeIdentifier(tblName); @@ -857,7 +1157,7 @@ public Table getTable(String catName, String dbName, String tblName, String vali catName = normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!shouldCacheTable(catName, dbName, tblName)) { + if (!shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getTable(catName, dbName, tblName, validWriteIds); } Table tbl = sharedCache.getTableFromCache(catName, dbName, tblName); @@ -898,7 +1198,8 @@ public Table getTable(String catName, String dbName, String tblName, String vali @Override public boolean addPartition(Partition part) throws InvalidObjectException, MetaException { boolean succ = rawStore.addPartition(part); - if (succ) { + // in case of event based cache update, cache will be updated during commit. + if (succ && !canUseEvents) { String dbName = normalizeIdentifier(part.getDbName()); String tblName = normalizeIdentifier(part.getTableName()); String catName = part.isSetCatName() ? normalizeIdentifier(part.getCatName()) : DEFAULT_CATALOG_NAME; @@ -914,7 +1215,8 @@ public boolean addPartition(Partition part) throws InvalidObjectException, MetaE public boolean addPartitions(String catName, String dbName, String tblName, List parts) throws InvalidObjectException, MetaException { boolean succ = rawStore.addPartitions(catName, dbName, tblName, parts); - if (succ) { + // in case of event based cache update, cache will be updated during commit. + if (succ && !canUseEvents) { catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); tblName = normalizeIdentifier(tblName); @@ -930,7 +1232,8 @@ public boolean addPartitions(String catName, String dbName, String tblName, List public boolean addPartitions(String catName, String dbName, String tblName, PartitionSpecProxy partitionSpec, boolean ifNotExists) throws InvalidObjectException, MetaException { boolean succ = rawStore.addPartitions(catName, dbName, tblName, partitionSpec, ifNotExists); - if (succ) { + // in case of event based cache update, cache will be updated during commit. + if (succ && !canUseEvents) { catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); tblName = normalizeIdentifier(tblName); @@ -959,7 +1262,7 @@ public Partition getPartition(String catName, String dbName, String tblName, catName = normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!shouldCacheTable(catName, dbName, tblName)) { + if (!shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getPartition( catName, dbName, tblName, part_vals, validWriteIds); } @@ -990,7 +1293,7 @@ public boolean doesPartitionExist(String catName, String dbName, String tblName, catName = normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!shouldCacheTable(catName, dbName, tblName)) { + if (!shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.doesPartitionExist(catName, dbName, tblName, partKeys, part_vals); } Table tbl = sharedCache.getTableFromCache(catName, dbName, tblName); @@ -1005,7 +1308,8 @@ public boolean doesPartitionExist(String catName, String dbName, String tblName, public boolean dropPartition(String catName, String dbName, String tblName, List part_vals) throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { boolean succ = rawStore.dropPartition(catName, dbName, tblName, part_vals); - if (succ) { + // in case of event based cache update, cache will be updated during commit. + if (succ && !canUseEvents) { catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); tblName = normalizeIdentifier(tblName); @@ -1021,6 +1325,10 @@ public boolean dropPartition(String catName, String dbName, String tblName, List public void dropPartitions(String catName, String dbName, String tblName, List partNames) throws MetaException, NoSuchObjectException { rawStore.dropPartitions(catName, dbName, tblName, partNames); + // in case of event based cache update, cache will be updated during commit. + if (canUseEvents) { + return; + } catName = normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); @@ -1040,7 +1348,7 @@ public void dropPartitions(String catName, String dbName, String tblName, List getTables(String catName, String dbName, String pattern) throws MetaException { - if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed.get()) { + if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed.get() || + (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getTables(catName, dbName, pattern); } return sharedCache.listCachedTableNames(StringUtils.normalizeIdentifier(catName), @@ -1106,7 +1419,8 @@ public void updateCreationMetadata(String catName, String dbname, String tablena @Override public List getTables(String catName, String dbName, String pattern, TableType tableType) throws MetaException { - if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed.get()) { + if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed.get() + || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getTables(catName, dbName, pattern, tableType); } return sharedCache.listCachedTableNames(StringUtils.normalizeIdentifier(catName), @@ -1123,7 +1437,8 @@ public void updateCreationMetadata(String catName, String dbname, String tablena public List getTableMeta(String catName, String dbNames, String tableNames, List tableTypes) throws MetaException { // TODO Check if all required tables are allowed, if so, get it from cache - if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed.get()) { + if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed.get() || + (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getTableMeta(catName, dbNames, tableNames, tableTypes); } return sharedCache.getTableMeta(StringUtils.normalizeIdentifier(catName), @@ -1134,6 +1449,9 @@ public void updateCreationMetadata(String catName, String dbname, String tablena @Override public List getTableObjectsByName(String catName, String dbName, List tblNames) throws MetaException, UnknownDBException { + if (canUseEvents && rawStore.isActiveTransaction()) { + return rawStore.getTableObjectsByName(catName, dbName, tblNames); + } dbName = normalizeIdentifier(dbName); catName = normalizeIdentifier(catName); boolean missSomeInCache = false; @@ -1168,7 +1486,8 @@ public void updateCreationMetadata(String catName, String dbname, String tablena @Override public List getAllTables(String catName, String dbName) throws MetaException { - if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed.get()) { + if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed.get() || + (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getAllTables(catName, dbName); } return sharedCache.listCachedTableNames(StringUtils.normalizeIdentifier(catName), @@ -1188,7 +1507,7 @@ public void updateCreationMetadata(String catName, String dbname, String tablena catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!shouldCacheTable(catName, dbName, tblName)) { + if (!shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.listPartitionNames(catName, dbName, tblName, max_parts); } Table tbl = sharedCache.getTableFromCache(catName, dbName, tblName); @@ -1218,6 +1537,10 @@ public Partition alterPartition(String catName, String dbName, String tblName, List partVals, Partition newPart, String validWriteIds) throws InvalidObjectException, MetaException { newPart = rawStore.alterPartition(catName, dbName, tblName, partVals, newPart, validWriteIds); + // in case of event based cache update, cache will be updated during commit. + if (canUseEvents) { + return newPart; + } catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); tblName = normalizeIdentifier(tblName); @@ -1235,6 +1558,10 @@ public Partition alterPartition(String catName, String dbName, String tblName, throws InvalidObjectException, MetaException { newParts = rawStore.alterPartitions( catName, dbName, tblName, partValsList, newParts, writeId, validWriteIds); + // in case of event based cache update, cache will be updated during commit. + if (canUseEvents) { + return newParts; + } catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); tblName = normalizeIdentifier(tblName); @@ -1286,7 +1613,7 @@ public boolean getPartitionsByExpr(String catName, String dbName, String tblName catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!shouldCacheTable(catName, dbName, tblName)) { + if (!shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getPartitionsByExpr(catName, dbName, tblName, expr, defaultPartitionName, maxParts, result); } List partNames = new LinkedList<>(); @@ -1317,7 +1644,7 @@ public int getNumPartitionsByExpr(String catName, String dbName, String tblName, catName = normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!shouldCacheTable(catName, dbName, tblName)) { + if (!shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getNumPartitionsByExpr(catName, dbName, tblName, expr); } String defaultPartName = MetastoreConf.getVar(getConf(), ConfVars.DEFAULTPARTITIONNAME); @@ -1332,7 +1659,8 @@ public int getNumPartitionsByExpr(String catName, String dbName, String tblName, return partNames.size(); } - private static List partNameToVals(String name) { + @VisibleForTesting + public static List partNameToVals(String name) { if (name == null) { return null; } @@ -1350,7 +1678,7 @@ public int getNumPartitionsByExpr(String catName, String dbName, String tblName, catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!shouldCacheTable(catName, dbName, tblName)) { + if (!shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getPartitionsByNames(catName, dbName, tblName, partNames); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); @@ -1537,7 +1865,7 @@ public Partition getPartitionWithAuth(String catName, String dbName, String tblN catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!shouldCacheTable(catName, dbName, tblName)) { + if (!shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getPartitionWithAuth(catName, dbName, tblName, partVals, userName, groupNames); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); @@ -1562,7 +1890,7 @@ public Partition getPartitionWithAuth(String catName, String dbName, String tblN catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!shouldCacheTable(catName, dbName, tblName)) { + if (!shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.getPartitionsWithAuth(catName, dbName, tblName, maxParts, userName, groupNames); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); @@ -1591,7 +1919,7 @@ public Partition getPartitionWithAuth(String catName, String dbName, String tblN catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!shouldCacheTable(catName, dbName, tblName)) { + if (!shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.listPartitionNamesPs(catName, dbName, tblName, partSpecs, maxParts); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); @@ -1620,7 +1948,7 @@ public Partition getPartitionWithAuth(String catName, String dbName, String tblN catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!shouldCacheTable(catName, dbName, tblName)) { + if (!shouldCacheTable(catName, dbName, tblName) || (canUseEvents && rawStore.isActiveTransaction())) { return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partSpecs, maxParts, userName, groupNames); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); @@ -1702,29 +2030,58 @@ private ColumnStatistics adjustColStatForGet(Map tableParams, return colStat; } + private static void updateTableColumnsStatsInternal(Configuration conf, ColumnStatistics colStats, + Map newParams, String validWriteIds, + long writeId) throws MetaException { + String catName = colStats.getStatsDesc().isSetCatName() ? + normalizeIdentifier(colStats.getStatsDesc().getCatName()) : + getDefaultCatalog(conf); + String dbName = normalizeIdentifier(colStats.getStatsDesc().getDbName()); + String tblName = normalizeIdentifier(colStats.getStatsDesc().getTableName()); + if (!shouldCacheTable(catName, dbName, tblName)) { + return; + } + Table table = sharedCache.getTableFromCache(catName, dbName, tblName); + if (table == null) { + // The table is not yet loaded in cache + return; + } + + boolean isTxn = TxnUtils.isTransactionalTable(table.getParameters()); + if (isTxn && validWriteIds != null) { + if (!areTxnStatsSupported) { + StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); + } else { + String errorMsg = ObjectStore.verifyStatsChangeCtx( + table.getParameters(), newParams, writeId, validWriteIds, true); + if (errorMsg != null) { + throw new MetaException(errorMsg); + } + if (!ObjectStore.isCurrentStatsValidForTheQuery(conf, newParams, table.getWriteId(), + validWriteIds, true)) { + // Make sure we set the flag to invalid regardless of the current value. + StatsSetupConst.setBasicStatsState(newParams, StatsSetupConst.FALSE); + LOG.info("Removed COLUMN_STATS_ACCURATE from the parameters of the table " + + table.getDbName() + "." + table.getTableName()); + } + } + } + + table.setWriteId(writeId); + table.setParameters(newParams); + sharedCache.alterTableInCache(catName, dbName, tblName, table); + sharedCache.updateTableColStatsInCache(catName, dbName, tblName, colStats.getStatsObj()); + } + @Override public Map updateTableColumnStatistics(ColumnStatistics colStats, String validWriteIds, long writeId) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { Map newParams = rawStore.updateTableColumnStatistics( colStats, validWriteIds, writeId); - if (newParams != null) { - String catName = colStats.getStatsDesc().isSetCatName() ? - normalizeIdentifier(colStats.getStatsDesc().getCatName()) : - getDefaultCatalog(conf); - String dbName = normalizeIdentifier(colStats.getStatsDesc().getDbName()); - String tblName = normalizeIdentifier(colStats.getStatsDesc().getTableName()); - if (!shouldCacheTable(catName, dbName, tblName)) { - return newParams; - } - Table table = sharedCache.getTableFromCache(catName, dbName, tblName); - if (table == null) { - // The table is not yet loaded in cache - return newParams; - } - table.setParameters(newParams); - sharedCache.alterTableInCache(catName, dbName, tblName, table); - sharedCache.updateTableColStatsInCache(catName, dbName, tblName, colStats.getStatsObj()); + // in case of event based cache update, cache will be updated during commit. + if (newParams != null && !canUseEvents) { + updateTableColumnsStatsInternal(conf, colStats, newParams, null, writeId); } return newParams; } @@ -1765,7 +2122,8 @@ public boolean deleteTableColumnStatistics(String catName, String dbName, String String colName) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { boolean succ = rawStore.deleteTableColumnStatistics(catName, dbName, tblName, colName); - if (succ) { + // in case of event based cache update, cache is updated during commit txn + if (succ && !canUseEvents) { catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); tblName = normalizeIdentifier(tblName); @@ -1783,7 +2141,8 @@ public boolean deleteTableColumnStatistics(String catName, String dbName, String throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { Map newParams = rawStore.updatePartitionColumnStatistics( colStats, partVals, validWriteIds, writeId); - if (newParams != null) { + // in case of event based cache update, cache is updated during commit txn + if (newParams != null && !canUseEvents) { String catName = colStats.getStatsDesc().isSetCatName() ? normalizeIdentifier(colStats.getStatsDesc().getCatName()) : DEFAULT_CATALOG_NAME; String dbName = normalizeIdentifier(colStats.getStatsDesc().getDbName()); @@ -1822,7 +2181,8 @@ public boolean deletePartitionColumnStatistics(String catName, String dbName, St throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { boolean succ = rawStore.deletePartitionColumnStatistics(catName, dbName, tblName, partName, partVals, colName); - if (succ) { + // in case of event based cache update, cache is updated during commit txn. + if (succ && !canUseEvents) { catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); tblName = normalizeIdentifier(tblName); @@ -1851,8 +2211,9 @@ public AggrStats get_aggr_stats_for(String catName, String dbName, String tblNam tblName = StringUtils.normalizeIdentifier(tblName); // TODO: we currently cannot do transactional checks for stats here // (incl. due to lack of sync w.r.t. the below rawStore call). - if (!shouldCacheTable(catName, dbName, tblName) || writeIdList != null) { - rawStore.get_aggr_stats_for( + //TODO : need to calculate aggregate locally in cached store + if (!shouldCacheTable(catName, dbName, tblName) || writeIdList != null || canUseEvents) { + return rawStore.get_aggr_stats_for( catName, dbName, tblName, partNames, colNames, writeIdList); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); @@ -2246,6 +2607,10 @@ public int getDatabaseCount() throws MetaException { // TODO constraintCache List constraintNames = rawStore.createTableWithConstraints(tbl, primaryKeys, foreignKeys, uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints); + // in case of event based cache update, cache is updated during commit. + if (canUseEvents) { + return constraintNames; + } String dbName = normalizeIdentifier(tbl.getDbName()); String tblName = normalizeIdentifier(tbl.getTableName()); String catName = tbl.isSetCatName() ? normalizeIdentifier(tbl.getCatName()) : diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java index c24e7160ac..ce9e383f70 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java @@ -210,7 +210,7 @@ void cachePartition(Partition part, SharedCache sharedCache) { } } - boolean cachePartitions(List parts, SharedCache sharedCache) { + boolean cachePartitions(Iterable parts, SharedCache sharedCache) { try { tableLock.writeLock().lock(); for (Partition part : parts) { @@ -292,6 +292,9 @@ public Partition removePartition(List partVal, SharedCache sharedCache) tableLock.writeLock().lock(); PartitionWrapper wrapper = partitionCache.remove(CacheUtils.buildPartitionCacheKey(partVal)); + if (wrapper == null) { + return null; + } isPartitionCacheDirty.set(true); part = CacheUtils.assemble(wrapper, sharedCache); if (wrapper.getSdHash() != null) { @@ -1171,6 +1174,10 @@ public void removeTableFromCache(String catName, String dbName, String tblName) } TableWrapper tblWrapper = tableCache.remove(CacheUtils.buildTableKey(catName, dbName, tblName)); + if (tblWrapper == null) { + //in case of retry, ignore second try. + return; + } byte[] sdHash = tblWrapper.getSdHash(); if (sdHash != null) { decrSd(sdHash); @@ -1408,7 +1415,7 @@ public void addPartitionToCache(String catName, String dbName, String tblName, P } public void addPartitionsToCache(String catName, String dbName, String tblName, - List parts) { + Iterable parts) { try { cacheLock.readLock().lock(); TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/DeletePartitionColumnStatEvent.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/DeletePartitionColumnStatEvent.java new file mode 100644 index 0000000000..d64b57d493 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/DeletePartitionColumnStatEvent.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; + +import java.util.List; + +/** + * DeletePartitionColumnStatEvent + * Event generated for partition column stat delete event. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public class DeletePartitionColumnStatEvent extends ListenerEvent { + private String catName, dbName, tableName, colName, partName; + + private List partVals; + + /** + * @param catName catalog name + * @param dbName database name + * @param tableName table name + * @param partName partition column name + * @param partVals partition value + * @param colName column name + * @param handler handler that is firing the event + */ + public DeletePartitionColumnStatEvent(String catName, String dbName, String tableName, String partName, + List partVals, String colName, IHMSHandler handler) { + super(true, handler); + this.catName = catName; + this.dbName = dbName; + this.tableName = tableName; + this.colName = colName; + this.partName = partName; + this.partVals = partVals; + } + + public String getCatName() { + return catName; + } + + public String getDBName() { + return dbName; + } + + public String getTableName() { + return tableName; + } + + public String getColName() { + return colName; + } + + public String getPartName() { + return partName; + } + + public List getPartVals() { + return partVals; + } +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/DeleteTableColumnStatEvent.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/DeleteTableColumnStatEvent.java new file mode 100644 index 0000000000..7638744281 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/DeleteTableColumnStatEvent.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; + +/** + * DeleteTableColumnStatEvent + * Event generated for table column stat delete event. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public class DeleteTableColumnStatEvent extends ListenerEvent { + private String catName, dbName, tableName, colName; + + /** + * @param catName catalog name + * @param dbName database name + * @param tableName table name + * @param colName column name + * @param handler handler that is firing the event + */ + public DeleteTableColumnStatEvent(String catName, String dbName, String tableName, String colName, IHMSHandler handler) { + super(true, handler); + this.catName = catName; + this.dbName = dbName; + this.tableName = tableName; + this.colName = colName; + } + + public String getCatName() { + return catName; + } + + public String getDBName() { + return dbName; + } + + public String getTableName() { + return tableName; + } + + public String getColName() { + return colName; + } +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdatePartitionColumnStatEvent.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdatePartitionColumnStatEvent.java new file mode 100644 index 0000000000..a61b98c702 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdatePartitionColumnStatEvent.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; + +import java.util.List; +import java.util.Map; + +/** + * UpdatePartitionColumnStatEvent + * Event generated for partition column stat update event. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public class UpdatePartitionColumnStatEvent extends ListenerEvent { + private ColumnStatistics partColStats; + private String validWriteIds; + private long writeId; + private Map parameters; + private List partVals; + + /** + * @param statsObj Columns statistics Info. + * @param partVals partition names + * @param parameters table parameters to be updated after stats are updated. + * @param validWriteIds valid write id list for the query. + * @param writeId writeId for the query. + * @param handler handler that is firing the event + */ + public UpdatePartitionColumnStatEvent(ColumnStatistics statsObj, List partVals, Map parameters, + String validWriteIds, long writeId, IHMSHandler handler) { + super(true, handler); + this.partColStats = statsObj; + this.validWriteIds = validWriteIds; + this.writeId = writeId; + this.parameters = parameters; + this.partVals = partVals; + } + + /** + * @param statsObj Columns statistics Info. + * @param partVals partition names + * @param handler handler that is firing the event + */ + public UpdatePartitionColumnStatEvent(ColumnStatistics statsObj, List partVals, IHMSHandler handler) { + super(true, handler); + this.partColStats = statsObj; + this.partVals = partVals; + this.validWriteIds = null; + this.writeId = 0; + this.parameters = null; + } + + public ColumnStatistics getPartColStats() { + return partColStats; + } + + public String getValidWriteIds() { + return validWriteIds; + } + + public long getWriteId() { + return writeId; + } + + public Map getPartParameters() { + return parameters; + } + + public List getPartVals() { + return partVals; + } +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdateTableColumnStatEvent.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdateTableColumnStatEvent.java new file mode 100644 index 0000000000..cf236179a4 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdateTableColumnStatEvent.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; + +import java.util.List; +import java.util.Map; + +/** + * UpdateTableColumnStatEvent + * Event generated for table column stat update event. + */ +@InterfaceAudience.Public +@InterfaceStability.Stable +public class UpdateTableColumnStatEvent extends ListenerEvent { + private ColumnStatistics colStats; + private String validWriteIds; + private long writeId; + private Map parameters; + + /** + * @param colStats Columns statistics Info. + * @param parameters table parameters to be updated after stats are updated. + * @param validWriteIds valid write id list for the query. + * @param colStats writeId for the query. + * @param handler handler that is firing the event + */ + public UpdateTableColumnStatEvent(ColumnStatistics colStats, Map parameters, String validWriteIds, + long writeId, IHMSHandler handler) { + super(true, handler); + this.colStats = colStats; + this.validWriteIds = validWriteIds; + this.writeId = writeId; + this.parameters = parameters; + } + + /** + * @param colStats Columns statistics Info. + * @param handler handler that is firing the event + */ + public UpdateTableColumnStatEvent(ColumnStatistics colStats, IHMSHandler handler) { + super(true, handler); + this.colStats = colStats; + this.validWriteIds = null; + this.writeId = 0; + this.parameters = null; + } + + public ColumnStatistics getColStats() { + return colStats; + } + + public String getValidWriteIds() { + return validWriteIds; + } + + public long getWriteId() { + return writeId; + } + + public Map getTableParameters() { + return parameters; + } +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DeletePartitionColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DeletePartitionColumnStatMessage.java new file mode 100644 index 0000000000..d8166c3ce2 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DeletePartitionColumnStatMessage.java @@ -0,0 +1,36 @@ +/* * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging; + +import java.util.List; + +/** + * HCat message sent when an partition column statistics is deleted. + */ +public abstract class DeletePartitionColumnStatMessage extends EventMessage { + + protected DeletePartitionColumnStatMessage() { + super(EventType.DELETE_PARTITION_COLUMN_STAT); + } + + public abstract String getColName(); + + public abstract String getPartName(); + + public abstract List getPartValues(); +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DeleteTableColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DeleteTableColumnStatMessage.java new file mode 100644 index 0000000000..61893b23d8 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DeleteTableColumnStatMessage.java @@ -0,0 +1,30 @@ +/* * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging; + +/** + * HCat message sent when an table table statistics is deleted. + */ +public abstract class DeleteTableColumnStatMessage extends EventMessage { + + protected DeleteTableColumnStatMessage() { + super(EventType.DELETE_TABLE_COLUMN_STAT); + } + + public abstract String getColName(); +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java index 1262c12475..8b3e474f4b 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java @@ -61,7 +61,11 @@ ABORT_TXN(MessageBuilder.ABORT_TXN_EVENT), ALLOC_WRITE_ID(MessageBuilder.ALLOC_WRITE_ID_EVENT), ALTER_CATALOG(MessageBuilder.ALTER_CATALOG_EVENT), - ACID_WRITE(MessageBuilder.ACID_WRITE_EVENT); + ACID_WRITE(MessageBuilder.ACID_WRITE_EVENT), + UPDATE_TABLE_COLUMN_STAT(MessageBuilder.UPDATE_TBL_COL_STAT_EVENT), + DELETE_TABLE_COLUMN_STAT(MessageBuilder.DELETE_TBL_COL_STAT_EVENT), + UPDATE_PARTITION_COLUMN_STAT(MessageBuilder.UPDATE_PART_COL_STAT_EVENT), + DELETE_PARTITION_COLUMN_STAT(MessageBuilder.DELETE_PART_COL_STAT_EVENT); private String typeString; diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java index 787b9b2d77..6add6c2e45 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TxnToWriteId; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.events.AcidWriteEvent; import org.apache.hadoop.hive.metastore.messaging.json.JSONAbortTxnMessage; @@ -73,6 +74,10 @@ import org.apache.hadoop.hive.metastore.messaging.json.JSONDropTableMessage; import org.apache.hadoop.hive.metastore.messaging.json.JSONInsertMessage; import org.apache.hadoop.hive.metastore.messaging.json.JSONOpenTxnMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONUpdateTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONUpdatePartitionColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONDeleteTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONDeletePartitionColumnStatMessage; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.thrift.TBase; import org.apache.thrift.TDeserializer; @@ -118,6 +123,10 @@ public static final String ALLOC_WRITE_ID_EVENT = "ALLOC_WRITE_ID_EVENT"; public static final String ALTER_CATALOG_EVENT = "ALTER_CATALOG"; public static final String ACID_WRITE_EVENT = "ACID_WRITE_EVENT"; + public static final String UPDATE_TBL_COL_STAT_EVENT = "UPDATE_TBL_COL_STAT_EVENT"; + public static final String DELETE_TBL_COL_STAT_EVENT = "DELETE_TBL_COL_STAT_EVENT"; + public static final String UPDATE_PART_COL_STAT_EVENT = "UPDATE_PART_COL_STAT_EVENT"; + public static final String DELETE_PART_COL_STAT_EVENT = "DELETE_PART_COL_STAT_EVENT"; protected static final Configuration conf = MetastoreConf.newMetastoreConf(); @@ -277,6 +286,30 @@ public AcidWriteMessage buildAcidWriteMessage(AcidWriteEvent acidWriteEvent, files); } + public JSONUpdateTableColumnStatMessage buildUpdateTableColumnStatMessage(ColumnStatistics colStats, + Map parameters, + String validWriteIds, long writeId) { + return new JSONUpdateTableColumnStatMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, now(), colStats, parameters, + validWriteIds, writeId); + } + + public JSONDeleteTableColumnStatMessage buildDeleteTableColumnStatMessage(String dbName, String colName) { + return new JSONDeleteTableColumnStatMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, now(), dbName, colName); + } + + public JSONUpdatePartitionColumnStatMessage buildUpdatePartitionColumnStatMessage(ColumnStatistics colStats, + List partVals, Map parameters, + String validWriteIds, long writeId) { + return new JSONUpdatePartitionColumnStatMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, now(), colStats, partVals, + parameters, validWriteIds, writeId); + } + + public JSONDeletePartitionColumnStatMessage buildDeletePartitionColumnStatMessage(String dbName, String colName, + String partName, List partValues) { + return new JSONDeletePartitionColumnStatMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, now(), dbName, + colName, partName, partValues); + } + private long now() { return System.currentTimeMillis() / 1000; } @@ -342,6 +375,11 @@ public static Table getTableObj(ObjectNode jsonTree) throws Exception { return tableObj; } + public static String createTableColumnStatJson(ColumnStatistics tableColumnStat) throws TException { + TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); + return serializer.toString(tableColumnStat, "UTF-8"); + } + /* * TODO: Some thoughts here : We have a current todo to move some of these methods over to * MessageFactory instead of being here, so we can override them, but before we move them over, diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java index b701d84ac4..8a7031fe97 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java @@ -74,6 +74,14 @@ public EventMessage getEventMessage(String eventTypeString, String messageBody) return getAllocWriteIdMessage(messageBody); case ACID_WRITE: return getAcidWriteMessage(messageBody); + case UPDATE_TABLE_COLUMN_STAT: + return getUpdateTableColumnStatMessage(messageBody); + case DELETE_TABLE_COLUMN_STAT: + return getDeleteTableColumnStatMessage(messageBody); + case UPDATE_PARTITION_COLUMN_STAT: + return getUpdatePartitionColumnStatMessage(messageBody); + case DELETE_PARTITION_COLUMN_STAT: + return getDeletePartitionColumnStatMessage(messageBody); default: throw new IllegalArgumentException("Unsupported event-type: " + eventTypeString); } @@ -195,6 +203,26 @@ public EventMessage getEventMessage(String eventTypeString, String messageBody) */ public abstract AcidWriteMessage getAcidWriteMessage(String messageBody); + /** + * Method to de-serialize UpdateTableColumnStatMessage instance. + */ + public abstract UpdateTableColumnStatMessage getUpdateTableColumnStatMessage(String messageBody); + + /** + * Method to de-serialize DeleteTableColumnStatMessage instance. + */ + public abstract DeleteTableColumnStatMessage getDeleteTableColumnStatMessage(String messageBody); + + /** + * Method to de-serialize UpdatePartitionColumnStatMessage instance. + */ + public abstract UpdatePartitionColumnStatMessage getUpdatePartitionColumnStatMessage(String messageBody); + + /** + * Method to de-serialize DeletePartitionColumnStatMessage instance. + */ + public abstract DeletePartitionColumnStatMessage getDeletePartitionColumnStatMessage(String messageBody); + // Protection against construction. protected MessageDeserializer() {} } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/UpdatePartitionColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/UpdatePartitionColumnStatMessage.java new file mode 100644 index 0000000000..d15c6e691c --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/UpdatePartitionColumnStatMessage.java @@ -0,0 +1,42 @@ +/* * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging; + +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import java.util.List; +import java.util.Map; + +/** + * HCat message sent when an table partition statistics update is done. + */ +public abstract class UpdatePartitionColumnStatMessage extends EventMessage { + + protected UpdatePartitionColumnStatMessage() { + super(EventType.UPDATE_TABLE_COLUMN_STAT); + } + + public abstract ColumnStatistics getColumnStatistics(); + + public abstract String getValidWriteIds(); + + public abstract Long getWriteId(); + + public abstract Map getParameters(); + + public abstract List getPartVals(); +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/UpdateTableColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/UpdateTableColumnStatMessage.java new file mode 100644 index 0000000000..ed8944b3da --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/UpdateTableColumnStatMessage.java @@ -0,0 +1,39 @@ +/* * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging; + +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import java.util.Map; + +/** + * HCat message sent when an table column statistics update is done. + */ +public abstract class UpdateTableColumnStatMessage extends EventMessage { + + protected UpdateTableColumnStatMessage() { + super(EventType.UPDATE_TABLE_COLUMN_STAT); + } + + public abstract ColumnStatistics getColumnStatistics(); + + public abstract String getValidWriteIds(); + + public abstract Long getWriteId(); + + public abstract Map getParameters(); +} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDeletePartitionColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDeletePartitionColumnStatMessage.java new file mode 100644 index 0000000000..a06348e56b --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDeletePartitionColumnStatMessage.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging.json; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.hive.metastore.messaging.DeletePartitionColumnStatMessage; +import java.util.List; + +/** + * JSON implementation of JSONDeletePartitionColumnStatMessage + */ +public class JSONDeletePartitionColumnStatMessage extends DeletePartitionColumnStatMessage { + + @JsonProperty + private Long timestamp; + + @JsonProperty + private String server, servicePrincipal, database, colName, partName; + + @JsonProperty + private List partValues; + + /** + * Default constructor, needed for Jackson. + */ + public JSONDeletePartitionColumnStatMessage() { + } + + public JSONDeletePartitionColumnStatMessage(String server, String servicePrincipal, Long timestamp, + String dbName, String colName, String partName, + List partValues) { + this.timestamp = timestamp; + this.server = server; + this.servicePrincipal = servicePrincipal; + this.database = dbName; + this.partValues = partValues; + this.partName = partName; + this.colName = colName; + } + + @Override + public Long getTimestamp() { + return timestamp; + } + + @Override + public String getDB() { + return database; + } + + @Override + public String getServicePrincipal() { + return servicePrincipal; + } + + @Override + public String getServer() { + return server; + } + + @Override + public String getColName() { + return colName; + } + + @Override + public String getPartName() { + return partName; + } + + @Override + public List getPartValues() { + return partValues; + } + + @Override + public String toString() { + try { + return JSONMessageDeserializer.mapper.writeValueAsString(this); + } catch (Exception exception) { + throw new IllegalArgumentException("Could not serialize: ", exception); + } + } +} + diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDeleteTableColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDeleteTableColumnStatMessage.java new file mode 100644 index 0000000000..4fdba253f3 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDeleteTableColumnStatMessage.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging.json; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.hive.metastore.messaging.DeleteTableColumnStatMessage; + +/** + * JSON implementation of JSONDeleteTableColumnStatMessage + */ +public class JSONDeleteTableColumnStatMessage extends DeleteTableColumnStatMessage { + + @JsonProperty + private Long timestamp; + + @JsonProperty + private String server, servicePrincipal, database, colName; + + /** + * Default constructor, needed for Jackson. + */ + public JSONDeleteTableColumnStatMessage() { + } + + public JSONDeleteTableColumnStatMessage(String server, String servicePrincipal, Long timestamp, + String dbName, String colName) { + this.timestamp = timestamp; + this.server = server; + this.servicePrincipal = servicePrincipal; + this.database = dbName; + this.colName = colName; + } + + @Override + public Long getTimestamp() { + return timestamp; + } + + @Override + public String getDB() { + return database; + } + + @Override + public String getServicePrincipal() { + return servicePrincipal; + } + + @Override + public String getServer() { + return server; + } + + @Override + public String getColName() { + return colName; + } + + @Override + public String toString() { + try { + return JSONMessageDeserializer.mapper.writeValueAsString(this); + } catch (Exception exception) { + throw new IllegalArgumentException("Could not serialize: ", exception); + } + } +} + diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java index 3a480aa19a..ef7a947b28 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java @@ -42,6 +42,10 @@ import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer; import org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage; import org.apache.hadoop.hive.metastore.messaging.AcidWriteMessage; +import org.apache.hadoop.hive.metastore.messaging.UpdateTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.DeleteTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.UpdatePartitionColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.DeletePartitionColumnStatMessage; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -270,4 +274,40 @@ public AcidWriteMessage getAcidWriteMessage(String messageBody) { throw new IllegalArgumentException("Could not construct AcidWriteMessage", e); } } + + @Override + public UpdateTableColumnStatMessage getUpdateTableColumnStatMessage(String messageBody) { + try { + return mapper.readValue(messageBody, JSONUpdateTableColumnStatMessage.class); + } catch (Exception e) { + throw new IllegalArgumentException("Could not construct UpdateTableColumnStatMessage", e); + } + } + + @Override + public DeleteTableColumnStatMessage getDeleteTableColumnStatMessage(String messageBody) { + try { + return mapper.readValue(messageBody, JSONDeleteTableColumnStatMessage.class); + } catch (Exception e) { + throw new IllegalArgumentException("Could not construct UpdateTableColumnStatMessage", e); + } + } + + @Override + public UpdatePartitionColumnStatMessage getUpdatePartitionColumnStatMessage(String messageBody) { + try { + return mapper.readValue(messageBody, JSONUpdatePartitionColumnStatMessage.class); + } catch (Exception e) { + throw new IllegalArgumentException("Could not construct UpdatePartitionColumnStatMessage", e); + } + } + + @Override + public DeletePartitionColumnStatMessage getDeletePartitionColumnStatMessage(String messageBody) { + try { + return mapper.readValue(messageBody, JSONDeletePartitionColumnStatMessage.class); + } catch (Exception e) { + throw new IllegalArgumentException("Could not construct UpdatePartitionColumnStatMessage", e); + } + } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONUpdatePartitionColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONUpdatePartitionColumnStatMessage.java new file mode 100644 index 0000000000..3f20091976 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONUpdatePartitionColumnStatMessage.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging.json; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.messaging.MessageBuilder; +import org.apache.hadoop.hive.metastore.messaging.UpdatePartitionColumnStatMessage; +import org.apache.thrift.TException; + +import java.util.List; +import java.util.Map; + +/** + * JSON implementation of JSONUpdatePartitionColumnStatMessage + */ +public class JSONUpdatePartitionColumnStatMessage extends UpdatePartitionColumnStatMessage { + + @JsonProperty + private Long writeId, timestamp; + + @JsonProperty + private String validWriteIds, server, servicePrincipal, database; + + @JsonProperty + private String colStatsJson; + + @JsonProperty + private Map parameters; + + @JsonProperty + private List partVals; + + /** + * Default constructor, needed for Jackson. + */ + public JSONUpdatePartitionColumnStatMessage() { + } + + public JSONUpdatePartitionColumnStatMessage(String server, String servicePrincipal, Long timestamp, + ColumnStatistics colStats, List partVals, + Map parameters, String validWriteIds, long writeId) { + this.timestamp = timestamp; + this.server = server; + this.servicePrincipal = servicePrincipal; + this.writeId = writeId; + this.validWriteIds = validWriteIds; + this.database = colStats.getStatsDesc().getDbName(); + this.partVals = partVals; + try { + this.colStatsJson = MessageBuilder.createTableColumnStatJson(colStats); + } catch (TException e) { + throw new IllegalArgumentException("Could not serialize JSONUpdatePartitionColumnStatMessage : ", e); + } + this.parameters = parameters; + } + + @Override + public Long getTimestamp() { + return timestamp; + } + + @Override + public String getDB() { + return database; + } + + @Override + public String getServicePrincipal() { + return servicePrincipal; + } + + @Override + public String getServer() { + return server; + } + + @Override + public ColumnStatistics getColumnStatistics() { + try { + return (ColumnStatistics) MessageBuilder.getTObj(colStatsJson, ColumnStatistics.class); + } catch (Exception e) { + throw new RuntimeException("failed to get the ColumnStatistics object ", e); + } + } + + @Override + public String getValidWriteIds() { + return validWriteIds; + } + + @Override + public Long getWriteId() { + return writeId; + } + + @Override + public Map getParameters() { + return parameters; + } + + @Override + public List getPartVals() { + return partVals; + } + + @Override + public String toString() { + try { + return JSONMessageDeserializer.mapper.writeValueAsString(this); + } catch (Exception exception) { + throw new IllegalArgumentException("Could not serialize: ", exception); + } + } +} + diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONUpdateTableColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONUpdateTableColumnStatMessage.java new file mode 100644 index 0000000000..d99ef1197e --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONUpdateTableColumnStatMessage.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging.json; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.messaging.MessageBuilder; +import org.apache.hadoop.hive.metastore.messaging.UpdateTableColumnStatMessage; +import org.apache.thrift.TException; +import java.util.Map; + +/** + * JSON implementation of JSONUpdateTableColumnStatMessage + */ +public class JSONUpdateTableColumnStatMessage extends UpdateTableColumnStatMessage { + + @JsonProperty + private Long writeId, timestamp; + + @JsonProperty + private String validWriteIds, server, servicePrincipal, database; + + @JsonProperty + private String colStatsJson; + + @JsonProperty + Map parameters; + + /** + * Default constructor, needed for Jackson. + */ + public JSONUpdateTableColumnStatMessage() { + } + + public JSONUpdateTableColumnStatMessage(String server, String servicePrincipal, Long timestamp, + ColumnStatistics colStats, Map parameters, String validWriteIds, long writeId) { + this.timestamp = timestamp; + this.server = server; + this.servicePrincipal = servicePrincipal; + this.writeId = writeId; + this.validWriteIds = validWriteIds; + this.database = colStats.getStatsDesc().getDbName(); + try { + this.colStatsJson = MessageBuilder.createTableColumnStatJson(colStats); + } catch (TException e) { + throw new IllegalArgumentException("Could not serialize JSONUpdateTableColumnStatMessage : ", e); + } + this.parameters = parameters; + } + + @Override + public Long getTimestamp() { + return timestamp; + } + + @Override + public String getDB() { + return database; + } + + @Override + public String getServicePrincipal() { + return servicePrincipal; + } + + @Override + public String getServer() { + return server; + } + + @Override + public ColumnStatistics getColumnStatistics() { + try { + return (ColumnStatistics) MessageBuilder.getTObj(colStatsJson, ColumnStatistics.class); + } catch (Exception e) { + throw new RuntimeException("failed to get the ColumnStatistics object ", e); + } + } + + @Override + public String getValidWriteIds() { + return validWriteIds; + } + + @Override + public Long getWriteId() { + return writeId; + } + + @Override + public Map getParameters() { + return parameters; + } + + @Override + public String toString() { + try { + return JSONMessageDeserializer.mapper.writeValueAsString(this); + } catch (Exception exception) { + throw new IllegalArgumentException("Could not serialize: ", exception); + } + } +} + diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/gzip/DeSerializer.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/gzip/DeSerializer.java index 49596b5a71..e951481fa8 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/gzip/DeSerializer.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/gzip/DeSerializer.java @@ -42,6 +42,10 @@ import org.apache.hadoop.hive.metastore.messaging.DropTableMessage; import org.apache.hadoop.hive.metastore.messaging.InsertMessage; import org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage; +import org.apache.hadoop.hive.metastore.messaging.UpdateTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.UpdatePartitionColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.DeleteTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.DeletePartitionColumnStatMessage; import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageDeserializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -196,4 +200,24 @@ public AllocWriteIdMessage getAllocWriteIdMessage(String messageBody) { public AcidWriteMessage getAcidWriteMessage(String messageBody) { return super.getAcidWriteMessage(deCompress(messageBody)); } + + @Override + public UpdateTableColumnStatMessage getUpdateTableColumnStatMessage(String messageBody) { + return super.getUpdateTableColumnStatMessage(deCompress(messageBody)); + } + + @Override + public UpdatePartitionColumnStatMessage getUpdatePartitionColumnStatMessage(String messageBody) { + return super.getUpdatePartitionColumnStatMessage(deCompress(messageBody)); + } + + @Override + public DeleteTableColumnStatMessage getDeleteTableColumnStatMessage(String messageBody) { + return super.getDeleteTableColumnStatMessage(deCompress(messageBody)); + } + + @Override + public DeletePartitionColumnStatMessage getDeletePartitionColumnStatMessage(String messageBody) { + return super.getDeletePartitionColumnStatMessage(deCompress(messageBody)); + } } diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java index 93b2f23e62..88d5e716e1 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java @@ -60,7 +60,7 @@ public void testAlterTableAddColNotUpdateStats() throws MetaException, InvalidOb getDefaultCatalog(conf), oldTable.getDbName(), oldTable.getTableName(), Arrays.asList("col1", "col2", "col3")); HiveAlterHandler handler = new HiveAlterHandler(); handler.setConf(conf); - handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null); + handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null, conf, null); } @Test @@ -86,7 +86,7 @@ public void testAlterTableDelColUpdateStats() throws Exception { HiveAlterHandler handler = new HiveAlterHandler(); handler.setConf(conf); try { - handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null); + handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null, conf, null); } catch (Throwable t) { System.err.println(t); t.printStackTrace(System.err); @@ -121,7 +121,7 @@ public void testAlterTableChangePosNotUpdateStats() throws MetaException, Invali getDefaultCatalog(conf), oldTable.getDbName(), oldTable.getTableName(), Arrays.asList("col1", "col2", "col3", "col4")); HiveAlterHandler handler = new HiveAlterHandler(); handler.setConf(conf); - handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null); + handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null, conf, null); } }