diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index e4ef46f..5b830c2 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -239,7 +239,7 @@ static void prewarm(RawStore rawStore) { List partitionColStats = null; AggrStats aggrStatsAllPartitions = null; AggrStats aggrStatsAllButDefaultPartition = null; - if (table.isSetPartitionKeys()) { + if (!table.getPartitionKeys().isEmpty()) { Deadline.startTimer("getPartitions"); partitions = rawStore.getPartitions(catName, dbName, tblName, Integer.MAX_VALUE); Deadline.stopTimer(); @@ -310,6 +310,7 @@ static void prewarm(RawStore rawStore) { LOG.debug("Processed database: {}. Cached {} / {} databases so far.", dbName, ++numberOfDatabasesCachedSoFar, databases.size()); } + sharedCache.clearDirtyFlags(); completePrewarm(startTime); } } @@ -549,7 +550,7 @@ private void updateTableColStats(RawStore rawStore, String catName, String dbNam rawStore.openTransaction(); try { Table table = rawStore.getTable(catName, dbName, tblName); - if (!table.isSetPartitionKeys()) { + if (table.getPartitionKeys().isEmpty()) { List colNames = MetaStoreUtils.getColumnNamesForTable(table); Deadline.startTimer("getTableColumnStatistics"); @@ -600,12 +601,6 @@ private void updateTablePartitionColStats(RawStore rawStore, String catName, Str List partitionColStats = rawStore.getPartitionColumnStatistics(catName, dbName, tblName, partNames, colNames); Deadline.stopTimer(); - sharedCache.refreshPartitionColStatsInCache(catName, dbName, tblName, partitionColStats); - List parts = rawStore.getPartitionsByNames(catName, dbName, tblName, partNames); - // Also save partitions for consistency as they have the stats state. - for (Partition part : parts) { - sharedCache.alterPartitionInCache(catName, dbName, tblName, part.getValues(), part); - } committed = rawStore.commitTransaction(); } catch (MetaException | NoSuchObjectException e) { LOG.info("Updating CachedStore: unable to read partitions of table: " + tblName, e); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java index c24e716..5d6a7d8 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java @@ -1057,7 +1057,7 @@ public boolean populateTableInCache(Table table, ColumnStatistics tableColStats, } LOG.debug("Current cache size: {} bytes", currentCacheSizeInBytes); } - if (!table.isSetPartitionKeys() && (tableColStats != null)) { + if (table.getPartitionKeys().isEmpty() && (tableColStats != null)) { if (!tblWrapper.updateTableColStats(tableColStats.getStatsObj())) { return false; } @@ -1085,6 +1085,10 @@ public boolean populateTableInCache(Table table, ColumnStatistics tableColStats, tblWrapper.cacheAggrPartitionColStats(aggrStatsAllPartitions, aggrStatsAllButDefaultPartition); } + tblWrapper.isPartitionCacheDirty.set(false); + tblWrapper.isTableColStatsCacheDirty.set(false); + tblWrapper.isPartitionColStatsCacheDirty.set(false); + tblWrapper.isAggrPartitionColStatsCacheDirty.set(false); try { cacheLock.writeLock().lock(); // 2. Skip overwriting exisiting table object @@ -1288,7 +1292,9 @@ public void refreshTablesInCache(String catName, String dbName, List tabl tableCache.clear(); tableCache = newTableCache; } finally { - cacheLock.writeLock().unlock(); + if (cacheLock.writeLock().isHeldByCurrentThread()) { + cacheLock.writeLock().unlock(); + } } } @@ -1692,6 +1698,12 @@ void resetCatalogCache() { isCatalogCacheDirty.set(false); } + void clearDirtyFlags() { + isCatalogCacheDirty.set(false); + isDatabaseCacheDirty.set(false); + isTableCacheDirty.set(false); + } + public long getUpdateCount() { return cacheUpdateCount.get(); } diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java index bb20d9f..b0cc1fc 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java @@ -374,10 +374,10 @@ public void testPartitionOps() throws Exception { sharedCache.getSdCache().clear(); } - //@Test + @Test public void testTableColStatsOps() throws Exception { // Add a db via ObjectStore - String dbName = "testTableColStatsOps"; + String dbName = "testTableColStatsOps".toLowerCase(); String dbOwner = "user1"; Database db = createTestDb(dbName, dbOwner); objectStore.createDatabase(db); @@ -407,9 +407,7 @@ public void testTableColStatsOps() throws Exception { cols.add(col1); cols.add(col2); cols.add(col3); - FieldSchema ptnCol1 = new FieldSchema("part1", "string", "string partition column"); List ptnCols = new ArrayList(); - ptnCols.add(ptnCol1); Table tbl = createTestTbl(dbName, tblName, tblOwner, cols, ptnCols); objectStore.createTable(tbl); tbl = objectStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName); @@ -465,6 +463,21 @@ public void testTableColStatsOps() throws Exception { ColumnStatistics newStats = cachedStore.getTableColumnStatistics(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(col1.getName(), col2.getName(), col3.getName())); + newStats.setIsStatsCompliantIsSet(false); + Assert.assertEquals(stats, newStats); + + boolStats.setNumNulls(col3Nulls-1); + // Save to DB + objectStore.updateTableColumnStatistics(stats, null, -1); + + // We update twice to accurately detect if cache is dirty or not + updateCache(cachedStore); + updateCache(cachedStore); + + newStats = + cachedStore.getTableColumnStatistics(DEFAULT_CATALOG_NAME, dbName, tblName, + Arrays.asList(col1.getName(), col2.getName(), col3.getName())); + newStats.setIsStatsCompliantIsSet(false); Assert.assertEquals(stats, newStats); // Clean up @@ -664,7 +677,7 @@ public void testSharedStorePartition() { @Test public void testAggrStatsRepeatedRead() throws Exception { - String dbName = "testTableColStatsOps"; + String dbName = "testAggrStatsRepeatedRead"; String tblName = "tbl"; String colName = "f1"; @@ -736,7 +749,7 @@ public void testAggrStatsRepeatedRead() throws Exception { @Test public void testPartitionAggrStats() throws Exception { - String dbName = "testTableColStatsOps1"; + String dbName = "testPartitionAggrStats"; String tblName = "tbl1"; String colName = "f1";