diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index e4ef46f..5c86eb5 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -239,7 +239,7 @@ static void prewarm(RawStore rawStore) { List partitionColStats = null; AggrStats aggrStatsAllPartitions = null; AggrStats aggrStatsAllButDefaultPartition = null; - if (table.isSetPartitionKeys()) { + if (!table.getPartitionKeys().isEmpty()) { Deadline.startTimer("getPartitions"); partitions = rawStore.getPartitions(catName, dbName, tblName, Integer.MAX_VALUE); Deadline.stopTimer(); @@ -310,6 +310,7 @@ static void prewarm(RawStore rawStore) { LOG.debug("Processed database: {}. Cached {} / {} databases so far.", dbName, ++numberOfDatabasesCachedSoFar, databases.size()); } + sharedCache.clearDirtyFlags(); completePrewarm(startTime); } } @@ -549,7 +550,7 @@ private void updateTableColStats(RawStore rawStore, String catName, String dbNam rawStore.openTransaction(); try { Table table = rawStore.getTable(catName, dbName, tblName); - if (!table.isSetPartitionKeys()) { + if (table.getPartitionKeys().isEmpty()) { List colNames = MetaStoreUtils.getColumnNamesForTable(table); Deadline.startTimer("getTableColumnStatistics"); @@ -595,17 +596,19 @@ private void updateTablePartitionColStats(RawStore rawStore, String catName, Str Table table = rawStore.getTable(catName, dbName, tblName); List colNames = MetaStoreUtils.getColumnNamesForTable(table); List partNames = rawStore.listPartitionNames(catName, dbName, tblName, (short) -1); + Deadline.startTimer("getPartitionsByNames"); + List parts = rawStore.getPartitionsByNames(catName, dbName, tblName, partNames); + Deadline.stopTimer(); + // Save partitions for consistency as they have the stats state. + for (Partition part : parts) { + sharedCache.alterPartitionInCache(catName, dbName, tblName, part.getValues(), part); + } // Get partition column stats for this table Deadline.startTimer("getPartitionColumnStatistics"); List partitionColStats = rawStore.getPartitionColumnStatistics(catName, dbName, tblName, partNames, colNames); Deadline.stopTimer(); sharedCache.refreshPartitionColStatsInCache(catName, dbName, tblName, partitionColStats); - List parts = rawStore.getPartitionsByNames(catName, dbName, tblName, partNames); - // Also save partitions for consistency as they have the stats state. - for (Partition part : parts) { - sharedCache.alterPartitionInCache(catName, dbName, tblName, part.getValues(), part); - } committed = rawStore.commitTransaction(); } catch (MetaException | NoSuchObjectException e) { LOG.info("Updating CachedStore: unable to read partitions of table: " + tblName, e); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java index c24e716..5d6a7d8 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java @@ -1057,7 +1057,7 @@ public boolean populateTableInCache(Table table, ColumnStatistics tableColStats, } LOG.debug("Current cache size: {} bytes", currentCacheSizeInBytes); } - if (!table.isSetPartitionKeys() && (tableColStats != null)) { + if (table.getPartitionKeys().isEmpty() && (tableColStats != null)) { if (!tblWrapper.updateTableColStats(tableColStats.getStatsObj())) { return false; } @@ -1085,6 +1085,10 @@ public boolean populateTableInCache(Table table, ColumnStatistics tableColStats, tblWrapper.cacheAggrPartitionColStats(aggrStatsAllPartitions, aggrStatsAllButDefaultPartition); } + tblWrapper.isPartitionCacheDirty.set(false); + tblWrapper.isTableColStatsCacheDirty.set(false); + tblWrapper.isPartitionColStatsCacheDirty.set(false); + tblWrapper.isAggrPartitionColStatsCacheDirty.set(false); try { cacheLock.writeLock().lock(); // 2. Skip overwriting exisiting table object @@ -1288,7 +1292,9 @@ public void refreshTablesInCache(String catName, String dbName, List tabl tableCache.clear(); tableCache = newTableCache; } finally { - cacheLock.writeLock().unlock(); + if (cacheLock.writeLock().isHeldByCurrentThread()) { + cacheLock.writeLock().unlock(); + } } } @@ -1692,6 +1698,12 @@ void resetCatalogCache() { isCatalogCacheDirty.set(false); } + void clearDirtyFlags() { + isCatalogCacheDirty.set(false); + isDatabaseCacheDirty.set(false); + isTableCacheDirty.set(false); + } + public long getUpdateCount() { return cacheUpdateCount.get(); }