diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java index 944c813..d50fa13 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java @@ -40,6 +40,10 @@ public static String buildDbKey(String catName, String dbName) { return buildKey(catName.toLowerCase(), dbName.toLowerCase()); } + public static String buildDbKeyWithDelimiterSuffix(String catName, String dbName) { + return buildKey(catName.toLowerCase(), dbName.toLowerCase()) + delimit; + } + /** * Builds a key for the partition cache which is concatenation of partition values, each value * separated by a delimiter diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 2d738f6..1fac51e 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -490,7 +490,7 @@ static void prewarm(RawStore rawStore) { List partitionColStats = null; AggrStats aggrStatsAllPartitions = null; AggrStats aggrStatsAllButDefaultPartition = null; - if (table.isSetPartitionKeys()) { + if (!table.getPartitionKeys().isEmpty()) { Deadline.startTimer("getPartitions"); partitions = rawStore.getPartitions(catName, dbName, tblName, Integer.MAX_VALUE); Deadline.stopTimer(); @@ -561,6 +561,7 @@ static void prewarm(RawStore rawStore) { LOG.debug("Processed database: {}. Cached {} / {} databases so far.", dbName, ++numberOfDatabasesCachedSoFar, databases.size()); } + sharedCache.clearDirtyFlags(); completePrewarm(startTime); } } @@ -724,6 +725,7 @@ public void run() { } else { try { triggerPreWarm(rawStore); + shouldRunPrewarm = false; } catch (Exception e) { LOG.error("Prewarm failure", e); return; @@ -815,7 +817,6 @@ private void updateTableColStats(RawStore rawStore, String catName, String dbNam if (table != null && !table.isSetPartitionKeys()) { List colNames = MetaStoreUtils.getColumnNamesForTable(table); Deadline.startTimer("getTableColumnStatistics"); - ColumnStatistics tableColStats = rawStore.getTableColumnStatistics(catName, dbName, tblName, colNames); Deadline.stopTimer(); @@ -865,7 +866,9 @@ private void updateTablePartitionColStats(RawStore rawStore, String catName, Str rawStore.getPartitionColumnStatistics(catName, dbName, tblName, partNames, colNames); Deadline.stopTimer(); sharedCache.refreshPartitionColStatsInCache(catName, dbName, tblName, partitionColStats); + Deadline.startTimer("getPartitionsByNames"); List parts = rawStore.getPartitionsByNames(catName, dbName, tblName, partNames); + Deadline.stopTimer(); // Also save partitions for consistency as they have the stats state. for (Partition part : parts) { sharedCache.alterPartitionInCache(catName, dbName, tblName, part.getValues(), part); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java index 1c23022..a0636b6 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java @@ -1200,7 +1200,7 @@ public boolean populateTableInCache(Table table, ColumnStatistics tableColStats, LOG.debug("Current cache size: {} bytes", currentCacheSizeInBytes); } if (!table.isSetPartitionKeys() && (tableColStats != null)) { - if (!tblWrapper.updateTableColStats(tableColStats.getStatsObj())) { + if (table.getPartitionKeys().isEmpty() && (tableColStats != null)) { return false; } } else { @@ -1227,6 +1227,10 @@ public boolean populateTableInCache(Table table, ColumnStatistics tableColStats, tblWrapper.cacheAggrPartitionColStats(aggrStatsAllPartitions, aggrStatsAllButDefaultPartition); } + tblWrapper.isPartitionCacheDirty.set(false); + tblWrapper.isTableColStatsCacheDirty.set(false); + tblWrapper.isPartitionColStatsCacheDirty.set(false); + tblWrapper.isAggrPartitionColStatsCacheDirty.set(false); try { cacheLock.writeLock().lock(); // 2. Skip overwriting exisiting table object @@ -1317,11 +1321,13 @@ public void removeTableFromCache(String catName, String dbName, String tblName) //in case of retry, ignore second try. return; } - byte[] sdHash = tblWrapper.getSdHash(); - if (sdHash != null) { - decrSd(sdHash); + if (tblWrapper != null) { + byte[] sdHash = tblWrapper.getSdHash(); + if (sdHash != null) { + decrSd(sdHash); + } + isTableCacheDirty.set(true); } - isTableCacheDirty.set(true); } finally { cacheLock.writeLock().unlock(); } @@ -1438,27 +1444,34 @@ public void alterTableAndStatsInCache(String catName, String dbName, String tblN public void refreshTablesInCache(String catName, String dbName, List tables) { try { - cacheLock.writeLock().lock(); if (isTableCacheDirty.compareAndSet(true, false)) { LOG.debug("Skipping table cache update; the table list we have is dirty."); return; } - Map newTableCache = new HashMap<>(); + Map newCacheForDB = new TreeMap<>(); for (Table tbl : tables) { String tblName = StringUtils.normalizeIdentifier(tbl.getTableName()); - TableWrapper tblWrapper = - tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { tblWrapper.updateTableObj(tbl, this); } else { tblWrapper = createTableWrapper(catName, dbName, tblName, tbl); } - newTableCache.put(CacheUtils.buildTableKey(catName, dbName, tblName), tblWrapper); + newCacheForDB.put(CacheUtils.buildTableKey(catName, dbName, tblName), tblWrapper); } - tableCache.clear(); - tableCache = newTableCache; + cacheLock.writeLock().lock(); + Iterator> entryIterator = tableCache.entrySet().iterator(); + while (entryIterator.hasNext()) { + String key = entryIterator.next().getKey(); + if (key.startsWith(CacheUtils.buildDbKeyWithDelimiterSuffix(catName, dbName))) { + entryIterator.remove(); + } + } + tableCache.putAll(newCacheForDB); } finally { - cacheLock.writeLock().unlock(); + if (cacheLock.writeLock().isHeldByCurrentThread()) { + cacheLock.writeLock().unlock(); + } } } @@ -1904,6 +1917,12 @@ void resetCatalogCache() { isCatalogCacheDirty.set(false); } + void clearDirtyFlags() { + isCatalogCacheDirty.set(false); + isDatabaseCacheDirty.set(false); + isTableCacheDirty.set(false); + } + public long getUpdateCount() { return cacheUpdateCount.get(); }