diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index a3725c5395..8aceb30031 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -1005,7 +1005,7 @@ public WMFullResourcePlan getResourcePlan(String name) throws NoSuchObjectExcept public List getAllResourcePlans() throws MetaException { return objectStore.getAllResourcePlans(); } - + @Override public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan resourcePlan, boolean canActivateDisabled, boolean canDeactivate, boolean isReplace) @@ -1097,11 +1097,4 @@ public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerNa String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { objectStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); } - - @Override - public List getPartitionColStatsForDatabase(String dbName) - throws MetaException, NoSuchObjectException { - // TODO Auto-generated method stub - return null; - } } diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java index 6c1a0b98cc..bb14493792 100644 --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -151,7 +151,7 @@ public synchronized void init(HiveConf hiveConf) { } // Initialize cachedstore with background prewarm. The prewarm will only start if configured. - CachedStore.initSharedCacheAsync(hiveConf); + CachedStore.initSharedCache(hiveConf); cliService = new CLIService(this); addService(cliService); diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index c6e34a8a22..f86a386a10 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -7753,7 +7753,7 @@ public static void main(String[] args) throws Throwable { }, 10); // This will only initialize the cache if configured. - CachedStore.initSharedCacheAsync(conf); + CachedStore.initSharedCache(conf); //Start Metrics for Standalone (Remote) Mode if (MetastoreConf.getBoolVar(conf, ConfVars.METRICS_ENABLED)) { diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 7b44df4128..2848236c40 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -8022,35 +8022,6 @@ protected String describeResult() { } @Override - public List getPartitionColStatsForDatabase(String dbName) - throws MetaException, NoSuchObjectException { - final boolean enableBitVector = - MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_BITVECTOR); - return new GetHelper>(dbName, null, true, false) { - @Override - protected List getSqlResult( - GetHelper> ctx) throws MetaException { - return directSql.getColStatsForAllTablePartitions(dbName, enableBitVector); - } - - @Override - protected List getJdoResult( - GetHelper> ctx) - throws MetaException, NoSuchObjectException { - // This is fast path for query optimizations, if we can find this info - // quickly using directSql, do it. No point in failing back to slow path - // here. - throw new MetaException("Jdo path is not implemented for getPartitionColStatsForDatabase."); - } - - @Override - protected String describeResult() { - return null; - } - }.run(true); - } - - @Override public void flushCache() { // NOP as there's no caching } @@ -9964,7 +9935,7 @@ public WMFullResourcePlan alterResourcePlan(String name, WMNullableResourcePlan } else { result = handleSimpleAlter(name, changes, canActivateDisabled, canDeactivate); } - + commited = commitTransaction(); return result; } catch (Exception e) { diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index f500d63725..f9e984da27 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -606,17 +606,6 @@ AggrStats get_aggr_stats_for(String dbName, String tblName, List partNames, List colNames) throws MetaException, NoSuchObjectException; /** - * Get column stats for all partitions of all tables in the database - * - * @param dbName - * @return List of column stats objects for all partitions of all tables in the database - * @throws MetaException - * @throws NoSuchObjectException - */ - List getPartitionColStatsForDatabase(String dbName) - throws MetaException, NoSuchObjectException; - - /** * Get the next notification event. * @param rqst Request containing information on the last processed notification. * @return list of notifications, sorted by eventId diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java index f0f650ddcf..7f8dd438cb 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java @@ -27,8 +27,8 @@ import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.cache.CachedStore.PartitionWrapper; -import org.apache.hadoop.hive.metastore.cache.CachedStore.TableWrapper; +import org.apache.hadoop.hive.metastore.cache.SharedCache.PartitionWrapper; +import org.apache.hadoop.hive.metastore.cache.SharedCache.TableWrapper; import org.apache.hadoop.hive.metastore.utils.StringUtils; public class CacheUtils { @@ -59,6 +59,19 @@ public static String buildKey(String dbName, String tableName, List part return key; } + public static String buildKey(List partVals) { + String key = ""; + if (CollectionUtils.isNotEmpty(partVals)) { + key = String.join(delimit, partVals); + } + return key; + } + + public static String buildKey(List partVals, String colName) { + String key = buildKey(partVals); + return key + delimit + colName; + } + public static String buildKeyWithDelimit(String dbName, String tableName, List partVals) { return buildKey(dbName, tableName, partVals) + delimit; } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 0d132f2074..b309edb14c 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -32,8 +32,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -91,7 +90,6 @@ import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TableMeta; import org.apache.hadoop.hive.metastore.api.Type; @@ -119,111 +117,28 @@ // TODO constraintCache // TODO need sd nested copy? // TODO String intern -// TODO restructure HBaseStore // TODO monitor event queue // TODO initial load slow? // TODO size estimation -// TODO factor in extrapolation logic (using partitions found) during aggregate stats calculation public class CachedStore implements RawStore, Configurable { private static ScheduledExecutorService cacheUpdateMaster = null; - private static ReentrantReadWriteLock databaseCacheLock = new ReentrantReadWriteLock(true); - private static AtomicBoolean isDatabaseCacheDirty = new AtomicBoolean(false); - private static ReentrantReadWriteLock tableCacheLock = new ReentrantReadWriteLock(true); - private static AtomicBoolean isTableCacheDirty = new AtomicBoolean(false); - private static ReentrantReadWriteLock partitionCacheLock = new ReentrantReadWriteLock(true); - private static AtomicBoolean isPartitionCacheDirty = new AtomicBoolean(false); - private static ReentrantReadWriteLock tableColStatsCacheLock = new ReentrantReadWriteLock(true); - private static AtomicBoolean isTableColStatsCacheDirty = new AtomicBoolean(false); - private static ReentrantReadWriteLock partitionColStatsCacheLock = new ReentrantReadWriteLock( - true); - private static ReentrantReadWriteLock partitionAggrColStatsCacheLock = - new ReentrantReadWriteLock(true); - private static AtomicBoolean isPartitionAggrColStatsCacheDirty = new AtomicBoolean(false); - private static AtomicBoolean isPartitionColStatsCacheDirty = new AtomicBoolean(false); private static List whitelistPatterns = null; private static List blacklistPatterns = null; + // Default value set to 100 milliseconds for test purpose + private static long cacheRefreshPeriod = 100; + private static boolean isCachePrewarmed = false; private RawStore rawStore = null; private Configuration conf; private PartitionExpressionProxy expressionProxy = null; - // Default value set to 100 milliseconds for test purpose - private static long cacheRefreshPeriod = 100; - - /** A wrapper over SharedCache. Allows one to get SharedCache safely; should be merged - * into SharedCache itself (see the TODO on the class). */ - private static final SharedCacheWrapper sharedCacheWrapper = new SharedCacheWrapper(); + private static final SharedCache sharedCache = new SharedCache(); static final private Logger LOG = LoggerFactory.getLogger(CachedStore.class.getName()); - static class TableWrapper { - Table t; - String location; - Map parameters; - byte[] sdHash; - TableWrapper(Table t, byte[] sdHash, String location, Map parameters) { - this.t = t; - this.sdHash = sdHash; - this.location = location; - this.parameters = parameters; - } - public Table getTable() { - return t; - } - public byte[] getSdHash() { - return sdHash; - } - public String getLocation() { - return location; - } - public Map getParameters() { - return parameters; - } - } - - static class PartitionWrapper { - Partition p; - String location; - Map parameters; - byte[] sdHash; - PartitionWrapper(Partition p, byte[] sdHash, String location, Map parameters) { - this.p = p; - this.sdHash = sdHash; - this.location = location; - this.parameters = parameters; - } - public Partition getPartition() { - return p; - } - public byte[] getSdHash() { - return sdHash; - } - public String getLocation() { - return location; - } - public Map getParameters() { - return parameters; - } - } - - static class StorageDescriptorWrapper { - StorageDescriptor sd; - int refCount = 0; - StorageDescriptorWrapper(StorageDescriptor sd, int refCount) { - this.sd = sd; - this.refCount = refCount; - } - public StorageDescriptor getSd() { - return sd; - } - public int getRefCount() { - return refCount; - } - } - public CachedStore() { } - public static void initSharedCacheAsync(Configuration conf) { + public static void initSharedCache(Configuration conf) { String clazzName = null; boolean isEnabled = false; try { @@ -236,7 +151,6 @@ public static void initSharedCacheAsync(Configuration conf) { LOG.debug("CachedStore is not enabled; using " + clazzName); return; } - sharedCacheWrapper.startInit(conf); } @Override @@ -260,6 +174,29 @@ public void setConf(Configuration conf) { expressionProxy = PartFilterExprUtil.createExpressionProxy(conf); } initBlackListWhiteList(conf); + startCacheUpdateService(conf, false, true); + } + + public void setConfForTest(Configuration conf) { + String rawStoreClassName = + MetastoreConf.getVar(conf, ConfVars.CACHED_RAW_STORE_IMPL, ObjectStore.class.getName()); + if (rawStore == null) { + try { + rawStore = (JavaUtils.getClass(rawStoreClassName, RawStore.class)).newInstance(); + } catch (Exception e) { + throw new RuntimeException("Cannot instantiate " + rawStoreClassName, e); + } + } + rawStore.setConf(conf); + Configuration oldConf = this.conf; + this.conf = conf; + if (expressionProxy != null && conf != oldConf) { + LOG.warn("Unexpected setConf when we were already configured"); + } + if (expressionProxy == null || conf != oldConf) { + expressionProxy = PartFilterExprUtil.createExpressionProxy(conf); + } + initBlackListWhiteList(conf); } @VisibleForTesting @@ -268,20 +205,11 @@ static void prewarm(RawStore rawStore) throws Exception { Deadline.registerIfNot(1000000); List dbNames = rawStore.getAllDatabases(); LOG.info("Number of databases to prewarm: " + dbNames.size()); - SharedCache sharedCache = sharedCacheWrapper.getUnsafe(); for (int i = 0; i < dbNames.size(); i++) { String dbName = StringUtils.normalizeIdentifier(dbNames.get(i)); - // Cache partition column stats - Deadline.startTimer("getColStatsForDatabase"); - List colStatsForDB = - rawStore.getPartitionColStatsForDatabase(dbName); - Deadline.stopTimer(); - if (colStatsForDB != null) { - sharedCache.addPartitionColStatsToCache(colStatsForDB); - } - LOG.info("Caching database: {}. Cached {} / {} databases so far.", dbName, i, dbNames.size()); + LOG.info("Processed database: {}. Cached {} / {} databases so far.", dbName, i, dbNames.size()); Database db = rawStore.getDatabase(dbName); - sharedCache.addDatabaseToCache(dbName, db); + sharedCache.addDatabaseToCache(db); List tblNames = rawStore.getAllTables(dbName); LOG.debug("Tables in database: {} : {}", dbName, tblNames); for (int j = 0; j < tblNames.size(); j++) { @@ -290,59 +218,60 @@ static void prewarm(RawStore rawStore) throws Exception { LOG.info("Not caching database: {}'s table: {}", dbName, tblName); continue; } - LOG.info("Caching database: {}'s table: {}. Cached {} / {} tables so far.", dbName, + LOG.info("Processed database: {}'s table: {}. Cached {} / {} tables so far.", dbName, tblName, j, tblNames.size()); Table table = null; table = rawStore.getTable(dbName, tblName); + ColumnStatistics tableColStats = null; + List partitions = null; + List partitionColStats = null; + AggrStats aggrStatsAllPartitions = null; + AggrStats aggrStatsAllButDefaultPartition = null; // It is possible the table is deleted during fetching tables of the database, // in that case, continue with the next table if (table == null) { continue; } - sharedCache.addTableToCache(dbName, tblName, table); - if (table.getPartitionKeys() != null && table.getPartitionKeys().size() > 0) { - Deadline.startTimer("getPartitions"); - List partitions = rawStore.getPartitions(dbName, tblName, Integer.MAX_VALUE); - Deadline.stopTimer(); - for (Partition partition : partitions) { - sharedCache.addPartitionToCache(dbName, tblName, partition); - } - } - // Cache table column stats + // Get table column stats List colNames = MetaStoreUtils.getColumnNamesForTable(table); Deadline.startTimer("getTableColumnStatistics"); - ColumnStatistics tableColStats = - rawStore.getTableColumnStatistics(dbName, tblName, colNames); + tableColStats = rawStore.getTableColumnStatistics(dbName, tblName, colNames); Deadline.stopTimer(); - if ((tableColStats != null) && (tableColStats.getStatsObjSize() > 0)) { - sharedCache.addTableColStatsToCache(dbName, tblName, tableColStats.getStatsObj()); - } - // Cache aggregate stats for all partitions of a table and for all but default partition - List partNames = rawStore.listPartitionNames(dbName, tblName, (short) -1); - if ((partNames != null) && (partNames.size() > 0)) { - AggrStats aggrStatsAllPartitions = - rawStore.get_aggr_stats_for(dbName, tblName, partNames, colNames); - // Remove default partition from partition names and get aggregate - // stats again - List partKeys = table.getPartitionKeys(); - String defaultPartitionValue = MetastoreConf.getVar(rawStore.getConf(), ConfVars.DEFAULTPARTITIONNAME); - List partCols = new ArrayList(); - List partVals = new ArrayList(); - for (FieldSchema fs : partKeys) { - partCols.add(fs.getName()); - partVals.add(defaultPartitionValue); + if (table.isSetPartitionKeys()) { + List partNames = rawStore.listPartitionNames(dbName, tblName, (short) -1); + Deadline.startTimer("getPartitions"); + partitions = rawStore.getPartitions(dbName, tblName, Integer.MAX_VALUE); + Deadline.stopTimer(); + // Get partition column stats for this table + Deadline.startTimer("getPartitionColumnStatistics"); + partitionColStats = + rawStore.getPartitionColumnStatistics(dbName, tblName, partNames, colNames); + Deadline.stopTimer(); + if ((partNames != null) && (partNames.size() > 0)) { + // Get aggregate stats for all partitions of a table and for all but default partition + aggrStatsAllPartitions = + rawStore.get_aggr_stats_for(dbName, tblName, partNames, colNames); + // Remove default partition from partition names and get aggregate + // stats again + List partKeys = table.getPartitionKeys(); + String defaultPartitionValue = + MetastoreConf.getVar(rawStore.getConf(), ConfVars.DEFAULTPARTITIONNAME); + List partCols = new ArrayList(); + List partVals = new ArrayList(); + for (FieldSchema fs : partKeys) { + partCols.add(fs.getName()); + partVals.add(defaultPartitionValue); + } + String defaultPartitionName = FileUtils.makePartName(partCols, partVals); + partNames.remove(defaultPartitionName); + aggrStatsAllButDefaultPartition = + rawStore.get_aggr_stats_for(dbName, tblName, partNames, colNames); } - String defaultPartitionName = FileUtils.makePartName(partCols, partVals); - partNames.remove(defaultPartitionName); - AggrStats aggrStatsAllButDefaultPartition = - rawStore.get_aggr_stats_for(dbName, tblName, partNames, colNames); - sharedCache.addAggregateStatsToCache(dbName, tblName, aggrStatsAllPartitions, - aggrStatsAllButDefaultPartition); } + sharedCache.populateTableInCache(table, tableColStats, partitions, partitionColStats, + aggrStatsAllPartitions, aggrStatsAllButDefaultPartition); } } - // Notify all blocked threads that prewarm is complete now - sharedCacheWrapper.notifyAllBlocked(); } private static void initBlackListWhiteList(Configuration conf) { @@ -357,7 +286,8 @@ private static void initBlackListWhiteList(Configuration conf) { } @VisibleForTesting - synchronized static void startCacheUpdateService(Configuration conf) { + synchronized static void startCacheUpdateService(Configuration conf, boolean runOnlyOnce, + boolean shouldRunPrewarm) { if (cacheUpdateMaster == null) { initBlackListWhiteList(conf); if (!MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST)) { @@ -374,7 +304,14 @@ public Thread newThread(Runnable r) { return t; } }); - cacheUpdateMaster.scheduleAtFixedRate(new CacheUpdateMasterWork(conf), 0, cacheRefreshPeriod, + if (!runOnlyOnce) { + cacheUpdateMaster.scheduleAtFixedRate(new CacheUpdateMasterWork(conf, shouldRunPrewarm), 0, + cacheRefreshPeriod, TimeUnit.MILLISECONDS); + } + } + if (runOnlyOnce) { + // Some tests control the execution of the background update thread + cacheUpdateMaster.schedule(new CacheUpdateMasterWork(conf, shouldRunPrewarm), 0, TimeUnit.MILLISECONDS); } } @@ -403,10 +340,11 @@ static void setCacheRefreshPeriod(long time) { } static class CacheUpdateMasterWork implements Runnable { - private boolean isFirstRun = true; + private boolean shouldRunPrewarm = true; private final RawStore rawStore; - public CacheUpdateMasterWork(Configuration conf) { + public CacheUpdateMasterWork(Configuration conf, boolean shouldRunPrewarm) { + this.shouldRunPrewarm = shouldRunPrewarm; String rawStoreClassName = MetastoreConf.getVar(conf, ConfVars.CACHED_RAW_STORE_IMPL, ObjectStore.class.getName()); try { @@ -415,15 +353,14 @@ public CacheUpdateMasterWork(Configuration conf) { } catch (InstantiationException | IllegalAccessException | MetaException e) { // MetaException here really means ClassNotFound (see the utility method). // So, if any of these happen, that means we can never succeed. - sharedCacheWrapper.updateInitState(e, true); throw new RuntimeException("Cannot instantiate " + rawStoreClassName, e); } } @Override public void run() { - if (isFirstRun) { - while (isFirstRun) { + if (shouldRunPrewarm) { + while (!isCachePrewarmed) { try { long startTime = System.nanoTime(); LOG.info("Prewarming CachedStore"); @@ -433,11 +370,9 @@ public void run() { LOG.info("Time taken in prewarming = " + (endTime - startTime) / 1000000 + "ms"); } catch (Exception e) { LOG.error("Prewarm failure", e); - sharedCacheWrapper.updateInitState(e, false); return; } - sharedCacheWrapper.updateInitState(null, false); - isFirstRun = false; + isCachePrewarmed = true; } } else { // TODO: prewarm and update can probably be merged. @@ -454,111 +389,31 @@ public void update() { // Update the database in cache updateDatabases(rawStore, dbNames); for (String dbName : dbNames) { - updateDatabasePartitionColStats(rawStore, dbName); // Update the tables in cache updateTables(rawStore, dbName); - List tblNames = getAllTablesInternal(dbName, sharedCacheWrapper.getUnsafe()); + List tblNames = rawStore.getAllTables(dbName); for (String tblName : tblNames) { if (!shouldCacheTable(dbName, tblName)) { continue; } - // Update the partitions for a table in cache - updateTablePartitions(rawStore, dbName, tblName); // Update the table column stats for a table in cache updateTableColStats(rawStore, dbName, tblName); - // Update aggregate column stats cache - updateAggregateStatsCache(rawStore, dbName, tblName); + // Update the partitions for a table in cache + updateTablePartitions(rawStore, dbName, tblName); + // Update the partition col stats for a table in cache + updateTablePartitionColStats(rawStore, dbName, tblName); + // Update aggregate partition column stats for a table in cache + updateTableAggregatePartitionColStats(rawStore, dbName, tblName); } } + sharedCache.incrementUpdateCount(); } } catch (Exception e) { LOG.error("Updating CachedStore: error happen when refresh; ignoring", e); } } - private void updateDatabasePartitionColStats(RawStore rawStore, String dbName) { - try { - Deadline.startTimer("getColStatsForDatabasePartitions"); - List colStatsForDB = - rawStore.getPartitionColStatsForDatabase(dbName); - Deadline.stopTimer(); - if (colStatsForDB != null) { - if (partitionColStatsCacheLock.writeLock().tryLock()) { - // Skip background updates if we detect change - if (isPartitionColStatsCacheDirty.compareAndSet(true, false)) { - LOG.debug("Skipping partition column stats cache update; the partition column stats " - + "list we have is dirty."); - return; - } - sharedCacheWrapper.getUnsafe() - .refreshPartitionColStats(StringUtils.normalizeIdentifier(dbName), colStatsForDB); - } - } - } catch (MetaException | NoSuchObjectException e) { - LOG.info("Updating CachedStore: unable to read partitions column stats of database: {}", - dbName, e); - } finally { - if (partitionColStatsCacheLock.isWriteLockedByCurrentThread()) { - partitionColStatsCacheLock.writeLock().unlock(); - } - } - } - - // Update cached aggregate stats for all partitions of a table and for all - // but default partition - private void updateAggregateStatsCache(RawStore rawStore, String dbName, String tblName) { - try { - Table table = rawStore.getTable(dbName, tblName); - List partNames = rawStore.listPartitionNames(dbName, tblName, (short) -1); - List colNames = MetaStoreUtils.getColumnNamesForTable(table); - if ((partNames != null) && (partNames.size() > 0)) { - Deadline.startTimer("getAggregareStatsForAllPartitions"); - AggrStats aggrStatsAllPartitions = - rawStore.get_aggr_stats_for(dbName, tblName, partNames, colNames); - Deadline.stopTimer(); - // Remove default partition from partition names and get aggregate stats again - List partKeys = table.getPartitionKeys(); - String defaultPartitionValue = - MetastoreConf.getVar(rawStore.getConf(), ConfVars.DEFAULTPARTITIONNAME); - List partCols = new ArrayList(); - List partVals = new ArrayList(); - for (FieldSchema fs : partKeys) { - partCols.add(fs.getName()); - partVals.add(defaultPartitionValue); - } - String defaultPartitionName = FileUtils.makePartName(partCols, partVals); - partNames.remove(defaultPartitionName); - Deadline.startTimer("getAggregareStatsForAllPartitionsExceptDefault"); - AggrStats aggrStatsAllButDefaultPartition = - rawStore.get_aggr_stats_for(dbName, tblName, partNames, colNames); - Deadline.stopTimer(); - if ((aggrStatsAllPartitions != null) && (aggrStatsAllButDefaultPartition != null)) { - if (partitionAggrColStatsCacheLock.writeLock().tryLock()) { - // Skip background updates if we detect change - if (isPartitionAggrColStatsCacheDirty.compareAndSet(true, false)) { - LOG.debug( - "Skipping aggregate column stats cache update; the aggregate column stats we " - + "have is dirty."); - return; - } - sharedCacheWrapper.getUnsafe().refreshAggregateStatsCache( - StringUtils.normalizeIdentifier(dbName), StringUtils.normalizeIdentifier(tblName), - aggrStatsAllPartitions, aggrStatsAllButDefaultPartition); - } - } - } - } catch (MetaException | NoSuchObjectException e) { - LOG.info("Updating CachedStore: unable to read aggregate column stats of table: " + tblName, - e); - } finally { - if (partitionAggrColStatsCacheLock.isWriteLockedByCurrentThread()) { - partitionAggrColStatsCacheLock.writeLock().unlock(); - } - } - } - private void updateDatabases(RawStore rawStore, List dbNames) { - // Prepare the list of databases List databases = new ArrayList<>(); for (String dbName : dbNames) { Database db; @@ -569,24 +424,9 @@ private void updateDatabases(RawStore rawStore, List dbNames) { LOG.info("Updating CachedStore: database - " + dbName + " does not exist.", e); } } - // Update the cached database objects - try { - if (databaseCacheLock.writeLock().tryLock()) { - // Skip background updates if we detect change - if (isDatabaseCacheDirty.compareAndSet(true, false)) { - LOG.debug("Skipping database cache update; the database list we have is dirty."); - return; - } - sharedCacheWrapper.getUnsafe().refreshDatabases(databases); - } - } finally { - if (databaseCacheLock.isWriteLockedByCurrentThread()) { - databaseCacheLock.writeLock().unlock(); - } - } + sharedCache.refreshDatabasesInCache(databases); } - // Update the cached table objects private void updateTables(RawStore rawStore, String dbName) { List tables = new ArrayList<>(); try { @@ -595,81 +435,99 @@ private void updateTables(RawStore rawStore, String dbName) { if (!shouldCacheTable(dbName, tblName)) { continue; } - Table table = - rawStore.getTable(StringUtils.normalizeIdentifier(dbName), - StringUtils.normalizeIdentifier(tblName)); + Table table = rawStore.getTable(StringUtils.normalizeIdentifier(dbName), + StringUtils.normalizeIdentifier(tblName)); tables.add(table); } - if (tableCacheLock.writeLock().tryLock()) { - // Skip background updates if we detect change - if (isTableCacheDirty.compareAndSet(true, false)) { - LOG.debug("Skipping table cache update; the table list we have is dirty."); - return; - } - sharedCacheWrapper.getUnsafe().refreshTables(dbName, tables); - } + sharedCache.refreshTablesInCache(dbName, tables); } catch (MetaException e) { - LOG.info("Updating CachedStore: unable to read tables for database - " + dbName, e); - } finally { - if (tableCacheLock.isWriteLockedByCurrentThread()) { - tableCacheLock.writeLock().unlock(); + LOG.debug("Unable to refresh cached tables for database: " + dbName, e); + } + } + + private void updateTableColStats(RawStore rawStore, String dbName, String tblName) { + try { + Table table = rawStore.getTable(dbName, tblName); + if (!table.isSetPartitionKeys()) { + List colNames = MetaStoreUtils.getColumnNamesForTable(table); + Deadline.startTimer("getTableColumnStatistics"); + ColumnStatistics tableColStats = + rawStore.getTableColumnStatistics(dbName, tblName, colNames); + Deadline.stopTimer(); + if (tableColStats != null) { + sharedCache.refreshTableColStatsInCache(StringUtils.normalizeIdentifier(dbName), + StringUtils.normalizeIdentifier(tblName), tableColStats.getStatsObj()); + } } + } catch (MetaException | NoSuchObjectException e) { + LOG.info("Unable to refresh table column stats for table: " + tblName, e); } } - // Update the cached partition objects for a table private void updateTablePartitions(RawStore rawStore, String dbName, String tblName) { try { Deadline.startTimer("getPartitions"); List partitions = rawStore.getPartitions(dbName, tblName, Integer.MAX_VALUE); Deadline.stopTimer(); - if (partitionCacheLock.writeLock().tryLock()) { - // Skip background updates if we detect change - if (isPartitionCacheDirty.compareAndSet(true, false)) { - LOG.debug("Skipping partition cache update; the partition list we have is dirty."); - return; - } - sharedCacheWrapper.getUnsafe().refreshPartitions( - StringUtils.normalizeIdentifier(dbName), - StringUtils.normalizeIdentifier(tblName), partitions); - } + sharedCache.refreshPartitionsInCache(StringUtils.normalizeIdentifier(dbName), + StringUtils.normalizeIdentifier(tblName), partitions); } catch (MetaException | NoSuchObjectException e) { LOG.info("Updating CachedStore: unable to read partitions of table: " + tblName, e); - } finally { - if (partitionCacheLock.isWriteLockedByCurrentThread()) { - partitionCacheLock.writeLock().unlock(); - } } } - // Update the cached col stats for this table - private void updateTableColStats(RawStore rawStore, String dbName, String tblName) { + private void updateTablePartitionColStats(RawStore rawStore, String dbName, String tblName) { try { Table table = rawStore.getTable(dbName, tblName); List colNames = MetaStoreUtils.getColumnNamesForTable(table); - Deadline.startTimer("getTableColumnStatistics"); - ColumnStatistics tableColStats = - rawStore.getTableColumnStatistics(dbName, tblName, colNames); + List partNames = rawStore.listPartitionNames(dbName, tblName, (short) -1); + // Get partition column stats for this table + Deadline.startTimer("getPartitionColumnStatistics"); + List partitionColStats = + rawStore.getPartitionColumnStatistics(dbName, tblName, partNames, colNames); Deadline.stopTimer(); - if (tableColStats != null) { - if (tableColStatsCacheLock.writeLock().tryLock()) { - // Skip background updates if we detect change - if (isTableColStatsCacheDirty.compareAndSet(true, false)) { - LOG.debug("Skipping table column stats cache update; the table column stats list we " - + "have is dirty."); - return; - } - sharedCacheWrapper.getUnsafe().refreshTableColStats( - StringUtils.normalizeIdentifier(dbName), - StringUtils.normalizeIdentifier(tblName), tableColStats.getStatsObj()); + sharedCache.refreshPartitionColStatsInCache(dbName, tblName, partitionColStats); + } catch (MetaException | NoSuchObjectException e) { + LOG.info("Updating CachedStore: unable to read partitions of table: " + tblName, e); + } + } + + // Update cached aggregate stats for all partitions of a table and for all + // but default partition + private void updateTableAggregatePartitionColStats(RawStore rawStore, String dbName, + String tblName) { + try { + Table table = rawStore.getTable(dbName, tblName); + List partNames = rawStore.listPartitionNames(dbName, tblName, (short) -1); + List colNames = MetaStoreUtils.getColumnNamesForTable(table); + if ((partNames != null) && (partNames.size() > 0)) { + Deadline.startTimer("getAggregareStatsForAllPartitions"); + AggrStats aggrStatsAllPartitions = + rawStore.get_aggr_stats_for(dbName, tblName, partNames, colNames); + Deadline.stopTimer(); + // Remove default partition from partition names and get aggregate stats again + List partKeys = table.getPartitionKeys(); + String defaultPartitionValue = + MetastoreConf.getVar(rawStore.getConf(), ConfVars.DEFAULTPARTITIONNAME); + List partCols = new ArrayList(); + List partVals = new ArrayList(); + for (FieldSchema fs : partKeys) { + partCols.add(fs.getName()); + partVals.add(defaultPartitionValue); } + String defaultPartitionName = FileUtils.makePartName(partCols, partVals); + partNames.remove(defaultPartitionName); + Deadline.startTimer("getAggregareStatsForAllPartitionsExceptDefault"); + AggrStats aggrStatsAllButDefaultPartition = + rawStore.get_aggr_stats_for(dbName, tblName, partNames, colNames); + Deadline.stopTimer(); + sharedCache.refreshAggregateStatsInCache(StringUtils.normalizeIdentifier(dbName), + StringUtils.normalizeIdentifier(tblName), aggrStatsAllPartitions, + aggrStatsAllButDefaultPartition); } } catch (MetaException | NoSuchObjectException e) { - LOG.info("Updating CachedStore: unable to read table column stats of table: " + tblName, e); - } finally { - if (tableColStatsCacheLock.isWriteLockedByCurrentThread()) { - tableColStatsCacheLock.writeLock().unlock(); - } + LOG.info("Updating CachedStore: unable to read aggregate column stats of table: " + tblName, + e); } } } @@ -707,33 +565,17 @@ public void rollbackTransaction() { @Override public void createDatabase(Database db) throws InvalidObjectException, MetaException { rawStore.createDatabase(db); - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return; - try { - // Wait if background cache update is happening - databaseCacheLock.readLock().lock(); - isDatabaseCacheDirty.set(true); - sharedCache.addDatabaseToCache(StringUtils.normalizeIdentifier(db.getName()), - db.deepCopy()); - } finally { - databaseCacheLock.readLock().unlock(); - } + sharedCache.addDatabaseToCache(db); } @Override public Database getDatabase(String dbName) throws NoSuchObjectException { - SharedCache sharedCache; - - if (!sharedCacheWrapper.isInitialized()) { + if (!isCachePrewarmed) { return rawStore.getDatabase(dbName); } - - try { - sharedCache = sharedCacheWrapper.get(); - } catch (MetaException e) { - throw new RuntimeException(e); // TODO: why doesn't getDatabase throw MetaEx? - } - Database db = sharedCache.getDatabaseFromCache(StringUtils.normalizeIdentifier(dbName)); + dbName = dbName.toLowerCase(); + Database db = + sharedCache.getDatabaseFromCache(StringUtils.normalizeIdentifier(dbName)); if (db == null) { throw new NoSuchObjectException(); } @@ -741,64 +583,39 @@ public Database getDatabase(String dbName) throws NoSuchObjectException { } @Override - public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaException { - boolean succ = rawStore.dropDatabase(dbname); + public boolean dropDatabase(String dbName) throws NoSuchObjectException, MetaException { + boolean succ = rawStore.dropDatabase(dbName); if (succ) { - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return succ; - try { - // Wait if background cache update is happening - databaseCacheLock.readLock().lock(); - isDatabaseCacheDirty.set(true); - sharedCache.removeDatabaseFromCache(StringUtils.normalizeIdentifier(dbname)); - } finally { - databaseCacheLock.readLock().unlock(); - } + dbName = dbName.toLowerCase(); + sharedCache.removeDatabaseFromCache(StringUtils.normalizeIdentifier(dbName)); } return succ; } @Override - public boolean alterDatabase(String dbName, Database db) throws NoSuchObjectException, - MetaException { + public boolean alterDatabase(String dbName, Database db) + throws NoSuchObjectException, MetaException { boolean succ = rawStore.alterDatabase(dbName, db); if (succ) { - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return succ; - try { - // Wait if background cache update is happening - databaseCacheLock.readLock().lock(); - isDatabaseCacheDirty.set(true); - sharedCache.alterDatabaseInCache(StringUtils.normalizeIdentifier(dbName), db); - } finally { - databaseCacheLock.readLock().unlock(); - } + dbName = dbName.toLowerCase(); + sharedCache.alterDatabaseInCache(StringUtils.normalizeIdentifier(dbName), db); } return succ; } @Override public List getDatabases(String pattern) throws MetaException { - if (!sharedCacheWrapper.isInitialized()) { + if (!isCachePrewarmed) { return rawStore.getDatabases(pattern); } - SharedCache sharedCache = sharedCacheWrapper.get(); - List results = new ArrayList<>(); - for (String dbName : sharedCache.listCachedDatabases()) { - dbName = StringUtils.normalizeIdentifier(dbName); - if (CacheUtils.matches(dbName, pattern)) { - results.add(dbName); - } - } - return results; + return sharedCache.listCachedDatabases(pattern); } @Override public List getAllDatabases() throws MetaException { - if (!sharedCacheWrapper.isInitialized()) { + if (!isCachePrewarmed) { return rawStore.getAllDatabases(); } - SharedCache sharedCache = sharedCacheWrapper.get(); return sharedCache.listCachedDatabases(); } @@ -843,22 +660,13 @@ public void createTable(Table tbl) throws InvalidObjectException, MetaException if (!shouldCacheTable(dbName, tblName)) { return; } - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return; validateTableType(tbl); - try { - // Wait if background cache update is happening - tableCacheLock.readLock().lock(); - isTableCacheDirty.set(true); - sharedCache.addTableToCache(dbName, tblName, tbl); - } finally { - tableCacheLock.readLock().unlock(); - } + sharedCache.addTableToCache(dbName, tblName, tbl); } @Override - public boolean dropTable(String dbName, String tblName) throws MetaException, - NoSuchObjectException, InvalidObjectException, InvalidInputException { + public boolean dropTable(String dbName, String tblName) + throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { boolean succ = rawStore.dropTable(dbName, tblName); if (succ) { dbName = StringUtils.normalizeIdentifier(dbName); @@ -866,26 +674,7 @@ public boolean dropTable(String dbName, String tblName) throws MetaException, if (!shouldCacheTable(dbName, tblName)) { return succ; } - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return succ; - // Remove table - try { - // Wait if background table cache update is happening - tableCacheLock.readLock().lock(); - isTableCacheDirty.set(true); - sharedCache.removeTableFromCache(dbName, tblName); - } finally { - tableCacheLock.readLock().unlock(); - } - // Remove table col stats - try { - // Wait if background table col stats cache update is happening - tableColStatsCacheLock.readLock().lock(); - isTableColStatsCacheDirty.set(true); - sharedCache.removeTableColStatsFromCache(dbName, tblName); - } finally { - tableColStatsCacheLock.readLock().unlock(); - } + sharedCache.removeTableFromCache(dbName, tblName); } return succ; } @@ -894,11 +683,14 @@ public boolean dropTable(String dbName, String tblName) throws MetaException, public Table getTable(String dbName, String tblName) throws MetaException { dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!sharedCacheWrapper.isInitialized() || !shouldCacheTable(dbName, tblName)) { + if (!shouldCacheTable(dbName, tblName)) { return rawStore.getTable(dbName, tblName); } - SharedCache sharedCache = sharedCacheWrapper.get(); Table tbl = sharedCache.getTableFromCache(dbName, tblName); + if (tbl == null) { + // This table is not yet loaded in cache + return rawStore.getTable(dbName, tblName); + } if (tbl != null) { tbl.unsetPrivileges(); tbl.setRewriteEnabled(tbl.isRewriteEnabled()); @@ -915,25 +707,7 @@ public boolean addPartition(Partition part) throws InvalidObjectException, MetaE if (!shouldCacheTable(dbName, tblName)) { return succ; } - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return succ; - try { - // Wait if background cache update is happening - partitionCacheLock.readLock().lock(); - isPartitionCacheDirty.set(true); - sharedCache.addPartitionToCache(dbName, tblName, part); - } finally { - partitionCacheLock.readLock().unlock(); - } - // Remove aggregate partition col stats for this table - try { - // Wait if background cache update is happening - partitionAggrColStatsCacheLock.readLock().lock(); - isPartitionAggrColStatsCacheDirty.set(true); - sharedCache.removeAggrPartitionColStatsFromCache(dbName, tblName); - } finally { - partitionAggrColStatsCacheLock.readLock().unlock(); - } + sharedCache.addPartitionToCache(dbName, tblName, part); } return succ; } @@ -948,27 +722,7 @@ public boolean addPartitions(String dbName, String tblName, List part if (!shouldCacheTable(dbName, tblName)) { return succ; } - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return succ; - try { - // Wait if background cache update is happening - partitionCacheLock.readLock().lock(); - isPartitionCacheDirty.set(true); - for (Partition part : parts) { - sharedCache.addPartitionToCache(dbName, tblName, part); - } - } finally { - partitionCacheLock.readLock().unlock(); - } - // Remove aggregate partition col stats for this table - try { - // Wait if background cache update is happening - partitionAggrColStatsCacheLock.readLock().lock(); - isPartitionAggrColStatsCacheDirty.set(true); - sharedCache.removeAggrPartitionColStatsFromCache(dbName, tblName); - } finally { - partitionAggrColStatsCacheLock.readLock().unlock(); - } + sharedCache.addPartitionsToCache(dbName, tblName, parts); } return succ; } @@ -983,28 +737,10 @@ public boolean addPartitions(String dbName, String tblName, PartitionSpecProxy p if (!shouldCacheTable(dbName, tblName)) { return succ; } - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return succ; - try { - // Wait if background cache update is happening - partitionCacheLock.readLock().lock(); - isPartitionCacheDirty.set(true); - PartitionSpecProxy.PartitionIterator iterator = partitionSpec.getPartitionIterator(); - while (iterator.hasNext()) { - Partition part = iterator.next(); - sharedCache.addPartitionToCache(dbName, tblName, part); - } - } finally { - partitionCacheLock.readLock().unlock(); - } - // Remove aggregate partition col stats for this table - try { - // Wait if background cache update is happening - partitionAggrColStatsCacheLock.readLock().lock(); - isPartitionAggrColStatsCacheDirty.set(true); - sharedCache.removeAggrPartitionColStatsFromCache(dbName, tblName); - } finally { - partitionAggrColStatsCacheLock.readLock().unlock(); + PartitionSpecProxy.PartitionIterator iterator = partitionSpec.getPartitionIterator(); + while (iterator.hasNext()) { + Partition part = iterator.next(); + sharedCache.addPartitionToCache(dbName, tblName, part); } } return succ; @@ -1015,16 +751,13 @@ public Partition getPartition(String dbName, String tblName, List part_v throws MetaException, NoSuchObjectException { dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - - if (!sharedCacheWrapper.isInitialized() || !shouldCacheTable(dbName, tblName)) { + if (!shouldCacheTable(dbName, tblName)) { return rawStore.getPartition(dbName, tblName, part_vals); } - SharedCache sharedCache = sharedCacheWrapper.get(); - Partition part = - sharedCache.getPartitionFromCache(dbName, tblName, part_vals); + Partition part = sharedCache.getPartitionFromCache(dbName, tblName, part_vals); if (part == null) { - // TODO Manage privileges - throw new NoSuchObjectException("partition values=" + part_vals.toString()); + // The table containing the partition is not yet loaded in cache + return rawStore.getPartition(dbName, tblName, part_vals); } return part; } @@ -1034,10 +767,14 @@ public boolean doesPartitionExist(String dbName, String tblName, List part_vals) throws MetaException, NoSuchObjectException { dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!sharedCacheWrapper.isInitialized() || !shouldCacheTable(dbName, tblName)) { + if (!shouldCacheTable(dbName, tblName)) { + return rawStore.doesPartitionExist(dbName, tblName, part_vals); + } + Table tbl = sharedCache.getTableFromCache(dbName, tblName); + if (tbl == null) { + // The table containing the partition is not yet loaded in cache return rawStore.doesPartitionExist(dbName, tblName, part_vals); } - SharedCache sharedCache = sharedCacheWrapper.get(); return sharedCache.existPartitionFromCache(dbName, tblName, part_vals); } @@ -1051,48 +788,40 @@ public boolean dropPartition(String dbName, String tblName, List part_va if (!shouldCacheTable(dbName, tblName)) { return succ; } - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return succ; - // Remove partition - try { - // Wait if background cache update is happening - partitionCacheLock.readLock().lock(); - isPartitionCacheDirty.set(true); - sharedCache.removePartitionFromCache(dbName, tblName, part_vals); - } finally { - partitionCacheLock.readLock().unlock(); - } - // Remove partition col stats - try { - // Wait if background cache update is happening - partitionColStatsCacheLock.readLock().lock(); - isPartitionColStatsCacheDirty.set(true); - sharedCache.removePartitionColStatsFromCache(dbName, tblName, part_vals); - } finally { - partitionColStatsCacheLock.readLock().unlock(); - } - // Remove aggregate partition col stats for this table - try { - // Wait if background cache update is happening - partitionAggrColStatsCacheLock.readLock().lock(); - isPartitionAggrColStatsCacheDirty.set(true); - sharedCache.removeAggrPartitionColStatsFromCache(dbName, tblName); - } finally { - partitionAggrColStatsCacheLock.readLock().unlock(); - } + sharedCache.removePartitionFromCache(dbName, tblName, part_vals); } return succ; } @Override + public void dropPartitions(String dbName, String tblName, List partNames) + throws MetaException, NoSuchObjectException { + rawStore.dropPartitions(dbName, tblName, partNames); + dbName = StringUtils.normalizeIdentifier(dbName); + tblName = StringUtils.normalizeIdentifier(tblName); + if (!shouldCacheTable(dbName, tblName)) { + return; + } + List> partVals = new ArrayList>(); + for (String partName : partNames) { + partVals.add(partNameToVals(partName)); + } + sharedCache.removePartitionsFromCache(dbName, tblName, partVals); + } + + @Override public List getPartitions(String dbName, String tblName, int max) throws MetaException, NoSuchObjectException { dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!sharedCacheWrapper.isInitialized() || !shouldCacheTable(dbName, tblName)) { + if (!shouldCacheTable(dbName, tblName)) { + return rawStore.getPartitions(dbName, tblName, max); + } + Table tbl = sharedCache.getTableFromCache(dbName, tblName); + if (tbl == null) { + // The table containing the partitions is not yet loaded in cache return rawStore.getPartitions(dbName, tblName, max); } - SharedCache sharedCache = sharedCacheWrapper.get(); List parts = sharedCache.listCachedPartitions(dbName, tblName, max); return parts; } @@ -1107,71 +836,20 @@ public void alterTable(String dbName, String tblName, Table newTable) if (!shouldCacheTable(dbName, tblName) && !shouldCacheTable(dbName, newTblName)) { return; } - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return; - - if (shouldCacheTable(dbName, newTblName)) { - validateTableType(newTable); - // Update table cache - try { - // Wait if background cache update is happening - tableCacheLock.readLock().lock(); - isTableCacheDirty.set(true); - sharedCache.alterTableInCache(StringUtils.normalizeIdentifier(dbName), - StringUtils.normalizeIdentifier(tblName), newTable); - } finally { - tableCacheLock.readLock().unlock(); - } - // Update partition cache (key might have changed since table name is a - // component of key) - try { - // Wait if background cache update is happening - partitionCacheLock.readLock().lock(); - isPartitionCacheDirty.set(true); - sharedCache.alterTableInPartitionCache(StringUtils.normalizeIdentifier(dbName), - StringUtils.normalizeIdentifier(tblName), newTable); - } finally { - partitionCacheLock.readLock().unlock(); - } - } else { - // Remove the table and its cached partitions, stats etc, - // since it does not pass the whitelist/blacklist filter. - // Remove table - try { - // Wait if background cache update is happening - tableCacheLock.readLock().lock(); - isTableCacheDirty.set(true); - sharedCache.removeTableFromCache(dbName, tblName); - } finally { - tableCacheLock.readLock().unlock(); - } - // Remove partitions - try { - // Wait if background cache update is happening - partitionCacheLock.readLock().lock(); - isPartitionCacheDirty.set(true); - sharedCache.removePartitionsFromCache(dbName, tblName); - } finally { - partitionCacheLock.readLock().unlock(); - } - // Remove partition col stats - try { - // Wait if background cache update is happening - partitionColStatsCacheLock.readLock().lock(); - isPartitionColStatsCacheDirty.set(true); - sharedCache.removePartitionColStatsFromCache(dbName, tblName); - } finally { - partitionColStatsCacheLock.readLock().unlock(); - } - // Update aggregate partition col stats keys wherever applicable - try { - // Wait if background cache update is happening - partitionAggrColStatsCacheLock.readLock().lock(); - isPartitionAggrColStatsCacheDirty.set(true); - sharedCache.alterTableInAggrPartitionColStatsCache(dbName, tblName, newTable); - } finally { - partitionAggrColStatsCacheLock.readLock().unlock(); - } + Table tbl = sharedCache.getTableFromCache(dbName, tblName); + if (tbl == null) { + // The table is not yet loaded in cache + return; + } + if (shouldCacheTable(dbName, tblName) && shouldCacheTable(dbName, newTblName)) { + // If old table is in the cache and the new table can also be cached + sharedCache.alterTableInCache(dbName, tblName, newTable); + } else if (!shouldCacheTable(dbName, tblName) && shouldCacheTable(dbName, newTblName)) { + // If old table is *not* in the cache but the new table can be cached + sharedCache.addTableToCache(dbName, newTblName, newTable); + } else if (shouldCacheTable(dbName, tblName) && !shouldCacheTable(dbName, newTblName)) { + // If old table is in the cache but the new table *cannot* be cached + sharedCache.removeTableFromCache(dbName, tblName); } } @@ -1183,34 +861,21 @@ public void updateCreationMetadata(String dbname, String tablename, CreationMeta @Override public List getTables(String dbName, String pattern) throws MetaException { - if (!isBlacklistWhitelistEmpty(conf) || !sharedCacheWrapper.isInitialized()) { + if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed) { return rawStore.getTables(dbName, pattern); } - SharedCache sharedCache = sharedCacheWrapper.get(); - List tableNames = new ArrayList<>(); - for (Table table : sharedCache.listCachedTables(StringUtils.normalizeIdentifier(dbName))) { - if (CacheUtils.matches(table.getTableName(), pattern)) { - tableNames.add(table.getTableName()); - } - } - return tableNames; + return sharedCache.listCachedTableNames(StringUtils.normalizeIdentifier(dbName), pattern, + (short) -1); } @Override public List getTables(String dbName, String pattern, TableType tableType) throws MetaException { - if (!isBlacklistWhitelistEmpty(conf) || !sharedCacheWrapper.isInitialized()) { + if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed) { return rawStore.getTables(dbName, pattern, tableType); } - SharedCache sharedCache = sharedCacheWrapper.get(); - List tableNames = new ArrayList<>(); - for (Table table : sharedCache.listCachedTables(StringUtils.normalizeIdentifier(dbName))) { - if (CacheUtils.matches(table.getTableName(), pattern) && - table.getTableType().equals(tableType.toString())) { - tableNames.add(table.getTableName()); - } - } - return tableNames; + return sharedCache.listCachedTableNames(StringUtils.normalizeIdentifier(dbName), pattern, + tableType); } @Override @@ -1223,10 +888,9 @@ public void updateCreationMetadata(String dbname, String tablename, CreationMeta public List getTableMeta(String dbNames, String tableNames, List tableTypes) throws MetaException { // TODO Check if all required tables are allowed, if so, get it from cache - if (!isBlacklistWhitelistEmpty(conf) || !sharedCacheWrapper.isInitialized()) { + if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed) { return rawStore.getTableMeta(dbNames, tableNames, tableTypes); } - SharedCache sharedCache = sharedCacheWrapper.get(); return sharedCache.getTableMeta(StringUtils.normalizeIdentifier(dbNames), StringUtils.normalizeIdentifier(tableNames), tableTypes); } @@ -1243,10 +907,9 @@ public void updateCreationMetadata(String dbname, String tablename, CreationMeta break; } } - if (!sharedCacheWrapper.isInitialized() || missSomeInCache) { + if (!isCachePrewarmed || missSomeInCache) { return rawStore.getTableObjectsByName(dbName, tblNames); } - SharedCache sharedCache = sharedCacheWrapper.get(); List
tables = new ArrayList<>(); for (String tblName : tblNames) { tblName = StringUtils.normalizeIdentifier(tblName); @@ -1261,38 +924,20 @@ public void updateCreationMetadata(String dbname, String tablename, CreationMeta @Override public List getAllTables(String dbName) throws MetaException { - if (!isBlacklistWhitelistEmpty(conf) || !sharedCacheWrapper.isInitialized()) { + if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed) { return rawStore.getAllTables(dbName); } - SharedCache sharedCache = sharedCacheWrapper.get(); - return getAllTablesInternal(dbName, sharedCache); - } - - private static List getAllTablesInternal(String dbName, SharedCache sharedCache) { - List tblNames = new ArrayList<>(); - for (Table tbl : sharedCache.listCachedTables(StringUtils.normalizeIdentifier(dbName))) { - tblNames.add(StringUtils.normalizeIdentifier(tbl.getTableName())); - } - return tblNames; + return sharedCache.listCachedTableNames(StringUtils.normalizeIdentifier(dbName)); } @Override public List listTableNamesByFilter(String dbName, String filter, short max_tables) throws MetaException, UnknownDBException { - if (!isBlacklistWhitelistEmpty(conf) || !sharedCacheWrapper.isInitialized()) { + if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed) { return rawStore.listTableNamesByFilter(dbName, filter, max_tables); } - SharedCache sharedCache = sharedCacheWrapper.get(); - List tableNames = new ArrayList<>(); - int count = 0; - for (Table table : sharedCache.listCachedTables(StringUtils.normalizeIdentifier(dbName))) { - if (CacheUtils.matches(table.getTableName(), filter) - && (max_tables == -1 || count < max_tables)) { - tableNames.add(table.getTableName()); - count++; - } - } - return tableNames; + return sharedCache.listCachedTableNames(StringUtils.normalizeIdentifier(dbName), filter, + max_tables); } @Override @@ -1300,16 +945,19 @@ public void updateCreationMetadata(String dbname, String tablename, CreationMeta short max_parts) throws MetaException { dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!sharedCacheWrapper.isInitialized() || !shouldCacheTable(dbName, tblName)) { + if (!shouldCacheTable(dbName, tblName)) { + return rawStore.listPartitionNames(dbName, tblName, max_parts); + } + Table tbl = sharedCache.getTableFromCache(dbName, tblName); + if (tbl == null) { + // The table is not yet loaded in cache return rawStore.listPartitionNames(dbName, tblName, max_parts); } - SharedCache sharedCache = sharedCacheWrapper.get(); List partitionNames = new ArrayList<>(); - Table t = sharedCache.getTableFromCache(dbName, tblName); int count = 0; for (Partition part : sharedCache.listCachedPartitions(dbName, tblName, max_parts)) { if (max_parts == -1 || count < max_parts) { - partitionNames.add(Warehouse.makePartName(t.getPartitionKeys(), part.getValues())); + partitionNames.add(Warehouse.makePartName(tbl.getPartitionKeys(), part.getValues())); } } return partitionNames; @@ -1338,35 +986,7 @@ public void alterPartition(String dbName, String tblName, List partVals, if (!shouldCacheTable(dbName, tblName)) { return; } - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return; - // Update partition cache - try { - // Wait if background cache update is happening - partitionCacheLock.readLock().lock(); - isPartitionCacheDirty.set(true); - sharedCache.alterPartitionInCache(dbName, tblName, partVals, newPart); - } finally { - partitionCacheLock.readLock().unlock(); - } - // Update partition column stats cache - try { - // Wait if background cache update is happening - partitionColStatsCacheLock.readLock().lock(); - isPartitionColStatsCacheDirty.set(true); - sharedCache.alterPartitionInColStatsCache(dbName, tblName, partVals, newPart); - } finally { - partitionColStatsCacheLock.readLock().unlock(); - } - // Remove aggregate partition col stats for this table - try { - // Wait if background cache update is happening - partitionAggrColStatsCacheLock.readLock().lock(); - isPartitionAggrColStatsCacheDirty.set(true); - sharedCache.removeAggrPartitionColStatsFromCache(dbName, tblName); - } finally { - partitionAggrColStatsCacheLock.readLock().unlock(); - } + sharedCache.alterPartitionInCache(dbName, tblName, partVals, newPart); } @Override @@ -1378,43 +998,7 @@ public void alterPartitions(String dbName, String tblName, List> pa if (!shouldCacheTable(dbName, tblName)) { return; } - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return; - // Update partition cache - try { - // Wait if background cache update is happening - partitionCacheLock.readLock().lock(); - isPartitionCacheDirty.set(true); - for (int i = 0; i < partValsList.size(); i++) { - List partVals = partValsList.get(i); - Partition newPart = newParts.get(i); - sharedCache.alterPartitionInCache(dbName, tblName, partVals, newPart); - } - } finally { - partitionCacheLock.readLock().unlock(); - } - // Update partition column stats cache - try { - // Wait if background cache update is happening - partitionColStatsCacheLock.readLock().lock(); - isPartitionColStatsCacheDirty.set(true); - for (int i = 0; i < partValsList.size(); i++) { - List partVals = partValsList.get(i); - Partition newPart = newParts.get(i); - sharedCache.alterPartitionInColStatsCache(dbName, tblName, partVals, newPart); - } - } finally { - partitionColStatsCacheLock.readLock().unlock(); - } - // Remove aggregate partition col stats for this table - try { - // Wait if background cache update is happening - partitionAggrColStatsCacheLock.readLock().lock(); - isPartitionAggrColStatsCacheDirty.set(true); - sharedCache.removeAggrPartitionColStatsFromCache(dbName, tblName); - } finally { - partitionAggrColStatsCacheLock.readLock().unlock(); - } + sharedCache.alterPartitionsInCache(dbName, tblName, partValsList, newParts); } @Override @@ -1455,18 +1039,18 @@ public void alterIndex(String dbname, String baseTblName, String name, private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, String defaultPartName, short maxParts, List result, SharedCache sharedCache) - throws MetaException, NoSuchObjectException { - List parts = sharedCache.listCachedPartitions( - StringUtils.normalizeIdentifier(table.getDbName()), - StringUtils.normalizeIdentifier(table.getTableName()), maxParts); + throws MetaException, NoSuchObjectException { + List parts = + sharedCache.listCachedPartitions(StringUtils.normalizeIdentifier(table.getDbName()), + StringUtils.normalizeIdentifier(table.getTableName()), maxParts); for (Partition part : parts) { result.add(Warehouse.makePartName(table.getPartitionKeys(), part.getValues())); } if (defaultPartName == null || defaultPartName.isEmpty()) { defaultPartName = MetastoreConf.getVar(getConf(), ConfVars.DEFAULTPARTITIONNAME); } - return expressionProxy.filterPartitionsByExpr( - table.getPartitionKeys(), expr, defaultPartName, result); + return expressionProxy.filterPartitionsByExpr(table.getPartitionKeys(), expr, defaultPartName, + result); } @Override @@ -1481,13 +1065,17 @@ public boolean getPartitionsByExpr(String dbName, String tblName, byte[] expr, String defaultPartitionName, short maxParts, List result) throws TException { dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!sharedCacheWrapper.isInitialized() || !shouldCacheTable(dbName, tblName)) { + if (!shouldCacheTable(dbName, tblName)) { return rawStore.getPartitionsByExpr(dbName, tblName, expr, defaultPartitionName, maxParts, result); } - SharedCache sharedCache = sharedCacheWrapper.get(); List partNames = new LinkedList<>(); Table table = sharedCache.getTableFromCache(dbName, tblName); + if (table == null) { + // The table is not yet loaded in cache + return rawStore.getPartitionsByExpr(dbName, tblName, expr, defaultPartitionName, maxParts, + result); + } boolean hasUnknownPartitions = getPartitionNamesPrunedByExprNoTxn(table, expr, defaultPartitionName, maxParts, partNames, sharedCache); return hasUnknownPartitions; @@ -1504,13 +1092,16 @@ public int getNumPartitionsByExpr(String dbName, String tblName, byte[] expr) throws MetaException, NoSuchObjectException { dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!sharedCacheWrapper.isInitialized() || !shouldCacheTable(dbName, tblName)) { + if (!shouldCacheTable(dbName, tblName)) { return rawStore.getNumPartitionsByExpr(dbName, tblName, expr); } - SharedCache sharedCache = sharedCacheWrapper.get(); String defaultPartName = MetastoreConf.getVar(getConf(), ConfVars.DEFAULTPARTITIONNAME); List partNames = new LinkedList<>(); Table table = sharedCache.getTableFromCache(dbName, tblName); + if (table == null) { + // The table is not yet loaded in cache + return rawStore.getNumPartitionsByExpr(dbName, tblName, expr); + } getPartitionNamesPrunedByExprNoTxn(table, expr, defaultPartName, Short.MAX_VALUE, partNames, sharedCache); return partNames.size(); @@ -1531,10 +1122,14 @@ public int getNumPartitionsByExpr(String dbName, String tblName, byte[] expr) List partNames) throws MetaException, NoSuchObjectException { dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!sharedCacheWrapper.isInitialized() || !shouldCacheTable(dbName, tblName)) { + if (!shouldCacheTable(dbName, tblName)) { + return rawStore.getPartitionsByNames(dbName, tblName, partNames); + } + Table table = sharedCache.getTableFromCache(dbName, tblName); + if (table == null) { + // The table is not yet loaded in cache return rawStore.getPartitionsByNames(dbName, tblName, partNames); } - SharedCache sharedCache = sharedCacheWrapper.get(); List partitions = new ArrayList<>(); for (String partName : partNames) { Partition part = sharedCache.getPartitionFromCache(dbName, tblName, partNameToVals(partName)); @@ -1707,14 +1302,17 @@ public Partition getPartitionWithAuth(String dbName, String tblName, throws MetaException, NoSuchObjectException, InvalidObjectException { dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!sharedCacheWrapper.isInitialized() || !shouldCacheTable(dbName, tblName)) { + if (!shouldCacheTable(dbName, tblName)) { + return rawStore.getPartitionWithAuth(dbName, tblName, partVals, userName, groupNames); + } + Table table = sharedCache.getTableFromCache(dbName, tblName); + if (table == null) { + // The table is not yet loaded in cache return rawStore.getPartitionWithAuth(dbName, tblName, partVals, userName, groupNames); } - SharedCache sharedCache = sharedCacheWrapper.get(); Partition p = sharedCache.getPartitionFromCache(dbName, tblName, partVals); - if (p!=null) { - Table t = sharedCache.getTableFromCache(dbName, tblName); - String partName = Warehouse.makePartName(t.getPartitionKeys(), partVals); + if (p != null) { + String partName = Warehouse.makePartName(table.getPartitionKeys(), partVals); PrincipalPrivilegeSet privs = getPartitionPrivilegeSet(dbName, tblName, partName, userName, groupNames); p.setPrivileges(privs); @@ -1728,16 +1326,19 @@ public Partition getPartitionWithAuth(String dbName, String tblName, throws MetaException, NoSuchObjectException, InvalidObjectException { dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!sharedCacheWrapper.isInitialized() || !shouldCacheTable(dbName, tblName)) { + if (!shouldCacheTable(dbName, tblName)) { + return rawStore.getPartitionsWithAuth(dbName, tblName, maxParts, userName, groupNames); + } + Table table = sharedCache.getTableFromCache(dbName, tblName); + if (table == null) { + // The table is not yet loaded in cache return rawStore.getPartitionsWithAuth(dbName, tblName, maxParts, userName, groupNames); } - SharedCache sharedCache = sharedCacheWrapper.get(); - Table t = sharedCache.getTableFromCache(dbName, tblName); List partitions = new ArrayList<>(); int count = 0; for (Partition part : sharedCache.listCachedPartitions(dbName, tblName, maxParts)) { if (maxParts == -1 || count < maxParts) { - String partName = Warehouse.makePartName(t.getPartitionKeys(), part.getValues()); + String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); PrincipalPrivilegeSet privs = getPartitionPrivilegeSet(dbName, tblName, partName, userName, groupNames); part.setPrivileges(privs); @@ -1754,13 +1355,16 @@ public Partition getPartitionWithAuth(String dbName, String tblName, throws MetaException, NoSuchObjectException { dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!sharedCacheWrapper.isInitialized() || !shouldCacheTable(dbName, tblName)) { + if (!shouldCacheTable(dbName, tblName)) { + return rawStore.listPartitionNamesPs(dbName, tblName, partVals, maxParts); + } + Table table = sharedCache.getTableFromCache(dbName, tblName); + if (table == null) { + // The table is not yet loaded in cache return rawStore.listPartitionNamesPs(dbName, tblName, partVals, maxParts); } - SharedCache sharedCache = sharedCacheWrapper.get(); List partNames = new ArrayList<>(); int count = 0; - Table t = sharedCache.getTableFromCache(dbName, tblName); for (Partition part : sharedCache.listCachedPartitions(dbName, tblName, maxParts)) { boolean psMatch = true; for (int i=0;i partitions = new ArrayList<>(); - Table t = sharedCache.getTableFromCache(dbName, tblName); int count = 0; for (Partition part : sharedCache.listCachedPartitions(dbName, tblName, maxParts)) { boolean psMatch = true; @@ -1810,7 +1418,7 @@ public Partition getPartitionWithAuth(String dbName, String tblName, continue; } if (maxParts == -1 || count < maxParts) { - String partName = Warehouse.makePartName(t.getPartitionKeys(), part.getValues()); + String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); PrincipalPrivilegeSet privs = getPartitionPrivilegeSet(dbName, tblName, partName, userName, groupNames); part.setPrivileges(privs); @@ -1830,33 +1438,19 @@ public boolean updateTableColumnStatistics(ColumnStatistics colStats) if (!shouldCacheTable(dbName, tblName)) { return succ; } - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return succ; + Table table = sharedCache.getTableFromCache(dbName, tblName); + if (table == null) { + // The table is not yet loaded in cache + return succ; + } List statsObjs = colStats.getStatsObj(); - Table tbl = getTable(dbName, tblName); List colNames = new ArrayList<>(); for (ColumnStatisticsObj statsObj : statsObjs) { colNames.add(statsObj.getColName()); } - StatsSetupConst.setColumnStatsState(tbl.getParameters(), colNames); - // Update table - try { - // Wait if background cache update is happening - tableCacheLock.readLock().lock(); - isTableCacheDirty.set(true); - sharedCache.alterTableInCache(dbName, tblName, tbl); - } finally { - tableCacheLock.readLock().unlock(); - } - // Update table col stats - try { - // Wait if background cache update is happening - tableColStatsCacheLock.readLock().lock(); - isTableColStatsCacheDirty.set(true); - sharedCache.updateTableColStatsInCache(dbName, tblName, statsObjs); - } finally { - tableColStatsCacheLock.readLock().unlock(); - } + StatsSetupConst.setColumnStatsState(table.getParameters(), colNames); + sharedCache.alterTableInCache(dbName, tblName, table); + sharedCache.updateTableColStatsInCache(dbName, tblName, statsObjs); } return succ; } @@ -1866,24 +1460,18 @@ public ColumnStatistics getTableColumnStatistics(String dbName, String tblName, List colNames) throws MetaException, NoSuchObjectException { dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!sharedCacheWrapper.isInitialized() || !shouldCacheTable(dbName, tblName)) { + if (!shouldCacheTable(dbName, tblName)) { return rawStore.getTableColumnStatistics(dbName, tblName, colNames); } - SharedCache sharedCache = sharedCacheWrapper.get(); - ColumnStatisticsDesc csd = new ColumnStatisticsDesc(true, dbName, tblName); - List colStatObjs = new ArrayList<>(); - for (String colName : colNames) { - String colStatsCacheKey = CacheUtils.buildKey(dbName, tblName, colName); - ColumnStatisticsObj colStat = sharedCache.getCachedTableColStats(colStatsCacheKey); - if (colStat != null) { - colStatObjs.add(colStat); - } - } - if (colStatObjs.isEmpty()) { - return null; - } else { - return new ColumnStatistics(csd, colStatObjs); + Table table = sharedCache.getTableFromCache(dbName, tblName); + if (table == null) { + // The table is not yet loaded in cache + return rawStore.getTableColumnStatistics(dbName, tblName, colNames); } + ColumnStatisticsDesc csd = new ColumnStatisticsDesc(true, dbName, tblName); + List colStatObjs = + sharedCache.getTableColStatsFromCache(dbName, tblName, colNames); + return new ColumnStatistics(csd, colStatObjs); } @Override @@ -1896,16 +1484,7 @@ public boolean deleteTableColumnStatistics(String dbName, String tblName, String if (!shouldCacheTable(dbName, tblName)) { return succ; } - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return succ; - try { - // Wait if background cache update is happening - tableColStatsCacheLock.readLock().lock(); - isTableColStatsCacheDirty.set(true); - sharedCache.removeTableColStatsFromCache(dbName, tblName, colName); - } finally { - tableColStatsCacheLock.readLock().unlock(); - } + sharedCache.removeTableColStatsFromCache(dbName, tblName, colName); } return succ; } @@ -1920,8 +1499,6 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics colStats, List statsObjs = colStats.getStatsObj(); Partition part = getPartition(dbName, tblName, partVals); List colNames = new ArrayList<>(); @@ -1929,34 +1506,8 @@ public boolean updatePartitionColumnStatistics(ColumnStatistics colStats, List List colStats; dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); - if (!sharedCacheWrapper.isInitialized() || !shouldCacheTable(dbName, tblName)) { + if (!shouldCacheTable(dbName, tblName)) { rawStore.get_aggr_stats_for(dbName, tblName, partNames, colNames); } - SharedCache sharedCache = sharedCacheWrapper.get(); + Table table = sharedCache.getTableFromCache(dbName, tblName); + if (table == null) { + // The table is not yet loaded in cache + return rawStore.get_aggr_stats_for(dbName, tblName, partNames, colNames); + } List allPartNames = rawStore.listPartitionNames(dbName, tblName, (short) -1); if (partNames.size() == allPartNames.size()) { colStats = sharedCache.getAggrStatsFromCache(dbName, tblName, colNames, StatsType.ALL); @@ -2051,10 +1588,8 @@ private MergedColumnStatsForPartitions mergeColStatsForPartitions(String dbName, List colStatsWithPartInfoList = new ArrayList(); for (String partName : partNames) { - String colStatsCacheKey = - CacheUtils.buildKey(dbName, tblName, partNameToVals(partName), colName); ColumnStatisticsObj colStatsForPart = - sharedCache.getCachedPartitionColStats(colStatsCacheKey); + sharedCache.getPartitionColStatsFromCache(dbName, tblName, partNameToVals(partName), colName); if (colStatsForPart != null) { ColStatsObjWithSourceInfo colStatsWithPartInfo = new ColStatsObjWithSourceInfo(colStatsForPart, dbName, tblName, partName); @@ -2170,52 +1705,6 @@ public void setMetaStoreSchemaVersion(String version, String comment) } @Override - public void dropPartitions(String dbName, String tblName, List partNames) - throws MetaException, NoSuchObjectException { - rawStore.dropPartitions(dbName, tblName, partNames); - dbName = StringUtils.normalizeIdentifier(dbName); - tblName = StringUtils.normalizeIdentifier(tblName); - if (!shouldCacheTable(dbName, tblName)) { - return; - } - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return; - // Remove partitions - try { - // Wait if background cache update is happening - partitionCacheLock.readLock().lock(); - isPartitionCacheDirty.set(true); - for (String partName : partNames) { - List vals = partNameToVals(partName); - sharedCache.removePartitionFromCache(dbName, tblName, vals); - } - } finally { - partitionCacheLock.readLock().unlock(); - } - // Remove partition col stats - try { - // Wait if background cache update is happening - partitionColStatsCacheLock.readLock().lock(); - isPartitionColStatsCacheDirty.set(true); - for (String partName : partNames) { - List part_vals = partNameToVals(partName); - sharedCache.removePartitionColStatsFromCache(dbName, tblName, part_vals); - } - } finally { - partitionColStatsCacheLock.readLock().unlock(); - } - // Remove aggregate partition col stats for this table - try { - // Wait if background cache update is happening - partitionAggrColStatsCacheLock.readLock().lock(); - isPartitionAggrColStatsCacheDirty.set(true); - sharedCache.removeAggrPartitionColStatsFromCache(dbName, tblName); - } finally { - partitionAggrColStatsCacheLock.readLock().unlock(); - } - } - - @Override public List listPrincipalDBGrantsAll( String principalName, PrincipalType principalType) { return rawStore.listPrincipalDBGrantsAll(principalName, principalType); @@ -2435,8 +1924,6 @@ public int getDatabaseCount() throws MetaException { if (!shouldCacheTable(dbName, tblName)) { return constraintNames; } - SharedCache sharedCache = sharedCacheWrapper.get(); - if (sharedCache == null) return constraintNames; sharedCache.addTableToCache(StringUtils.normalizeIdentifier(tbl.getDbName()), StringUtils.normalizeIdentifier(tbl.getTableName()), tbl); return constraintNames; @@ -2477,12 +1964,6 @@ public void dropConstraint(String dbName, String tableName, return rawStore.addNotNullConstraints(nns); } - @Override - public List getPartitionColStatsForDatabase(String dbName) - throws MetaException, NoSuchObjectException { - return rawStore.getPartitionColStatsForDatabase(dbName); - } - public RawStore getRawStore() { return rawStore; } @@ -2497,119 +1978,6 @@ public String getMetastoreDbUuid() throws MetaException { return rawStore.getMetastoreDbUuid(); } - // TODO: this is only used to hide SharedCache instance from direct use; ideally, the stuff in - // CachedStore that's specific to SharedCache (e.g. update threads) should be refactored to - // be part of this, then this could be moved out of this file (or merged with SharedCache). - private static final class SharedCacheWrapper { - private enum InitState { - NOT_ENABLED, INITIALIZING, INITIALIZED, FAILED_FATAL - } - - private final SharedCache instance = new SharedCache(); - private final Object initLock = new Object(); - private volatile InitState initState = InitState.NOT_ENABLED; - // We preserve the old setConf init behavior, where a failed prewarm would fail the query - // and give a chance to another query to try prewarming again. Basically, we'd increment the - // count and all the queries waiting for prewarm would fail; however, we will retry the prewarm - // again infinitely, so some queries might succeed later. - private int initFailureCount; - private Throwable lastError; - - /** - * A callback to updates the initialization state. - * @param error Error, if any. Null means the initialization has succeeded. - * @param isFatal Whether the error (if present) is fatal, or whether init will be retried. - */ - void updateInitState(Throwable error, boolean isFatal) { - boolean isSuccessful = error == null; - synchronized (initLock) { - if (isSuccessful) { - initState = InitState.INITIALIZED; - } else if (isFatal) { - initState = InitState.FAILED_FATAL; - lastError = error; - } else { - ++initFailureCount; - lastError = error; - } - initLock.notifyAll(); - } - } - - void startInit(Configuration conf) { - LOG.info("Initializing shared cache"); - synchronized (initLock) { - assert initState == InitState.NOT_ENABLED; - initState = InitState.INITIALIZING; - } - // The first iteration of the update thread will prewarm the cache. - startCacheUpdateService(conf); - } - - /** - * Gets the SharedCache, waiting for initialization to happen if necessary. - * Fails on any initialization error, even if the init will be retried later. - */ - public SharedCache get() throws MetaException { - if (!waitForInit()) return null; - return instance; - } - - /** Gets the shared cache unsafely (may not be ready to use); used by init methods. */ - SharedCache getUnsafe() { - return instance; - } - - private boolean waitForInit() throws MetaException { - synchronized (initLock) { - int localFailureCount = initFailureCount; - while (true) { - switch (initState) { - case INITIALIZED: return true; - case NOT_ENABLED: return false; - case FAILED_FATAL: { - throw new RuntimeException("CachedStore prewarm had a fatal error", lastError); - } - case INITIALIZING: { - try { - initLock.wait(100); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new MetaException("Interrupted"); - } - break; - } - default: throw new AssertionError(initState); - } - // Fail if any errors occured; mimicks the old behavior where a setConf prewarm failure - // would fail the current task, but cause the next setConf to try prewarm again forever. - if (initFailureCount != localFailureCount) { - throw new RuntimeException("CachedStore prewarm failed", lastError); - } - } - } - } - - /** - * Notify all threads blocked on initialization, to continue work. (We allow read while prewarm - * is running; all write calls are blocked using waitForInitAndBlock). - */ - void notifyAllBlocked() { - synchronized (initLock) { - initLock.notifyAll(); - } - } - - boolean isInitialized() { - return initState.equals(InitState.INITIALIZED); - } - } - - @VisibleForTesting - void setInitializedForTest() { - sharedCacheWrapper.updateInitState(null, false); - } - @Override public void createResourcePlan(WMResourcePlan resourcePlan, String copyFrom, int defaultPoolSize) throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException { @@ -2721,6 +2089,10 @@ public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerNa rawStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); } + public long getCacheUpdateCount() { + return sharedCache.getUpdateCount(); + } + static boolean isNotInBlackList(String dbName, String tblName) { String str = dbName + "." + tblName; for (Pattern pattern : blacklistPatterns) { diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java index 32ea17495f..6318ad1b96 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java @@ -25,11 +25,14 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.TreeMap; - +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.hadoop.hive.metastore.StatObjectConverter; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.AggrStats; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -38,11 +41,7 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TableMeta; -import org.apache.hadoop.hive.metastore.cache.CachedStore.PartitionWrapper; -import org.apache.hadoop.hive.metastore.cache.CachedStore.StorageDescriptorWrapper; -import org.apache.hadoop.hive.metastore.cache.CachedStore.TableWrapper; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; import org.apache.hadoop.hive.metastore.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,15 +49,17 @@ import com.google.common.annotations.VisibleForTesting; public class SharedCache { - private Map databaseCache = new TreeMap<>(); - private Map tableCache = new TreeMap<>(); - private Map partitionCache = new TreeMap<>(); - private Map partitionColStatsCache = new TreeMap<>(); - private Map tableColStatsCache = new TreeMap<>(); + private static ReentrantReadWriteLock cacheLock = new ReentrantReadWriteLock(true); + // For caching Database objects. Key is database name + private Map databaseCache = new HashMap(); + private AtomicBoolean isDatabaseCacheDirty = new AtomicBoolean(false); + // For caching TableWrapper objects. Key is aggregate of database name and table name + private Map tableCache = new HashMap(); + private AtomicBoolean isTableCacheDirty = new AtomicBoolean(false); private Map sdCache = new HashMap<>(); - private Map> aggrColStatsCache = - new HashMap>(); private static MessageDigest md; + static final private Logger LOG = LoggerFactory.getLogger(SharedCache.class.getName()); + private AtomicLong cacheUpdateCount = new AtomicLong(0); static enum StatsType { ALL(0), ALLBUTDEFAULT(1); @@ -74,8 +75,6 @@ public int getPosition() { } } - private static final Logger LOG = LoggerFactory.getLogger(SharedCache.class); - static { try { md = MessageDigest.getInstance("MD5"); @@ -84,43 +83,747 @@ public int getPosition() { } } - public synchronized Database getDatabaseFromCache(String name) { - return databaseCache.get(name)!=null?databaseCache.get(name).deepCopy():null; + static class TableWrapper { + Table t; + String location; + Map parameters; + byte[] sdHash; + ReentrantReadWriteLock tableLock = new ReentrantReadWriteLock(true); + // For caching column stats for an unpartitioned table + // Key is column name and the value is the col stat object + private Map tableColStatsCache = + new HashMap(); + private AtomicBoolean isTableColStatsCacheDirty = new AtomicBoolean(false); + // For caching partition objects + // Ket is partition values and the value is a wrapper around the partition object + private Map partitionCache = new HashMap(); + private AtomicBoolean isPartitionCacheDirty = new AtomicBoolean(false); + // For caching column stats for a partitioned table + // Key is aggregate of partition values, column name and the value is the col stat object + private Map partitionColStatsCache = + new HashMap(); + private AtomicBoolean isPartitionColStatsCacheDirty = new AtomicBoolean(false); + // For caching aggregate column stats for all and all minus default partition + // Key is column name and the value is a list of 2 col stat objects + // (all partitions and all but default) + private Map> aggrColStatsCache = + new HashMap>(); + private AtomicBoolean isAggrPartitionColStatsCacheDirty = new AtomicBoolean(false); + + TableWrapper(Table t, byte[] sdHash, String location, Map parameters) { + this.t = t; + this.sdHash = sdHash; + this.location = location; + this.parameters = parameters; + } + + public Table getTable() { + return t; + } + + public void setTable(Table t) { + this.t = t; + } + + public byte[] getSdHash() { + return sdHash; + } + + public void setSdHash(byte[] sdHash) { + this.sdHash = sdHash; + } + + public String getLocation() { + return location; + } + + public void setLocation(String location) { + this.location = location; + } + + public Map getParameters() { + return parameters; + } + + public void setParameters(Map parameters) { + this.parameters = parameters; + } + + void cachePartition(Partition part, SharedCache sharedCache) { + try { + tableLock.writeLock().lock(); + PartitionWrapper wrapper = makePartitionWrapper(part, sharedCache); + partitionCache.put(CacheUtils.buildKey(part.getValues()), wrapper); + isPartitionCacheDirty.set(true); + // Invalidate cached aggregate stats + if (!aggrColStatsCache.isEmpty()) { + aggrColStatsCache.clear(); + } + } finally { + tableLock.writeLock().unlock(); + } + } + + void cachePartitions(List parts, SharedCache sharedCache) { + try { + tableLock.writeLock().lock(); + for (Partition part : parts) { + PartitionWrapper wrapper = makePartitionWrapper(part, sharedCache); + partitionCache.put(CacheUtils.buildKey(part.getValues()), wrapper); + isPartitionCacheDirty.set(true); + } + // Invalidate cached aggregate stats + if (!aggrColStatsCache.isEmpty()) { + aggrColStatsCache.clear(); + } + } finally { + tableLock.writeLock().unlock(); + } + } + + public Partition getPartition(List partVals, SharedCache sharedCache) { + Partition part = null; + try { + tableLock.readLock().lock(); + PartitionWrapper wrapper = partitionCache.get(CacheUtils.buildKey(partVals)); + if (wrapper == null) { + return null; + } + part = CacheUtils.assemble(wrapper, sharedCache); + } finally { + tableLock.readLock().unlock(); + } + return part; + } + + public List listPartitions(int max, SharedCache sharedCache) { + List parts = new ArrayList<>(); + int count = 0; + try { + tableLock.readLock().lock(); + for (PartitionWrapper wrapper : partitionCache.values()) { + if (max == -1 || count < max) { + parts.add(CacheUtils.assemble(wrapper, sharedCache)); + count++; + } + } + } finally { + tableLock.readLock().unlock(); + } + return parts; + } + + public boolean containsPartition(List partVals) { + boolean containsPart = false; + try { + tableLock.readLock().lock(); + containsPart = partitionCache.containsKey(CacheUtils.buildKey(partVals)); + } finally { + tableLock.readLock().unlock(); + } + return containsPart; + } + + public Partition removePartition(List partVal, SharedCache sharedCache) { + Partition part = null; + try { + tableLock.writeLock().lock(); + PartitionWrapper wrapper = partitionCache.remove(CacheUtils.buildKey(partVal)); + isPartitionCacheDirty.set(true); + if (wrapper.getSdHash() != null) { + sharedCache.decrSd(wrapper.getSdHash()); + } + part = CacheUtils.assemble(wrapper, sharedCache); + // Remove col stats + String partialKey = CacheUtils.buildKey(partVal); + Iterator> iterator = + partitionColStatsCache.entrySet().iterator(); + while (iterator.hasNext()) { + Entry entry = iterator.next(); + String key = entry.getKey(); + if (key.toLowerCase().startsWith(partialKey.toLowerCase())) { + iterator.remove(); + } + } + // Invalidate cached aggregate stats + if (!aggrColStatsCache.isEmpty()) { + aggrColStatsCache.clear(); + } + } finally { + tableLock.writeLock().unlock(); + } + return part; + } + + public void removePartitions(List> partVals, SharedCache sharedCache) { + try { + tableLock.writeLock().lock(); + for (List partVal : partVals) { + removePartition(partVal, sharedCache); + } + } finally { + tableLock.writeLock().unlock(); + } + } + + public void alterPartition(List partVals, Partition newPart, SharedCache sharedCache) { + try { + tableLock.writeLock().lock(); + removePartition(partVals, sharedCache); + cachePartition(newPart, sharedCache); + } finally { + tableLock.writeLock().unlock(); + } + } + + public void alterPartitions(List> partValsList, List newParts, + SharedCache sharedCache) { + try { + tableLock.writeLock().lock(); + for (int i = 0; i < partValsList.size(); i++) { + List partVals = partValsList.get(i); + Partition newPart = newParts.get(i); + alterPartition(partVals, newPart, sharedCache); + } + } finally { + tableLock.writeLock().unlock(); + } + } + + public void refreshPartitions(List partitions, SharedCache sharedCache) { + Map newPartitionCache = new HashMap(); + try { + tableLock.writeLock().lock(); + for (Partition part : partitions) { + if (isPartitionCacheDirty.compareAndSet(true, false)) { + LOG.debug("Skipping partition cache update for table: " + getTable().getTableName() + + "; the partition list we have is dirty."); + return; + } + String key = CacheUtils.buildKey(part.getValues()); + PartitionWrapper wrapper = partitionCache.get(key); + if (wrapper != null) { + if (wrapper.getSdHash() != null) { + sharedCache.decrSd(wrapper.getSdHash()); + } + } + wrapper = makePartitionWrapper(part, sharedCache); + newPartitionCache.put(key, wrapper); + } + partitionCache = newPartitionCache; + } finally { + tableLock.writeLock().unlock(); + } + } + + public void updateTableColStats(List colStatsForTable) { + try { + tableLock.writeLock().lock(); + for (ColumnStatisticsObj colStatObj : colStatsForTable) { + // Get old stats object if present + String key = colStatObj.getColName(); + ColumnStatisticsObj oldStatsObj = tableColStatsCache.get(key); + if (oldStatsObj != null) { + // Update existing stat object's field + StatObjectConverter.setFieldsIntoOldStats(oldStatsObj, colStatObj); + } else { + // No stats exist for this key; add a new object to the cache + // TODO: get rid of deepCopy after making sure callers don't use references + tableColStatsCache.put(key, colStatObj.deepCopy()); + } + } + isTableColStatsCacheDirty.set(true); + } finally { + tableLock.writeLock().unlock(); + } + } + + public void refreshTableColStats(List colStatsForTable) { + Map newTableColStatsCache = + new HashMap(); + try { + tableLock.writeLock().lock(); + for (ColumnStatisticsObj colStatObj : colStatsForTable) { + if (isTableColStatsCacheDirty.compareAndSet(true, false)) { + LOG.debug("Skipping table col stats cache update for table: " + + getTable().getTableName() + "; the table col stats list we have is dirty."); + return; + } + String key = colStatObj.getColName(); + // TODO: get rid of deepCopy after making sure callers don't use references + newTableColStatsCache.put(key, colStatObj.deepCopy()); + } + tableColStatsCache = newTableColStatsCache; + } finally { + tableLock.writeLock().unlock(); + } + } + + public List getCachedTableColStats(List colNames) { + List colStatObjs = new ArrayList(); + try { + tableLock.readLock().lock(); + for (String colName : colNames) { + ColumnStatisticsObj colStatObj = tableColStatsCache.get(colName); + if (colStatObj != null) { + colStatObjs.add(colStatObj); + } + } + } finally { + tableLock.readLock().unlock(); + } + return colStatObjs; + } + + public void removeTableColStats(String colName) { + try { + tableLock.writeLock().lock(); + tableColStatsCache.remove(colName); + isTableColStatsCacheDirty.set(true); + } finally { + tableLock.writeLock().unlock(); + } + } + + public ColumnStatisticsObj getPartitionColStats(List partVal, String colName) { + try { + tableLock.readLock().lock(); + return partitionColStatsCache.get(CacheUtils.buildKey(partVal, colName)); + } finally { + tableLock.readLock().unlock(); + } + } + + public void updatePartitionColStats(List partVal, + List colStatsObjs) { + try { + tableLock.writeLock().lock(); + addPartitionColStatsToCache(partVal, colStatsObjs); + isPartitionColStatsCacheDirty.set(true); + // Invalidate cached aggregate stats + if (!aggrColStatsCache.isEmpty()) { + aggrColStatsCache.clear(); + } + } finally { + tableLock.writeLock().unlock(); + } + } + + public void removePartitionColStats(List partVals, String colName) { + try { + tableLock.writeLock().lock(); + partitionColStatsCache.remove(CacheUtils.buildKey(partVals, colName)); + isPartitionColStatsCacheDirty.set(true); + // Invalidate cached aggregate stats + if (!aggrColStatsCache.isEmpty()) { + aggrColStatsCache.clear(); + } + } finally { + tableLock.writeLock().unlock(); + } + } + + private void addPartitionColStatsToCache(List partVal, + List colStatsObjs) { + for (ColumnStatisticsObj colStatObj : colStatsObjs) { + // Get old stats object if present + String key = CacheUtils.buildKey(partVal, colStatObj.getColName()); + ColumnStatisticsObj oldStatsObj = partitionColStatsCache.get(key); + if (oldStatsObj != null) { + // Update existing stat object's field + StatObjectConverter.setFieldsIntoOldStats(oldStatsObj, colStatObj); + } else { + // No stats exist for this key; add a new object to the cache + // TODO: get rid of deepCopy after making sure callers don't use references + partitionColStatsCache.put(key, colStatObj.deepCopy()); + } + } + } + + public void refreshPartitionColStats(List partitionColStats) { + Map newPartitionColStatsCache = + new HashMap(); + try { + tableLock.writeLock().lock(); + String tableName = StringUtils.normalizeIdentifier(getTable().getTableName()); + for (ColumnStatistics cs : partitionColStats) { + if (isPartitionColStatsCacheDirty.compareAndSet(true, false)) { + LOG.debug("Skipping partition column stats cache update for table: " + + getTable().getTableName() + "; the partition column stats list we have is dirty"); + return; + } + List partVal; + try { + partVal = Warehouse.makeValsFromName(cs.getStatsDesc().getPartName(), null); + List colStatsObjs = cs.getStatsObj(); + for (ColumnStatisticsObj colStatObj : colStatsObjs) { + if (isPartitionColStatsCacheDirty.compareAndSet(true, false)) { + LOG.debug("Skipping partition column stats cache update for table: " + + getTable().getTableName() + "; the partition column list we have is dirty"); + return; + } + String key = CacheUtils.buildKey(partVal, colStatObj.getColName()); + newPartitionColStatsCache.put(key, colStatObj.deepCopy()); + } + } catch (MetaException e) { + LOG.debug("Unable to cache partition column stats for table: " + tableName, e); + } + } + partitionColStatsCache = newPartitionColStatsCache; + } finally { + tableLock.writeLock().unlock(); + } + } + + public List getAggrPartitionColStats(List colNames, + StatsType statsType) { + List colStats = new ArrayList(); + try { + tableLock.readLock().lock(); + for (String colName : colNames) { + List colStatList = aggrColStatsCache.get(colName); + // If unable to find stats for a column, return null so we can build stats + if (colStatList == null) { + return null; + } + ColumnStatisticsObj colStatObj = colStatList.get(statsType.getPosition()); + // If unable to find stats for this StatsType, return null so we can build stats + if (colStatObj == null) { + return null; + } + colStats.add(colStatObj); + } + } finally { + tableLock.readLock().unlock(); + } + return colStats; + } + + public void cacheAggrPartitionColStats(AggrStats aggrStatsAllPartitions, + AggrStats aggrStatsAllButDefaultPartition) { + try { + tableLock.writeLock().lock(); + if (aggrStatsAllPartitions != null) { + for (ColumnStatisticsObj statObj : aggrStatsAllPartitions.getColStats()) { + if (statObj != null) { + List aggrStats = new ArrayList(); + aggrStats.add(StatsType.ALL.ordinal(), statObj.deepCopy()); + aggrColStatsCache.put(statObj.getColName(), aggrStats); + } + } + } + if (aggrStatsAllButDefaultPartition != null) { + for (ColumnStatisticsObj statObj : aggrStatsAllButDefaultPartition.getColStats()) { + if (statObj != null) { + List aggrStats = aggrColStatsCache.get(statObj.getColName()); + if (aggrStats == null) { + aggrStats = new ArrayList(); + } + aggrStats.add(StatsType.ALLBUTDEFAULT.ordinal(), statObj.deepCopy()); + } + } + } + isAggrPartitionColStatsCacheDirty.set(true); + } finally { + tableLock.writeLock().unlock(); + } + } + + public void refreshAggrPartitionColStats(AggrStats aggrStatsAllPartitions, + AggrStats aggrStatsAllButDefaultPartition) { + Map> newAggrColStatsCache = + new HashMap>(); + try { + tableLock.writeLock().lock(); + if (aggrStatsAllPartitions != null) { + for (ColumnStatisticsObj statObj : aggrStatsAllPartitions.getColStats()) { + if (isAggrPartitionColStatsCacheDirty.compareAndSet(true, false)) { + LOG.debug("Skipping aggregate stats cache update for table: " + + getTable().getTableName() + "; the aggregate stats list we have is dirty"); + return; + } + if (statObj != null) { + List aggrStats = new ArrayList(); + aggrStats.add(StatsType.ALL.ordinal(), statObj.deepCopy()); + newAggrColStatsCache.put(statObj.getColName(), aggrStats); + } + } + } + if (aggrStatsAllButDefaultPartition != null) { + for (ColumnStatisticsObj statObj : aggrStatsAllButDefaultPartition.getColStats()) { + if (isAggrPartitionColStatsCacheDirty.compareAndSet(true, false)) { + LOG.debug("Skipping aggregate stats cache update for table: " + + getTable().getTableName() + "; the aggregate stats list we have is dirty"); + return; + } + if (statObj != null) { + List aggrStats = newAggrColStatsCache.get(statObj.getColName()); + if (aggrStats == null) { + aggrStats = new ArrayList(); + } + aggrStats.add(StatsType.ALLBUTDEFAULT.ordinal(), statObj.deepCopy()); + } + } + } + aggrColStatsCache = newAggrColStatsCache; + } finally { + tableLock.writeLock().unlock(); + } + } + + private void updateTableObj(Table newTable, SharedCache sharedCache) { + byte[] sdHash = getSdHash(); + // Remove old table object's sd hash + if (sdHash != null) { + sharedCache.decrSd(sdHash); + } + Table tblCopy = newTable.deepCopy(); + if (tblCopy.getPartitionKeys() != null) { + for (FieldSchema fs : tblCopy.getPartitionKeys()) { + fs.setName(StringUtils.normalizeIdentifier(fs.getName())); + } + } + setTable(tblCopy); + if (tblCopy.getSd() != null) { + sdHash = MetaStoreUtils.hashStorageDescriptor(tblCopy.getSd(), md); + StorageDescriptor sd = tblCopy.getSd(); + sharedCache.increSd(sd, sdHash); + tblCopy.setSd(null); + setSdHash(sdHash); + setLocation(sd.getLocation()); + setParameters(sd.getParameters()); + } else { + setSdHash(null); + setLocation(null); + setParameters(null); + } + } + + private PartitionWrapper makePartitionWrapper(Partition part, SharedCache sharedCache) { + Partition partCopy = part.deepCopy(); + PartitionWrapper wrapper; + if (part.getSd() != null) { + byte[] sdHash = MetaStoreUtils.hashStorageDescriptor(part.getSd(), md); + StorageDescriptor sd = part.getSd(); + sharedCache.increSd(sd, sdHash); + partCopy.setSd(null); + wrapper = new PartitionWrapper(partCopy, sdHash, sd.getLocation(), sd.getParameters()); + } else { + wrapper = new PartitionWrapper(partCopy, null, null, null); + } + return wrapper; + } } - public synchronized void addDatabaseToCache(String dbName, Database db) { - Database dbCopy = db.deepCopy(); - dbCopy.setName(StringUtils.normalizeIdentifier(dbName)); - databaseCache.put(dbName, dbCopy); + static class PartitionWrapper { + Partition p; + String location; + Map parameters; + byte[] sdHash; + + PartitionWrapper(Partition p, byte[] sdHash, String location, Map parameters) { + this.p = p; + this.sdHash = sdHash; + this.location = location; + this.parameters = parameters; + } + + public Partition getPartition() { + return p; + } + + public byte[] getSdHash() { + return sdHash; + } + + public String getLocation() { + return location; + } + + public Map getParameters() { + return parameters; + } } - public synchronized void removeDatabaseFromCache(String dbName) { - databaseCache.remove(dbName); + static class StorageDescriptorWrapper { + StorageDescriptor sd; + int refCount = 0; + + StorageDescriptorWrapper(StorageDescriptor sd, int refCount) { + this.sd = sd; + this.refCount = refCount; + } + + public StorageDescriptor getSd() { + return sd; + } + + public int getRefCount() { + return refCount; + } + } + + public Database getDatabaseFromCache(String name) { + Database db = null; + try { + cacheLock.readLock().lock(); + if (databaseCache.get(name) != null) { + db = databaseCache.get(name).deepCopy(); + } + } finally { + cacheLock.readLock().unlock(); + } + return db; + } + + public void addDatabaseToCache(Database db) { + try { + cacheLock.writeLock().lock(); + Database dbCopy = db.deepCopy(); + // ObjectStore also stores db name in lowercase + dbCopy.setName(dbCopy.getName().toLowerCase()); + databaseCache.put(StringUtils.normalizeIdentifier(dbCopy.getName()), dbCopy); + isDatabaseCacheDirty.set(true); + } finally { + cacheLock.writeLock().unlock(); + } + } + + public void removeDatabaseFromCache(String dbName) { + try { + cacheLock.writeLock().lock(); + if (databaseCache.remove(dbName) != null) { + isDatabaseCacheDirty.set(true); + } + ; + } finally { + cacheLock.writeLock().unlock(); + } + } + + public List listCachedDatabases() { + List results = new ArrayList<>(); + try { + cacheLock.readLock().lock(); + results.addAll(databaseCache.keySet()); + } finally { + cacheLock.readLock().unlock(); + } + return results; + } + + public List listCachedDatabases(String pattern) { + List results = new ArrayList<>(); + try { + cacheLock.readLock().lock(); + for (String dbName : databaseCache.keySet()) { + dbName = StringUtils.normalizeIdentifier(dbName); + if (CacheUtils.matches(dbName, pattern)) { + results.add(dbName); + } + } + } finally { + cacheLock.readLock().unlock(); + } + return results; } - public synchronized List listCachedDatabases() { - return new ArrayList<>(databaseCache.keySet()); + public void alterDatabaseInCache(String dbName, Database newDb) { + try { + cacheLock.writeLock().lock(); + removeDatabaseFromCache(dbName); + addDatabaseToCache(newDb.deepCopy()); + isDatabaseCacheDirty.set(true); + } finally { + cacheLock.writeLock().unlock(); + } } - public synchronized void alterDatabaseInCache(String dbName, Database newDb) { - removeDatabaseFromCache(StringUtils.normalizeIdentifier(dbName)); - addDatabaseToCache(StringUtils.normalizeIdentifier(newDb.getName()), newDb.deepCopy()); + public void refreshDatabasesInCache(List databases) { + try { + cacheLock.writeLock().lock(); + if (isDatabaseCacheDirty.compareAndSet(true, false)) { + LOG.debug("Skipping database cache update; the database list we have is dirty."); + return; + } + databaseCache.clear(); + for (Database db : databases) { + addDatabaseToCache(db); + } + } finally { + cacheLock.writeLock().unlock(); + } } - public synchronized int getCachedDatabaseCount() { - return databaseCache.size(); + public int getCachedDatabaseCount() { + try { + cacheLock.readLock().lock(); + return databaseCache.size(); + } finally { + cacheLock.readLock().unlock(); + } } - public synchronized Table getTableFromCache(String dbName, String tableName) { - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tableName)); - if (tblWrapper == null) { - return null; + public void populateTableInCache(Table table, ColumnStatistics tableColStats, + List partitions, List partitionColStats, + AggrStats aggrStatsAllPartitions, AggrStats aggrStatsAllButDefaultPartition) { + String dbName = StringUtils.normalizeIdentifier(table.getDbName()); + String tableName = StringUtils.normalizeIdentifier(table.getTableName()); + TableWrapper tblWrapper = createTableWrapper(dbName, tableName, table); + if (!table.isSetPartitionKeys()) { + tblWrapper.updateTableColStats(tableColStats.getStatsObj()); + } else { + tblWrapper.cachePartitions(partitions, this); + for (ColumnStatistics cs : partitionColStats) { + List partVal; + try { + partVal = Warehouse.makeValsFromName(cs.getStatsDesc().getPartName(), null); + List colStats = cs.getStatsObj(); + tblWrapper.updatePartitionColStats(partVal, colStats); + } catch (MetaException e) { + LOG.debug("Unable to cache partition column stats for table: " + tableName, e); + } + } + tblWrapper.cacheAggrPartitionColStats(aggrStatsAllPartitions, + aggrStatsAllButDefaultPartition); + } + try { + cacheLock.writeLock().lock(); + tableCache.put(CacheUtils.buildKey(dbName, tableName), tblWrapper); + isTableCacheDirty.set(true); + } finally { + cacheLock.writeLock().unlock(); + } + } + + public Table getTableFromCache(String dbName, String tableName) { + Table t = null; + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tableName)); + if (tblWrapper != null) { + t = CacheUtils.assemble(tblWrapper, this); + } + } finally { + cacheLock.readLock().unlock(); } - Table t = CacheUtils.assemble(tblWrapper, this); return t; } - public synchronized void addTableToCache(String dbName, String tblName, Table tbl) { + public TableWrapper addTableToCache(String dbName, String tblName, Table tbl) { + try { + cacheLock.writeLock().lock(); + TableWrapper wrapper = createTableWrapper(dbName, tblName, tbl); + tableCache.put(CacheUtils.buildKey(dbName, tblName), wrapper); + isTableCacheDirty.set(true); + return wrapper; + } finally { + cacheLock.writeLock().unlock(); + } + } + + private TableWrapper createTableWrapper(String dbName, String tblName, Table tbl) { + TableWrapper wrapper; Table tblCopy = tbl.deepCopy(); tblCopy.setDbName(StringUtils.normalizeIdentifier(dbName)); tblCopy.setTableName(StringUtils.normalizeIdentifier(tblName)); @@ -129,7 +832,6 @@ public synchronized void addTableToCache(String dbName, String tblName, Table tb fs.setName(StringUtils.normalizeIdentifier(fs.getName())); } } - TableWrapper wrapper; if (tbl.getSd() != null) { byte[] sdHash = MetaStoreUtils.hashStorageDescriptor(tbl.getSd(), md); StorageDescriptor sd = tbl.getSd(); @@ -139,481 +841,446 @@ public synchronized void addTableToCache(String dbName, String tblName, Table tb } else { wrapper = new TableWrapper(tblCopy, null, null, null); } - tableCache.put(CacheUtils.buildKey(dbName, tblName), wrapper); + return wrapper; } - public synchronized void removeTableFromCache(String dbName, String tblName) { - TableWrapper tblWrapper = tableCache.remove(CacheUtils.buildKey(dbName, tblName)); - byte[] sdHash = tblWrapper.getSdHash(); - if (sdHash!=null) { - decrSd(sdHash); + public void removeTableFromCache(String dbName, String tblName) { + try { + cacheLock.writeLock().lock(); + TableWrapper tblWrapper = tableCache.remove(CacheUtils.buildKey(dbName, tblName)); + byte[] sdHash = tblWrapper.getSdHash(); + if (sdHash != null) { + decrSd(sdHash); + } + isTableCacheDirty.set(true); + } finally { + cacheLock.writeLock().unlock(); } } - public synchronized ColumnStatisticsObj getCachedTableColStats(String colStatsCacheKey) { - return tableColStatsCache.get(colStatsCacheKey)!=null?tableColStatsCache.get(colStatsCacheKey).deepCopy():null; - } - - public synchronized void removeTableColStatsFromCache(String dbName, String tblName) { - String partialKey = CacheUtils.buildKeyWithDelimit(dbName, tblName); - Iterator> iterator = - tableColStatsCache.entrySet().iterator(); - while (iterator.hasNext()) { - Entry entry = iterator.next(); - String key = entry.getKey(); - if (key.toLowerCase().startsWith(partialKey.toLowerCase())) { - iterator.remove(); + public void alterTableInCache(String dbName, String tblName, Table newTable) { + try { + cacheLock.writeLock().lock(); + TableWrapper tblWrapper = tableCache.remove(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + tblWrapper.updateTableObj(newTable, this); + String newDbName = StringUtils.normalizeIdentifier(newTable.getDbName()); + String newTblName = StringUtils.normalizeIdentifier(newTable.getTableName()); + tableCache.put(CacheUtils.buildKey(newDbName, newTblName), tblWrapper); + isTableCacheDirty.set(true); } + } finally { + cacheLock.writeLock().unlock(); } } - public synchronized void removeTableColStatsFromCache(String dbName, String tblName, - String colName) { - if (colName == null) { - removeTableColStatsFromCache(dbName, tblName); - } else { - tableColStatsCache.remove(CacheUtils.buildKey(dbName, tblName, colName)); + public List
listCachedTables(String dbName) { + List
tables = new ArrayList<>(); + try { + cacheLock.readLock().lock(); + for (TableWrapper wrapper : tableCache.values()) { + if (wrapper.getTable().getDbName().equals(dbName)) { + tables.add(CacheUtils.assemble(wrapper, this)); + } + } + } finally { + cacheLock.readLock().unlock(); } + return tables; } - public synchronized void updateTableColStatsInCache(String dbName, String tableName, - List colStatsForTable) { - for (ColumnStatisticsObj colStatObj : colStatsForTable) { - // Get old stats object if present - String key = CacheUtils.buildKey(dbName, tableName, colStatObj.getColName()); - ColumnStatisticsObj oldStatsObj = tableColStatsCache.get(key); - if (oldStatsObj != null) { - LOG.debug("CachedStore: updating table column stats for column: " + colStatObj.getColName() - + ", of table: " + tableName + " and database: " + dbName); - // Update existing stat object's field - StatObjectConverter.setFieldsIntoOldStats(oldStatsObj, colStatObj); - } else { - // No stats exist for this key; add a new object to the cache - tableColStatsCache.put(key, colStatObj); + public List listCachedTableNames(String dbName) { + List tableNames = new ArrayList<>(); + try { + cacheLock.readLock().lock(); + for (TableWrapper wrapper : tableCache.values()) { + if (wrapper.getTable().getDbName().equals(dbName)) { + tableNames.add(StringUtils.normalizeIdentifier(wrapper.getTable().getTableName())); + } } + } finally { + cacheLock.readLock().unlock(); } + return tableNames; } - public synchronized void alterTableInCache(String dbName, String tblName, Table newTable) { - removeTableFromCache(dbName, tblName); - addTableToCache(StringUtils.normalizeIdentifier(newTable.getDbName()), - StringUtils.normalizeIdentifier(newTable.getTableName()), newTable); + public List listCachedTableNames(String dbName, String pattern, short maxTables) { + List tableNames = new ArrayList(); + try { + cacheLock.readLock().lock(); + int count = 0; + for (TableWrapper wrapper : tableCache.values()) { + if ((wrapper.getTable().getDbName().equals(dbName)) + && CacheUtils.matches(wrapper.getTable().getTableName(), pattern) + && (maxTables == -1 || count < maxTables)) { + tableNames.add(StringUtils.normalizeIdentifier(wrapper.getTable().getTableName())); + count++; + } + } + } finally { + cacheLock.readLock().unlock(); + } + return tableNames; } - public synchronized void alterTableInPartitionCache(String dbName, String tblName, - Table newTable) { - if (!dbName.equals(newTable.getDbName()) || !tblName.equals(newTable.getTableName())) { - List partitions = listCachedPartitions(dbName, tblName, -1); - for (Partition part : partitions) { - removePartitionFromCache(part.getDbName(), part.getTableName(), part.getValues()); - part.setDbName(StringUtils.normalizeIdentifier(newTable.getDbName())); - part.setTableName(StringUtils.normalizeIdentifier(newTable.getTableName())); - addPartitionToCache(StringUtils.normalizeIdentifier(newTable.getDbName()), - StringUtils.normalizeIdentifier(newTable.getTableName()), part); + public List listCachedTableNames(String dbName, String pattern, TableType tableType) { + List tableNames = new ArrayList(); + try { + cacheLock.readLock().lock(); + for (TableWrapper wrapper : tableCache.values()) { + if ((wrapper.getTable().getDbName().equals(dbName)) + && CacheUtils.matches(wrapper.getTable().getTableName(), pattern) + && wrapper.getTable().getTableType().equals(tableType.toString())) { + tableNames.add(StringUtils.normalizeIdentifier(wrapper.getTable().getTableName())); + } } + } finally { + cacheLock.readLock().unlock(); } + return tableNames; } - public synchronized void alterTableInTableColStatsCache(String dbName, String tblName, - Table newTable) { - if (!dbName.equals(newTable.getDbName()) || !tblName.equals(newTable.getTableName())) { - String oldPartialTableStatsKey = CacheUtils.buildKeyWithDelimit(dbName, tblName); - Iterator> iterator = - tableColStatsCache.entrySet().iterator(); - Map newTableColStats = - new HashMap<>(); - while (iterator.hasNext()) { - Entry entry = iterator.next(); - String key = entry.getKey(); - ColumnStatisticsObj colStatObj = entry.getValue(); - if (key.toLowerCase().startsWith(oldPartialTableStatsKey.toLowerCase())) { - String[] decomposedKey = CacheUtils.splitTableColStats(key); - String newKey = CacheUtils.buildKey(decomposedKey[0], decomposedKey[1], decomposedKey[2]); - newTableColStats.put(newKey, colStatObj); - iterator.remove(); + public void refreshTablesInCache(String dbName, List
tables) { + try { + cacheLock.writeLock().lock(); + if (isTableCacheDirty.compareAndSet(true, false)) { + LOG.debug("Skipping table cache update; the table list we have is dirty."); + return; + } + Map newTableCache = new HashMap(); + for (Table tbl : tables) { + String tblName = StringUtils.normalizeIdentifier(tbl.getTableName()); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + tblWrapper.updateTableObj(tbl, this); + } else { + tblWrapper = createTableWrapper(dbName, tblName, tbl); } + newTableCache.put(CacheUtils.buildKey(dbName, tblName), tblWrapper); } - tableColStatsCache.putAll(newTableColStats); + tableCache.clear(); + tableCache = newTableCache; + } finally { + cacheLock.writeLock().unlock(); } } - public synchronized void alterTableInPartitionColStatsCache(String dbName, String tblName, - Table newTable) { - if (!dbName.equals(newTable.getDbName()) || !tblName.equals(newTable.getTableName())) { - List partitions = listCachedPartitions(dbName, tblName, -1); - Map newPartitionColStats = new HashMap<>(); - for (Partition part : partitions) { - String oldPartialPartitionKey = - CacheUtils.buildKeyWithDelimit(dbName, tblName, part.getValues()); - Iterator> iterator = - partitionColStatsCache.entrySet().iterator(); - while (iterator.hasNext()) { - Entry entry = iterator.next(); - String key = entry.getKey(); - ColumnStatisticsObj colStatObj = entry.getValue(); - if (key.toLowerCase().startsWith(oldPartialPartitionKey.toLowerCase())) { - Object[] decomposedKey = CacheUtils.splitPartitionColStats(key); - // New key has the new table name - String newKey = CacheUtils.buildKey((String) decomposedKey[0], newTable.getTableName(), - (List) decomposedKey[2], (String) decomposedKey[3]); - newPartitionColStats.put(newKey, colStatObj); - iterator.remove(); - } - } + public List getTableColStatsFromCache(String dbName, String tblName, + List colNames) { + List colStatObjs = new ArrayList(); + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + colStatObjs = tblWrapper.getCachedTableColStats(colNames); } - partitionColStatsCache.putAll(newPartitionColStats); + } finally { + cacheLock.readLock().unlock(); } + return colStatObjs; } - public synchronized void alterTableInAggrPartitionColStatsCache(String dbName, String tblName, - Table newTable) { - if (!dbName.equals(newTable.getDbName()) || !tblName.equals(newTable.getTableName())) { - Map> newAggrColStatsCache = - new HashMap>(); - String oldPartialKey = CacheUtils.buildKeyWithDelimit(dbName, tblName); - Iterator>> iterator = - aggrColStatsCache.entrySet().iterator(); - while (iterator.hasNext()) { - Entry> entry = iterator.next(); - String key = entry.getKey(); - List value = entry.getValue(); - if (key.toLowerCase().startsWith(oldPartialKey.toLowerCase())) { - Object[] decomposedKey = CacheUtils.splitAggrColStats(key); - // New key has the new table name - String newKey = CacheUtils.buildKey((String) decomposedKey[0], newTable.getTableName(), - (String) decomposedKey[2]); - newAggrColStatsCache.put(newKey, value); - iterator.remove(); - } + public void removeTableColStatsFromCache(String dbName, String tblName, String colName) { + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + tblWrapper.removeTableColStats(colName); } - aggrColStatsCache.putAll(newAggrColStatsCache); + } finally { + cacheLock.readLock().unlock(); } } - public synchronized int getCachedTableCount() { - return tableCache.size(); + public void updateTableColStatsInCache(String dbName, String tableName, + List colStatsForTable) { + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tableName)); + if (tblWrapper != null) { + tblWrapper.updateTableColStats(colStatsForTable); + } + } finally { + cacheLock.readLock().unlock(); + } } - public synchronized List
listCachedTables(String dbName) { - List
tables = new ArrayList<>(); - for (TableWrapper wrapper : tableCache.values()) { - if (wrapper.getTable().getDbName().equals(dbName)) { - tables.add(CacheUtils.assemble(wrapper, this)); + public void refreshTableColStatsInCache(String dbName, String tableName, + List colStatsForTable) { + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tableName)); + if (tblWrapper != null) { + tblWrapper.refreshTableColStats(colStatsForTable); } + } finally { + cacheLock.readLock().unlock(); + } + } + + public int getCachedTableCount() { + try { + cacheLock.readLock().lock(); + return tableCache.size(); + } finally { + cacheLock.readLock().unlock(); } - return tables; } - public synchronized List getTableMeta(String dbNames, String tableNames, List tableTypes) { + public List getTableMeta(String dbNames, String tableNames, + List tableTypes) { List tableMetas = new ArrayList<>(); - for (String dbName : listCachedDatabases()) { - if (CacheUtils.matches(dbName, dbNames)) { - for (Table table : listCachedTables(dbName)) { - if (CacheUtils.matches(table.getTableName(), tableNames)) { - if (tableTypes==null || tableTypes.contains(table.getTableType())) { - TableMeta metaData = new TableMeta( - dbName, table.getTableName(), table.getTableType()); + try { + cacheLock.readLock().lock(); + for (String dbName : listCachedDatabases()) { + if (CacheUtils.matches(dbName, dbNames)) { + for (Table table : listCachedTables(dbName)) { + if (CacheUtils.matches(table.getTableName(), tableNames)) { + if (tableTypes == null || tableTypes.contains(table.getTableType())) { + TableMeta metaData = + new TableMeta(dbName, table.getTableName(), table.getTableType()); metaData.setComments(table.getParameters().get("comment")); tableMetas.add(metaData); + } } } } } + } finally { + cacheLock.readLock().unlock(); } return tableMetas; } - public synchronized void addPartitionToCache(String dbName, String tblName, Partition part) { - Partition partCopy = part.deepCopy(); - PartitionWrapper wrapper; - if (part.getSd()!=null) { - byte[] sdHash = MetaStoreUtils.hashStorageDescriptor(part.getSd(), md); - StorageDescriptor sd = part.getSd(); - increSd(sd, sdHash); - partCopy.setSd(null); - wrapper = new PartitionWrapper(partCopy, sdHash, sd.getLocation(), sd.getParameters()); - } else { - wrapper = new PartitionWrapper(partCopy, null, null, null); + public void addPartitionToCache(String dbName, String tblName, Partition part) { + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + tblWrapper.cachePartition(part, this); + } + } finally { + cacheLock.readLock().unlock(); } - partitionCache.put(CacheUtils.buildKey(dbName, tblName, part.getValues()), wrapper); } - public synchronized Partition getPartitionFromCache(String key) { - PartitionWrapper wrapper = partitionCache.get(key); - if (wrapper == null) { - return null; + public void addPartitionsToCache(String dbName, String tblName, List parts) { + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + tblWrapper.cachePartitions(parts, this); + } + } finally { + cacheLock.readLock().unlock(); } - Partition p = CacheUtils.assemble(wrapper, this); - return p; } - public synchronized Partition getPartitionFromCache(String dbName, String tblName, List part_vals) { - return getPartitionFromCache(CacheUtils.buildKey(dbName, tblName, part_vals)); - } - - public synchronized boolean existPartitionFromCache(String dbName, String tblName, List part_vals) { - return partitionCache.containsKey(CacheUtils.buildKey(dbName, tblName, part_vals)); - } - - public synchronized Partition removePartitionFromCache(String dbName, String tblName, - List part_vals) { - PartitionWrapper wrapper = - partitionCache.remove(CacheUtils.buildKey(dbName, tblName, part_vals)); - if (wrapper.getSdHash() != null) { - decrSd(wrapper.getSdHash()); + public Partition getPartitionFromCache(String dbName, String tblName, + List partVals) { + Partition part = null; + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + part = tblWrapper.getPartition(partVals, this); + } + } finally { + cacheLock.readLock().unlock(); } - return wrapper.getPartition(); + return part; } - /** - * Given a db + table, remove all partitions for this table from the cache - * @param dbName - * @param tblName - * @return - */ - public synchronized void removePartitionsFromCache(String dbName, String tblName) { - String partialKey = CacheUtils.buildKeyWithDelimit(dbName, tblName); - Iterator> iterator = partitionCache.entrySet().iterator(); - while (iterator.hasNext()) { - Entry entry = iterator.next(); - String key = entry.getKey(); - PartitionWrapper wrapper = entry.getValue(); - if (key.toLowerCase().startsWith(partialKey.toLowerCase())) { - iterator.remove(); - if (wrapper.getSdHash() != null) { - decrSd(wrapper.getSdHash()); - } + public boolean existPartitionFromCache(String dbName, String tblName, List partVals) { + boolean existsPart = false; + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + existsPart = tblWrapper.containsPartition(partVals); } + } finally { + cacheLock.readLock().unlock(); } + return existsPart; } - // Remove cached column stats for all partitions of all tables in a db - public synchronized void removePartitionColStatsFromCache(String dbName) { - String partialKey = CacheUtils.buildKeyWithDelimit(dbName); - Iterator> iterator = - partitionColStatsCache.entrySet().iterator(); - while (iterator.hasNext()) { - Entry entry = iterator.next(); - String key = entry.getKey(); - if (key.toLowerCase().startsWith(partialKey.toLowerCase())) { - iterator.remove(); + public Partition removePartitionFromCache(String dbName, String tblName, + List partVals) { + Partition part = null; + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + part = tblWrapper.removePartition(partVals, this); } + } finally { + cacheLock.readLock().unlock(); } + return part; } - // Remove cached column stats for all partitions of a table - public synchronized void removePartitionColStatsFromCache(String dbName, String tblName) { - String partialKey = CacheUtils.buildKeyWithDelimit(dbName, tblName); - Iterator> iterator = - partitionColStatsCache.entrySet().iterator(); - while (iterator.hasNext()) { - Entry entry = iterator.next(); - String key = entry.getKey(); - if (key.toLowerCase().startsWith(partialKey.toLowerCase())) { - iterator.remove(); + public void removePartitionsFromCache(String dbName, String tblName, + List> partVals) { + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + tblWrapper.removePartitions(partVals, this); } + } finally { + cacheLock.readLock().unlock(); } } - // Remove cached column stats for a particular partition of a table - public synchronized void removePartitionColStatsFromCache(String dbName, String tblName, - List partVals) { - String partialKey = CacheUtils.buildKeyWithDelimit(dbName, tblName, partVals); - Iterator> iterator = - partitionColStatsCache.entrySet().iterator(); - while (iterator.hasNext()) { - Entry entry = iterator.next(); - String key = entry.getKey(); - if (key.toLowerCase().startsWith(partialKey.toLowerCase())) { - iterator.remove(); + public List listCachedPartitions(String dbName, String tblName, int max) { + List parts = new ArrayList(); + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + parts = tblWrapper.listPartitions(max, this); } + } finally { + cacheLock.readLock().unlock(); } + return parts; } - // Remove cached column stats for a particular partition and a particular column of a table - public synchronized void removePartitionColStatsFromCache(String dbName, String tblName, - List partVals, String colName) { - partitionColStatsCache.remove(CacheUtils.buildKey(dbName, tblName, partVals, colName)); + public void alterPartitionInCache(String dbName, String tblName, List partVals, + Partition newPart) { + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + tblWrapper.alterPartition(partVals, newPart, this); + } + } finally { + cacheLock.readLock().unlock(); + } } - public synchronized List listCachedPartitions(String dbName, String tblName, int max) { - List partitions = new ArrayList<>(); - int count = 0; - for (PartitionWrapper wrapper : partitionCache.values()) { - if (wrapper.getPartition().getDbName().equals(dbName) - && wrapper.getPartition().getTableName().equals(tblName) - && (max == -1 || count < max)) { - partitions.add(CacheUtils.assemble(wrapper, this)); - count++; + public void alterPartitionsInCache(String dbName, String tblName, List> partValsList, + List newParts) { + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + tblWrapper.alterPartitions(partValsList, newParts, this); } + } finally { + cacheLock.readLock().unlock(); } - return partitions; } - public synchronized void alterPartitionInCache(String dbName, String tblName, - List partVals, Partition newPart) { - removePartitionFromCache(dbName, tblName, partVals); - addPartitionToCache(StringUtils.normalizeIdentifier(newPart.getDbName()), - StringUtils.normalizeIdentifier(newPart.getTableName()), newPart); + public void refreshPartitionsInCache(String dbName, String tblName, List partitions) { + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + tblWrapper.refreshPartitions(partitions, this); + } + } finally { + cacheLock.readLock().unlock(); + } } - public synchronized void alterPartitionInColStatsCache(String dbName, String tblName, - List partVals, Partition newPart) { - String oldPartialPartitionKey = CacheUtils.buildKeyWithDelimit(dbName, tblName, partVals); - Map newPartitionColStats = new HashMap<>(); - Iterator> iterator = - partitionColStatsCache.entrySet().iterator(); - while (iterator.hasNext()) { - Entry entry = iterator.next(); - String key = entry.getKey(); - ColumnStatisticsObj colStatObj = entry.getValue(); - if (key.toLowerCase().startsWith(oldPartialPartitionKey.toLowerCase())) { - Object[] decomposedKey = CacheUtils.splitPartitionColStats(key); - String newKey = - CacheUtils.buildKey(StringUtils.normalizeIdentifier(newPart.getDbName()), - StringUtils.normalizeIdentifier(newPart.getTableName()), newPart.getValues(), - (String) decomposedKey[3]); - newPartitionColStats.put(newKey, colStatObj); - iterator.remove(); + public void removePartitionColStatsFromCache(String dbName, String tblName, + List partVals, String colName) { + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + tblWrapper.removePartitionColStats(partVals, colName); } + } finally { + cacheLock.readLock().unlock(); } - partitionColStatsCache.putAll(newPartitionColStats); } - public synchronized void updatePartitionColStatsInCache(String dbName, String tableName, + public void updatePartitionColStatsInCache(String dbName, String tableName, List partVals, List colStatsObjs) { - for (ColumnStatisticsObj colStatObj : colStatsObjs) { - // Get old stats object if present - String key = CacheUtils.buildKey(dbName, tableName, partVals, colStatObj.getColName()); - ColumnStatisticsObj oldStatsObj = partitionColStatsCache.get(key); - if (oldStatsObj != null) { - // Update existing stat object's field - LOG.debug("CachedStore: updating partition column stats for column: " - + colStatObj.getColName() + ", of table: " + tableName + " and database: " + dbName); - StatObjectConverter.setFieldsIntoOldStats(oldStatsObj, colStatObj); - } else { - // No stats exist for this key; add a new object to the cache - partitionColStatsCache.put(key, colStatObj); + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tableName)); + if (tblWrapper != null) { + tblWrapper.updatePartitionColStats(partVals, colStatsObjs); } + } finally { + cacheLock.readLock().unlock(); } } - public synchronized int getCachedPartitionCount() { - return partitionCache.size(); - } - - public synchronized ColumnStatisticsObj getCachedPartitionColStats(String key) { - return partitionColStatsCache.get(key)!=null?partitionColStatsCache.get(key).deepCopy():null; - } - - public synchronized void addPartitionColStatsToCache( - List colStatsForDB) { - for (ColStatsObjWithSourceInfo colStatWithSourceInfo : colStatsForDB) { - List partVals; - try { - partVals = Warehouse.getPartValuesFromPartName(colStatWithSourceInfo.getPartName()); - ColumnStatisticsObj colStatObj = colStatWithSourceInfo.getColStatsObj(); - String key = CacheUtils.buildKey(colStatWithSourceInfo.getDbName(), - colStatWithSourceInfo.getTblName(), partVals, colStatObj.getColName()); - partitionColStatsCache.put(key, colStatObj); - } catch (MetaException e) { - LOG.info("Unable to add partition stats for: {} to SharedCache", - colStatWithSourceInfo.getPartName(), e); + public ColumnStatisticsObj getPartitionColStatsFromCache(String dbName, String tblName, + List partVal, String colName) { + ColumnStatisticsObj colStatObj = null; + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null){ + colStatObj = tblWrapper.getPartitionColStats(partVal, colName); } + } finally { + cacheLock.readLock().unlock(); } - + return colStatObj; } - public synchronized void refreshPartitionColStats(String dbName, - List colStatsForDB) { - LOG.debug("CachedStore: updating cached partition column stats objects for database: {}", - dbName); - removePartitionColStatsFromCache(dbName); - addPartitionColStatsToCache(colStatsForDB); - } - - public synchronized void addAggregateStatsToCache(String dbName, String tblName, - AggrStats aggrStatsAllPartitions, AggrStats aggrStatsAllButDefaultPartition) { - if (aggrStatsAllPartitions != null) { - for (ColumnStatisticsObj colStatObj : aggrStatsAllPartitions.getColStats()) { - String key = CacheUtils.buildKey(dbName, tblName, colStatObj.getColName()); - List value = new ArrayList(); - value.add(StatsType.ALL.getPosition(), colStatObj); - aggrColStatsCache.put(key, value); - } - } - if (aggrStatsAllButDefaultPartition != null) { - for (ColumnStatisticsObj colStatObj : aggrStatsAllButDefaultPartition.getColStats()) { - String key = CacheUtils.buildKey(dbName, tblName, colStatObj.getColName()); - List value = aggrColStatsCache.get(key); - if ((value != null) && (value.size() > 0)) { - value.add(StatsType.ALLBUTDEFAULT.getPosition(), colStatObj); - } + public void refreshPartitionColStatsInCache(String dbName, String tblName, + List partitionColStats) { + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + tblWrapper.refreshPartitionColStats(partitionColStats); } + } finally { + cacheLock.readLock().unlock(); } } public List getAggrStatsFromCache(String dbName, String tblName, List colNames, StatsType statsType) { - List colStats = new ArrayList(); - for (String colName : colNames) { - String key = CacheUtils.buildKey(dbName, tblName, colName); - List colStatList = aggrColStatsCache.get(key); - // If unable to find stats for a column, return null so we can build stats - if (colStatList == null) { - return null; - } - ColumnStatisticsObj colStatObj = colStatList.get(statsType.getPosition()); - // If unable to find stats for this StatsType, return null so we can build - // stats - if (colStatObj == null) { - return null; + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + return tblWrapper.getAggrPartitionColStats(colNames, statsType); } - colStats.add(colStatObj); + } finally { + cacheLock.readLock().unlock(); } - return colStats; + return null; } - public synchronized void removeAggrPartitionColStatsFromCache(String dbName, String tblName) { - String partialKey = CacheUtils.buildKeyWithDelimit(dbName, tblName); - Iterator>> iterator = - aggrColStatsCache.entrySet().iterator(); - while (iterator.hasNext()) { - Entry> entry = iterator.next(); - String key = entry.getKey(); - if (key.toLowerCase().startsWith(partialKey.toLowerCase())) { - iterator.remove(); + public void addAggregateStatsToCache(String dbName, String tblName, + AggrStats aggrStatsAllPartitions, AggrStats aggrStatsAllButDefaultPartition) { + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null){ + tblWrapper.cacheAggrPartitionColStats(aggrStatsAllPartitions, + aggrStatsAllButDefaultPartition); } + } finally { + cacheLock.readLock().unlock(); } } - public synchronized void refreshAggregateStatsCache(String dbName, String tblName, + public void refreshAggregateStatsInCache(String dbName, String tblName, AggrStats aggrStatsAllPartitions, AggrStats aggrStatsAllButDefaultPartition) { - LOG.debug("CachedStore: updating aggregate stats cache for database: {}, table: {}", dbName, - tblName); - removeAggrPartitionColStatsFromCache(dbName, tblName); - addAggregateStatsToCache(dbName, tblName, aggrStatsAllPartitions, - aggrStatsAllButDefaultPartition); - } - - public synchronized void addTableColStatsToCache(String dbName, String tableName, - List colStatsForTable) { - for (ColumnStatisticsObj colStatObj : colStatsForTable) { - String key = CacheUtils.buildKey(dbName, tableName, colStatObj.getColName()); - tableColStatsCache.put(key, colStatObj); + try { + cacheLock.readLock().lock(); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildKey(dbName, tblName)); + if (tblWrapper != null) { + tblWrapper.refreshAggrPartitionColStats(aggrStatsAllPartitions, + aggrStatsAllButDefaultPartition); + } + } finally { + cacheLock.readLock().unlock(); } } - public synchronized void refreshTableColStats(String dbName, String tableName, - List colStatsForTable) { - LOG.debug("CachedStore: updating cached table column stats objects for database: " + dbName - + " and table: " + tableName); - // Remove all old cache entries for this table - removeTableColStatsFromCache(dbName, tableName); - // Add new entries to cache - addTableColStatsToCache(dbName, tableName, colStatsForTable); - } - public void increSd(StorageDescriptor sd, byte[] sdHash) { ByteArrayWrapper byteArray = new ByteArrayWrapper(sdHash); if (sdCache.containsKey(byteArray)) { @@ -640,45 +1307,6 @@ public StorageDescriptor getSdFromCache(byte[] sdHash) { return sdWrapper.getSd(); } - // Replace databases in databaseCache with the new list - public synchronized void refreshDatabases(List databases) { - LOG.debug("CachedStore: updating cached database objects"); - for (String dbName : listCachedDatabases()) { - removeDatabaseFromCache(dbName); - } - for (Database db : databases) { - addDatabaseToCache(db.getName(), db); - } - } - - // Replace tables in tableCache with the new list - public synchronized void refreshTables(String dbName, List
tables) { - LOG.debug("CachedStore: updating cached table objects for database: " + dbName); - for (Table tbl : listCachedTables(dbName)) { - removeTableFromCache(dbName, tbl.getTableName()); - } - for (Table tbl : tables) { - addTableToCache(dbName, tbl.getTableName(), tbl); - } - } - - public synchronized void refreshPartitions(String dbName, String tblName, - List partitions) { - LOG.debug("CachedStore: updating cached partition objects for database: " + dbName - + " and table: " + tblName); - Iterator> iterator = partitionCache.entrySet().iterator(); - while (iterator.hasNext()) { - PartitionWrapper partitionWrapper = iterator.next().getValue(); - if (partitionWrapper.getPartition().getDbName().equals(dbName) - && partitionWrapper.getPartition().getTableName().equals(tblName)) { - iterator.remove(); - } - } - for (Partition part : partitions) { - addPartitionToCache(dbName, tblName, part); - } - } - @VisibleForTesting Map getDatabaseCache() { return databaseCache; @@ -690,17 +1318,15 @@ public synchronized void refreshPartitions(String dbName, String tblName, } @VisibleForTesting - Map getPartitionCache() { - return partitionCache; - } - - @VisibleForTesting Map getSdCache() { return sdCache; } - @VisibleForTesting - Map getPartitionColStatsCache() { - return partitionColStatsCache; + public long getUpdateCount() { + return cacheUpdateCount.get(); + } + + public void incrementUpdateCount() { + cacheUpdateCount.incrementAndGet(); } } diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 50f873a013..83a4b808ea 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -433,6 +433,15 @@ private static String getPartitionValWithInvalidCharacter(List partVals, return colNames; } + public static List getColumnNamesForPartition(Partition partition) { + List colNames = new ArrayList<>(); + Iterator colsIterator = partition.getSd().getColsIterator(); + while (colsIterator.hasNext()) { + colNames.add(colsIterator.next().getName()); + } + return colNames; + } + /** * validateName * diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 75ea8c4a77..1a24cc9d42 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -1060,11 +1060,4 @@ public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerNa String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { objectStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); } - - @Override - public List getPartitionColStatsForDatabase(String dbName) - throws MetaException, NoSuchObjectException { - // TODO Auto-generated method stub - return null; - } } diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 207d842f94..aeb5d3330d 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -1056,11 +1056,4 @@ public void createWMTriggerToPoolMapping(String resourcePlanName, String trigger public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { } - - @Override - public List getPartitionColStatsForDatabase(String dbName) - throws MetaException, NoSuchObjectException { - // TODO Auto-generated method stub - return null; - } } diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java index ab6feb6f0b..5234ca2349 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java @@ -22,16 +22,17 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.TableType; -import org.apache.hadoop.hive.metastore.TestObjectStore.MockPartitionExpressionProxy; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; import org.apache.hadoop.hive.metastore.api.AggrStats; -import org.apache.hadoop.hive.metastore.api.BasicTxnInfo; import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; @@ -39,6 +40,9 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.InvalidInputException; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PrincipalType; @@ -68,19 +72,13 @@ public void setUp() throws Exception { objectStore = new ObjectStore(); objectStore.setConf(conf); cachedStore = new CachedStore(); - cachedStore.setConf(conf); - // Stop the CachedStore cache update service. We'll start it explicitly to control the test - CachedStore.stopCacheUpdateService(1); - cachedStore.setInitializedForTest(); - + cachedStore.setConfForTest(conf); // Stop the CachedStore cache update service. We'll start it explicitly to control the test CachedStore.stopCacheUpdateService(1); sharedCache = new SharedCache(); sharedCache.getDatabaseCache().clear(); sharedCache.getTableCache().clear(); - sharedCache.getPartitionCache().clear(); sharedCache.getSdCache().clear(); - sharedCache.getPartitionColStatsCache().clear(); } /********************************************************************************************** @@ -91,61 +89,48 @@ public void setUp() throws Exception { public void testDatabaseOps() throws Exception { // Add a db via ObjectStore String dbName = "testDatabaseOps"; - String dbDescription = "testDatabaseOps"; - String dbLocation = "file:/tmp"; - Map dbParams = new HashMap<>(); String dbOwner = "user1"; - Database db = new Database(dbName, dbDescription, dbLocation, dbParams); - db.setOwnerName(dbOwner); - db.setOwnerType(PrincipalType.USER); + Database db = createTestDb(dbName, dbOwner); objectStore.createDatabase(db); db = objectStore.getDatabase(dbName); // Prewarm CachedStore CachedStore.prewarm(objectStore); // Read database via CachedStore - Database dbNew = cachedStore.getDatabase(dbName); - Assert.assertEquals(db, dbNew); + Database dbRead = cachedStore.getDatabase(dbName); + Assert.assertEquals(db, dbRead); // Add another db via CachedStore final String dbName1 = "testDatabaseOps1"; - final String dbDescription1 = "testDatabaseOps1"; - Database db1 = new Database(dbName1, dbDescription1, dbLocation, dbParams); - db1.setOwnerName(dbOwner); - db1.setOwnerType(PrincipalType.USER); + Database db1 = createTestDb(dbName1, dbOwner); cachedStore.createDatabase(db1); db1 = cachedStore.getDatabase(dbName1); // Read db via ObjectStore - dbNew = objectStore.getDatabase(dbName1); - Assert.assertEquals(db1, dbNew); + dbRead = objectStore.getDatabase(dbName1); + Assert.assertEquals(db1, dbRead); // Alter the db via CachedStore (can only alter owner or parameters) - db = new Database(dbName, dbDescription, dbLocation, dbParams); dbOwner = "user2"; + db = new Database(db); db.setOwnerName(dbOwner); - db.setOwnerType(PrincipalType.USER); cachedStore.alterDatabase(dbName, db); db = cachedStore.getDatabase(dbName); // Read db via ObjectStore - dbNew = objectStore.getDatabase(dbName); - Assert.assertEquals(db, dbNew); + dbRead = objectStore.getDatabase(dbName); + Assert.assertEquals(db, dbRead); // Add another db via ObjectStore final String dbName2 = "testDatabaseOps2"; - final String dbDescription2 = "testDatabaseOps2"; - Database db2 = new Database(dbName2, dbDescription2, dbLocation, dbParams); - db2.setOwnerName(dbOwner); - db2.setOwnerType(PrincipalType.USER); + Database db2 = createTestDb(dbName2, dbOwner); objectStore.createDatabase(db2); db2 = objectStore.getDatabase(dbName2); // Alter db "testDatabaseOps" via ObjectStore dbOwner = "user1"; - db = new Database(dbName, dbDescription, dbLocation, dbParams); + db = new Database(db); db.setOwnerName(dbOwner); - db.setOwnerType(PrincipalType.USER); objectStore.alterDatabase(dbName, db); db = objectStore.getDatabase(dbName); @@ -153,20 +138,20 @@ public void testDatabaseOps() throws Exception { objectStore.dropDatabase(dbName1); // We update twice to accurately detect if cache is dirty or not - updateCache(cachedStore, 100, 500, 100); - updateCache(cachedStore, 100, 500, 100); + updateCache(cachedStore); + updateCache(cachedStore); // Read the newly added db via CachedStore - dbNew = cachedStore.getDatabase(dbName2); - Assert.assertEquals(db2, dbNew); + dbRead = cachedStore.getDatabase(dbName2); + Assert.assertEquals(db2, dbRead); // Read the altered db via CachedStore (altered user from "user2" to "user1") - dbNew = cachedStore.getDatabase(dbName); - Assert.assertEquals(db, dbNew); + dbRead = cachedStore.getDatabase(dbName); + Assert.assertEquals(db, dbRead); // Try to read the dropped db after cache update try { - dbNew = cachedStore.getDatabase(dbName1); + dbRead = cachedStore.getDatabase(dbName1); Assert.fail("The database: " + dbName1 + " should have been removed from the cache after running the update service"); } catch (NoSuchObjectException e) { @@ -176,41 +161,30 @@ public void testDatabaseOps() throws Exception { // Clean up objectStore.dropDatabase(dbName); objectStore.dropDatabase(dbName2); + sharedCache.getDatabaseCache().clear(); + sharedCache.getTableCache().clear(); + sharedCache.getSdCache().clear(); } @Test public void testTableOps() throws Exception { // Add a db via ObjectStore String dbName = "testTableOps"; - String dbDescription = "testTableOps"; - String dbLocation = "file:/tmp"; - Map dbParams = new HashMap<>(); String dbOwner = "user1"; - Database db = new Database(dbName, dbDescription, dbLocation, dbParams); - db.setOwnerName(dbOwner); - db.setOwnerType(PrincipalType.USER); + Database db = createTestDb(dbName, dbOwner); objectStore.createDatabase(db); db = objectStore.getDatabase(dbName); // Add a table via ObjectStore String tblName = "tbl"; String tblOwner = "user1"; - String serdeLocation = "file:/tmp"; FieldSchema col1 = new FieldSchema("col1", "int", "integer column"); FieldSchema col2 = new FieldSchema("col2", "string", "string column"); - List cols = new ArrayList<>(); + List cols = new ArrayList(); cols.add(col1); cols.add(col2); - Map serdeParams = new HashMap<>(); - Map tblParams = new HashMap<>(); - SerDeInfo serdeInfo = new SerDeInfo("serde", "seriallib", new HashMap<>()); - StorageDescriptor sd = - new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serdeInfo, null, - null, serdeParams); - sd.setStoredAsSubDirectories(false); - Table tbl = - new Table(tblName, dbName, tblOwner, 0, 0, 0, sd, new ArrayList<>(), tblParams, - null, null, TableType.MANAGED_TABLE.toString()); + List ptnCols = new ArrayList(); + Table tbl = createTestTbl(dbName, tblName, tblOwner, cols, ptnCols); objectStore.createTable(tbl); tbl = objectStore.getTable(dbName, tblName); @@ -218,36 +192,32 @@ public void testTableOps() throws Exception { CachedStore.prewarm(objectStore); // Read database, table via CachedStore - Database dbNew = cachedStore.getDatabase(dbName); - Assert.assertEquals(db, dbNew); - Table tblNew = cachedStore.getTable(dbName, tblName); - Assert.assertEquals(tbl, tblNew); + Database dbRead= cachedStore.getDatabase(dbName); + Assert.assertEquals(db, dbRead); + Table tblRead = cachedStore.getTable(dbName, tblName); + Assert.assertEquals(tbl, tblRead); // Add a new table via CachedStore String tblName1 = "tbl1"; - Table tbl1 = - new Table(tblName1, dbName, tblOwner, 0, 0, 0, sd, new ArrayList<>(), tblParams, - null, null, TableType.MANAGED_TABLE.toString()); + Table tbl1 = new Table(tbl); + tbl1.setTableName(tblName1); cachedStore.createTable(tbl1); tbl1 = cachedStore.getTable(dbName, tblName1); // Read via object store - tblNew = objectStore.getTable(dbName, tblName1); - Assert.assertEquals(tbl1, tblNew); + tblRead = objectStore.getTable(dbName, tblName1); + Assert.assertEquals(tbl1, tblRead); // Add a new table via ObjectStore String tblName2 = "tbl2"; - Table tbl2 = - new Table(tblName2, dbName, tblOwner, 0, 0, 0, sd, new ArrayList<>(), tblParams, - null, null, TableType.MANAGED_TABLE.toString()); + Table tbl2 = new Table(tbl); + tbl2.setTableName(tblName2); objectStore.createTable(tbl2); tbl2 = objectStore.getTable(dbName, tblName2); // Alter table "tbl" via ObjectStore tblOwner = "user2"; - tbl = - new Table(tblName, dbName, tblOwner, 0, 0, 0, sd, new ArrayList<>(), tblParams, - null, null, TableType.MANAGED_TABLE.toString()); + tbl.setOwner(tblOwner); objectStore.alterTable(dbName, tblName, tbl); tbl = objectStore.getTable(dbName, tblName); @@ -255,20 +225,20 @@ public void testTableOps() throws Exception { objectStore.dropTable(dbName, tblName1); // We update twice to accurately detect if cache is dirty or not - updateCache(cachedStore, 100, 500, 100); - updateCache(cachedStore, 100, 500, 100); + updateCache(cachedStore); + updateCache(cachedStore); // Read "tbl2" via CachedStore - tblNew = cachedStore.getTable(dbName, tblName2); - Assert.assertEquals(tbl2, tblNew); + tblRead = cachedStore.getTable(dbName, tblName2); + Assert.assertEquals(tbl2, tblRead); // Read the altered "tbl" via CachedStore - tblNew = cachedStore.getTable(dbName, tblName); - Assert.assertEquals(tbl, tblNew); + tblRead = cachedStore.getTable(dbName, tblName); + Assert.assertEquals(tbl, tblRead); // Try to read the dropped "tbl1" via CachedStore (should throw exception) - tblNew = cachedStore.getTable(dbName, tblName1); - Assert.assertNull(tblNew); + tblRead = cachedStore.getTable(dbName, tblName1); + Assert.assertNull(tblRead); // Should return "tbl" and "tbl2" List tblNames = cachedStore.getTables(dbName, "*"); @@ -280,54 +250,44 @@ public void testTableOps() throws Exception { objectStore.dropTable(dbName, tblName); objectStore.dropTable(dbName, tblName2); objectStore.dropDatabase(dbName); + sharedCache.getDatabaseCache().clear(); + sharedCache.getTableCache().clear(); + sharedCache.getSdCache().clear(); } @Test public void testPartitionOps() throws Exception { // Add a db via ObjectStore String dbName = "testPartitionOps"; - String dbDescription = "testPartitionOps"; - String dbLocation = "file:/tmp"; - Map dbParams = new HashMap<>(); String dbOwner = "user1"; - Database db = new Database(dbName, dbDescription, dbLocation, dbParams); - db.setOwnerName(dbOwner); - db.setOwnerType(PrincipalType.USER); + Database db = createTestDb(dbName, dbOwner); objectStore.createDatabase(db); db = objectStore.getDatabase(dbName); // Add a table via ObjectStore String tblName = "tbl"; String tblOwner = "user1"; - String serdeLocation = "file:/tmp"; FieldSchema col1 = new FieldSchema("col1", "int", "integer column"); FieldSchema col2 = new FieldSchema("col2", "string", "string column"); - List cols = new ArrayList<>(); + List cols = new ArrayList(); cols.add(col1); cols.add(col2); - Map serdeParams = new HashMap<>(); - Map tblParams = new HashMap<>(); - SerDeInfo serdeInfo = new SerDeInfo("serde", "seriallib", null); - StorageDescriptor sd = - new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serdeInfo, null, - null, serdeParams); FieldSchema ptnCol1 = new FieldSchema("part1", "string", "string partition column"); - List ptnCols = new ArrayList<>(); + List ptnCols = new ArrayList(); ptnCols.add(ptnCol1); - Table tbl = - new Table(tblName, dbName, tblOwner, 0, 0, 0, sd, ptnCols, tblParams, null, null, - TableType.MANAGED_TABLE.toString()); + Table tbl = createTestTbl(dbName, tblName, tblOwner, cols, ptnCols); objectStore.createTable(tbl); tbl = objectStore.getTable(dbName, tblName); + final String ptnColVal1 = "aaa"; - Map partParams = new HashMap<>(); + Map partParams = new HashMap(); Partition ptn1 = - new Partition(Arrays.asList(ptnColVal1), dbName, tblName, 0, 0, sd, partParams); + new Partition(Arrays.asList(ptnColVal1), dbName, tblName, 0, 0, tbl.getSd(), partParams); objectStore.addPartition(ptn1); ptn1 = objectStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal1)); final String ptnColVal2 = "bbb"; Partition ptn2 = - new Partition(Arrays.asList(ptnColVal2), dbName, tblName, 0, 0, sd, partParams); + new Partition(Arrays.asList(ptnColVal2), dbName, tblName, 0, 0, tbl.getSd(), partParams); objectStore.addPartition(ptn2); ptn2 = objectStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal2)); @@ -335,26 +295,26 @@ public void testPartitionOps() throws Exception { CachedStore.prewarm(objectStore); // Read database, table, partition via CachedStore - Database dbNew = cachedStore.getDatabase(dbName); - Assert.assertEquals(db, dbNew); - Table tblNew = cachedStore.getTable(dbName, tblName); - Assert.assertEquals(tbl, tblNew); - Partition newPtn1 = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal1)); - Assert.assertEquals(ptn1, newPtn1); - Partition newPtn2 = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal2)); - Assert.assertEquals(ptn2, newPtn2); + Database dbRead = cachedStore.getDatabase(dbName); + Assert.assertEquals(db, dbRead); + Table tblRead = cachedStore.getTable(dbName, tblName); + Assert.assertEquals(tbl, tblRead); + Partition ptn1Read = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal1)); + Assert.assertEquals(ptn1, ptn1Read); + Partition ptn2Read = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal2)); + Assert.assertEquals(ptn2, ptn2Read); // Add a new partition via ObjectStore final String ptnColVal3 = "ccc"; Partition ptn3 = - new Partition(Arrays.asList(ptnColVal3), dbName, tblName, 0, 0, sd, partParams); + new Partition(Arrays.asList(ptnColVal3), dbName, tblName, 0, 0, tbl.getSd(), partParams); objectStore.addPartition(ptn3); ptn3 = objectStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal3)); // Alter an existing partition ("aaa") via ObjectStore final String ptnColVal1Alt = "aaaAlt"; Partition ptn1Atl = - new Partition(Arrays.asList(ptnColVal1Alt), dbName, tblName, 0, 0, sd, partParams); + new Partition(Arrays.asList(ptnColVal1Alt), dbName, tblName, 0, 0, tbl.getSd(), partParams); objectStore.alterPartition(dbName, tblName, Arrays.asList(ptnColVal1), ptn1Atl); ptn1Atl = objectStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal1Alt)); @@ -362,45 +322,47 @@ public void testPartitionOps() throws Exception { objectStore.dropPartition(dbName, tblName, Arrays.asList(ptnColVal2)); // We update twice to accurately detect if cache is dirty or not - updateCache(cachedStore, 100, 500, 100); - updateCache(cachedStore, 100, 500, 100); + updateCache(cachedStore); + updateCache(cachedStore); // Read the newly added partition via CachedStore - Partition newPtn = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal3)); - Assert.assertEquals(ptn3, newPtn); + Partition ptnRead = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal3)); + Assert.assertEquals(ptn3, ptnRead); // Read the altered partition via CachedStore - newPtn = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal1Alt)); - Assert.assertEquals(ptn1Atl, newPtn); + ptnRead = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal1Alt)); + Assert.assertEquals(ptn1Atl, ptnRead); // Try to read the dropped partition via CachedStore try { - newPtn = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal2)); + ptnRead = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal2)); Assert.fail("The partition: " + ptnColVal2 + " should have been removed from the cache after running the update service"); } catch (NoSuchObjectException e) { // Expected } + // Clean up + objectStore.dropPartition(dbName, tblName, Arrays.asList(ptnColVal1Alt)); + objectStore.dropPartition(dbName, tblName, Arrays.asList(ptnColVal3)); + objectStore.dropTable(dbName, tblName); + objectStore.dropDatabase(dbName); + sharedCache.getDatabaseCache().clear(); + sharedCache.getTableCache().clear(); + sharedCache.getSdCache().clear(); } //@Test public void testTableColStatsOps() throws Exception { // Add a db via ObjectStore String dbName = "testTableColStatsOps"; - String dbDescription = "testTableColStatsOps"; - String dbLocation = "file:/tmp"; - Map dbParams = new HashMap<>(); String dbOwner = "user1"; - Database db = new Database(dbName, dbDescription, dbLocation, dbParams); - db.setOwnerName(dbOwner); - db.setOwnerType(PrincipalType.USER); + Database db = createTestDb(dbName, dbOwner); objectStore.createDatabase(db); db = objectStore.getDatabase(dbName); // Add a table via ObjectStore final String tblName = "tbl"; final String tblOwner = "user1"; - final String serdeLocation = "file:/tmp"; final FieldSchema col1 = new FieldSchema("col1", "int", "integer column"); // Stats values for col1 long col1LowVal = 5; @@ -422,15 +384,10 @@ public void testTableColStatsOps() throws Exception { cols.add(col1); cols.add(col2); cols.add(col3); - Map serdeParams = new HashMap<>(); - Map tblParams = new HashMap<>(); - final SerDeInfo serdeInfo = new SerDeInfo("serde", "seriallib", null); - StorageDescriptor sd = - new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, serdeInfo, null, - null, serdeParams); - Table tbl = - new Table(tblName, dbName, tblOwner, 0, 0, 0, sd, new ArrayList<>(), tblParams, - null, null, TableType.MANAGED_TABLE.toString()); + FieldSchema ptnCol1 = new FieldSchema("part1", "string", "string partition column"); + List ptnCols = new ArrayList(); + ptnCols.add(ptnCol1); + Table tbl = createTestTbl(dbName, tblName, tblOwner, cols, ptnCols); objectStore.createTable(tbl); tbl = objectStore.getTable(dbName, tblName); @@ -485,17 +442,213 @@ public void testTableColStatsOps() throws Exception { cachedStore.getTableColumnStatistics(dbName, tblName, Arrays.asList(col1.getName(), col2.getName(), col3.getName())); Assert.assertEquals(stats, newStats); + + // Clean up + objectStore.dropTable(dbName, tblName); + objectStore.dropDatabase(dbName); + sharedCache.getDatabaseCache().clear(); + sharedCache.getTableCache().clear(); + sharedCache.getSdCache().clear(); + } + + @Test + public void testMultiThreadedOps() throws Exception { + List dbNames = new ArrayList(Arrays.asList("db1", "db2", "db3", "db4", "db5")); + List> tasks = new ArrayList>(); + ExecutorService executor = Executors.newFixedThreadPool(10, new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + Thread t = Executors.defaultThreadFactory().newThread(r); + t.setDaemon(true); + return t; + } + }); + + // Create 5 dbs + for (String dbName : dbNames) { + Callable c = new Callable() { + public Object call() { + Database db = createTestDb(dbName, "user1"); + try { + cachedStore.createDatabase(db); + } catch (InvalidObjectException | MetaException e) { + e.printStackTrace(); + System.out.println(e); + } + return null; + } + }; + tasks.add(c); + } + executor.invokeAll(tasks); + for (String dbName : dbNames) { + // Read database via ObjectStore + Database dbOS = objectStore.getDatabase(dbName); + // Read database via CachedStore + Database dbCS = cachedStore.getDatabase(dbName); + Assert.assertEquals(dbOS, dbCS); + } + + // Created 5 tables under "db1" + List tblNames = + new ArrayList(Arrays.asList("tbl1", "tbl2", "tbl3", "tbl4", "tbl5")); + tasks.clear(); + for (String tblName : tblNames) { + FieldSchema col1 = new FieldSchema("col1", "int", "integer column"); + FieldSchema col2 = new FieldSchema("col2", "string", "string column"); + List cols = new ArrayList(); + cols.add(col1); + cols.add(col2); + FieldSchema ptnCol1 = new FieldSchema("part1", "string", "string partition column"); + List ptnCols = new ArrayList(); + ptnCols.add(ptnCol1); + Callable c = new Callable() { + public Object call() { + Table tbl = createTestTbl(dbNames.get(0), tblName, "user1", cols, ptnCols); + try { + cachedStore.createTable(tbl); + } catch (InvalidObjectException | MetaException e) { + e.printStackTrace(); + System.out.println(e); + } + return null; + } + }; + tasks.add(c); + } + executor.invokeAll(tasks); + for (String tblName : tblNames) { + // Read database via ObjectStore + Table tblOS = objectStore.getTable(dbNames.get(0), tblName); + // Read database via CachedStore + Table tblCS= cachedStore.getTable(dbNames.get(0), tblName); + Assert.assertEquals(tblOS, tblCS); + } + + // Add 5 partitions to all tables + List ptnVals = new ArrayList(Arrays.asList("aaa", "bbb", "ccc", "ddd", "eee")); + tasks.clear(); + for (String tblName : tblNames) { + Table tbl = cachedStore.getTable(dbNames.get(0), tblName); + for (String ptnVal : ptnVals) { + Map partParams = new HashMap(); + Callable c = new Callable() { + public Object call() { + Partition ptn = new Partition(Arrays.asList(ptnVal), dbNames.get(0), tblName, 0, 0, + tbl.getSd(), partParams); + try { + cachedStore.addPartition(ptn); + } catch (InvalidObjectException | MetaException e) { + e.printStackTrace(); + System.out.println(e); + } + return null; + } + }; + tasks.add(c); + } + } + executor.invokeAll(tasks); + for (String tblName : tblNames) { + for (String ptnVal : ptnVals) { + // Read database via ObjectStore + Partition ptnOS = objectStore.getPartition(dbNames.get(0), tblName, Arrays.asList(ptnVal)); + // Read database via CachedStore + Partition ptnCS= cachedStore.getPartition(dbNames.get(0), tblName, Arrays.asList(ptnVal)); + Assert.assertEquals(ptnOS, ptnCS); + } + } + + // Drop all partitions from "tbl1", "tbl2", "tbl3" and add 2 new partitions to "tbl4" and "tbl5" + List newPtnVals = new ArrayList(Arrays.asList("fff", "ggg")); + List dropPtnTblNames = new ArrayList(Arrays.asList("tbl1", "tbl2", "tbl3")); + List addPtnTblNames = new ArrayList(Arrays.asList("tbl4", "tbl5")); + tasks.clear(); + for (String tblName : dropPtnTblNames) { + for (String ptnVal : ptnVals) { + Callable c = new Callable() { + public Object call() { + try { + cachedStore.dropPartition(dbNames.get(0), tblName, Arrays.asList(ptnVal)); + } catch (InvalidObjectException | MetaException | NoSuchObjectException | InvalidInputException e) { + e.printStackTrace(); + System.out.println(e); + } + return null; + } + }; + tasks.add(c); + } + } + for (String tblName : addPtnTblNames) { + Table tbl = cachedStore.getTable(dbNames.get(0), tblName); + for (String ptnVal : newPtnVals) { + Map partParams = new HashMap(); + Callable c = new Callable() { + public Object call() { + Partition ptn = new Partition(Arrays.asList(ptnVal), dbNames.get(0), tblName, 0, 0, + tbl.getSd(), partParams); + try { + cachedStore.addPartition(ptn); + } catch (InvalidObjectException | MetaException e) { + e.printStackTrace(); + System.out.println(e); + } + return null; + } + }; + tasks.add(c); + } + } + executor.invokeAll(tasks); + for (String tblName : addPtnTblNames) { + for (String ptnVal : newPtnVals) { + // Read database via ObjectStore + Partition ptnOS = objectStore.getPartition(dbNames.get(0), tblName, Arrays.asList(ptnVal)); + // Read database via CachedStore + Partition ptnCS= cachedStore.getPartition(dbNames.get(0), tblName, Arrays.asList(ptnVal)); + Assert.assertEquals(ptnOS, ptnCS); + } + } + for (String tblName : dropPtnTblNames) { + List ptns = cachedStore.getPartitions(dbNames.get(0), tblName, 100); + Assert.assertEquals(0, ptns.size()); + } + } + + private Database createTestDb(String dbName, String dbOwner) { + String dbDescription = dbName; + String dbLocation = "file:/tmp"; + Map dbParams = new HashMap<>(); + Database db = new Database(dbName, dbDescription, dbLocation, dbParams); + db.setOwnerName(dbOwner); + db.setOwnerType(PrincipalType.USER); + return db; } - private void updateCache(CachedStore cachedStore, long frequency, long sleepTime, - long shutdownTimeout) throws InterruptedException { - // Set cache refresh period to 100 milliseconds - CachedStore.setCacheRefreshPeriod(100); + private Table createTestTbl(String dbName, String tblName, String tblOwner, + List cols, List ptnCols) { + String serdeLocation = "file:/tmp"; + Map serdeParams = new HashMap<>(); + Map tblParams = new HashMap<>(); + SerDeInfo serdeInfo = new SerDeInfo("serde", "seriallib", new HashMap<>()); + StorageDescriptor sd = new StorageDescriptor(cols, serdeLocation, "input", "output", false, 0, + serdeInfo, null, null, serdeParams); + sd.setStoredAsSubDirectories(false); + Table tbl = new Table(tblName, dbName, tblOwner, 0, 0, 0, sd, ptnCols, tblParams, null, null, + TableType.MANAGED_TABLE.toString()); + return tbl; + } + + // This method will return only after the cache has updated once + private void updateCache(CachedStore cachedStore) throws InterruptedException { + int maxTries = 100000; + long updateCountBefore = cachedStore.getCacheUpdateCount(); // Start the CachedStore update service - CachedStore.startCacheUpdateService(cachedStore.getConf()); - // Sleep for 500 ms so that cache update is complete - Thread.sleep(500); - // Stop cache update service + CachedStore.startCacheUpdateService(cachedStore.getConf(), true, false); + while ((cachedStore.getCacheUpdateCount() != (updateCountBefore + 1)) && (maxTries-- > 0)) { + Thread.sleep(1000); + } CachedStore.stopCacheUpdateService(100); } @@ -511,9 +664,9 @@ public void testSharedStoreDb() { Database newDb1 = new Database(); newDb1.setName("db1"); - sharedCache.addDatabaseToCache("db1", db1); - sharedCache.addDatabaseToCache("db2", db2); - sharedCache.addDatabaseToCache("db3", db3); + sharedCache.addDatabaseToCache(db1); + sharedCache.addDatabaseToCache(db2); + sharedCache.addDatabaseToCache(db3); Assert.assertEquals(sharedCache.getCachedDatabaseCount(), 3); @@ -665,22 +818,18 @@ public void testSharedStorePartition() { sharedCache.addPartitionToCache("db1", "tbl1", part3); sharedCache.addPartitionToCache("db1", "tbl2", part1); - Assert.assertEquals(sharedCache.getCachedPartitionCount(), 4); Assert.assertEquals(sharedCache.getSdCache().size(), 2); Partition t = sharedCache.getPartitionFromCache("db1", "tbl1", Arrays.asList("201701")); Assert.assertEquals(t.getSd().getLocation(), "loc1"); sharedCache.removePartitionFromCache("db1", "tbl2", Arrays.asList("201701")); - Assert.assertEquals(sharedCache.getCachedPartitionCount(), 3); Assert.assertEquals(sharedCache.getSdCache().size(), 2); sharedCache.alterPartitionInCache("db1", "tbl1", Arrays.asList("201701"), newPart1); - Assert.assertEquals(sharedCache.getCachedPartitionCount(), 3); Assert.assertEquals(sharedCache.getSdCache().size(), 3); sharedCache.removePartitionFromCache("db1", "tbl1", Arrays.asList("201702")); - Assert.assertEquals(sharedCache.getCachedPartitionCount(), 2); Assert.assertEquals(sharedCache.getSdCache().size(), 2); } @@ -755,10 +904,10 @@ public void testPartitionAggrStats() throws Exception { String dbName = "testTableColStatsOps1"; String tblName = "tbl1"; String colName = "f1"; - + Database db = new Database(dbName, null, "some_location", null); cachedStore.createDatabase(db); - + List cols = new ArrayList<>(); cols.add(new FieldSchema(colName, "int", null)); List partCols = new ArrayList<>(); @@ -766,29 +915,29 @@ public void testPartitionAggrStats() throws Exception { StorageDescriptor sd = new StorageDescriptor(cols, null, "input", "output", false, 0, new SerDeInfo("serde", "seriallib", new HashMap<>()), null, null, null); - + Table tbl = new Table(tblName, dbName, null, 0, 0, 0, sd, partCols, new HashMap<>(), null, null, TableType.MANAGED_TABLE.toString()); cachedStore.createTable(tbl); - + List partVals1 = new ArrayList<>(); partVals1.add("1"); List partVals2 = new ArrayList<>(); partVals2.add("2"); - + Partition ptn1 = new Partition(partVals1, dbName, tblName, 0, 0, sd, new HashMap<>()); cachedStore.addPartition(ptn1); Partition ptn2 = new Partition(partVals2, dbName, tblName, 0, 0, sd, new HashMap<>()); cachedStore.addPartition(ptn2); - + ColumnStatistics stats = new ColumnStatistics(); ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(true, dbName, tblName); statsDesc.setPartName("col"); List colStatObjs = new ArrayList<>(); - + ColumnStatisticsData data = new ColumnStatisticsData(); ColumnStatisticsObj colStats = new ColumnStatisticsObj(colName, "int", data); LongColumnStatsDataInspector longStats = new LongColumnStatsDataInspector(); @@ -798,15 +947,15 @@ public void testPartitionAggrStats() throws Exception { longStats.setNumDVs(30); data.setLongStats(longStats); colStatObjs.add(colStats); - + stats.setStatsDesc(statsDesc); stats.setStatsObj(colStatObjs); - + cachedStore.updatePartitionColumnStatistics(stats.deepCopy(), partVals1); - + longStats.setNumDVs(40); cachedStore.updatePartitionColumnStatistics(stats.deepCopy(), partVals2); - + List colNames = new ArrayList<>(); colNames.add(colName); List aggrPartVals = new ArrayList<>(); @@ -825,10 +974,10 @@ public void testPartitionAggrStatsBitVector() throws Exception { String dbName = "testTableColStatsOps2"; String tblName = "tbl2"; String colName = "f1"; - + Database db = new Database(dbName, null, "some_location", null); cachedStore.createDatabase(db); - + List cols = new ArrayList<>(); cols.add(new FieldSchema(colName, "int", null)); List partCols = new ArrayList<>(); @@ -836,29 +985,29 @@ public void testPartitionAggrStatsBitVector() throws Exception { StorageDescriptor sd = new StorageDescriptor(cols, null, "input", "output", false, 0, new SerDeInfo("serde", "seriallib", new HashMap<>()), null, null, null); - + Table tbl = new Table(tblName, dbName, null, 0, 0, 0, sd, partCols, new HashMap<>(), null, null, TableType.MANAGED_TABLE.toString()); cachedStore.createTable(tbl); - + List partVals1 = new ArrayList<>(); partVals1.add("1"); List partVals2 = new ArrayList<>(); partVals2.add("2"); - + Partition ptn1 = new Partition(partVals1, dbName, tblName, 0, 0, sd, new HashMap<>()); cachedStore.addPartition(ptn1); Partition ptn2 = new Partition(partVals2, dbName, tblName, 0, 0, sd, new HashMap<>()); cachedStore.addPartition(ptn2); - + ColumnStatistics stats = new ColumnStatistics(); ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(true, dbName, tblName); statsDesc.setPartName("col"); List colStatObjs = new ArrayList<>(); - + ColumnStatisticsData data = new ColumnStatisticsData(); ColumnStatisticsObj colStats = new ColumnStatisticsObj(colName, "int", data); LongColumnStatsDataInspector longStats = new LongColumnStatsDataInspector(); @@ -866,21 +1015,21 @@ public void testPartitionAggrStatsBitVector() throws Exception { longStats.setHighValue(100); longStats.setNumNulls(50); longStats.setNumDVs(30); - + HyperLogLog hll = HyperLogLog.builder().build(); hll.addLong(1); hll.addLong(2); hll.addLong(3); longStats.setBitVectors(hll.serialize()); - + data.setLongStats(longStats); colStatObjs.add(colStats); - + stats.setStatsDesc(statsDesc); stats.setStatsObj(colStatObjs); - + cachedStore.updatePartitionColumnStatistics(stats.deepCopy(), partVals1); - + longStats.setNumDVs(40); hll = HyperLogLog.builder().build(); hll.addLong(2); @@ -888,9 +1037,9 @@ public void testPartitionAggrStatsBitVector() throws Exception { hll.addLong(4); hll.addLong(5); longStats.setBitVectors(hll.serialize()); - + cachedStore.updatePartitionColumnStatistics(stats.deepCopy(), partVals2); - + List colNames = new ArrayList<>(); colNames.add(colName); List aggrPartVals = new ArrayList<>(); diff --git a/standalone-metastore/src/test/resources/log4j2.properties b/standalone-metastore/src/test/resources/log4j2.properties index 365687e1c9..712ab273ee 100644 --- a/standalone-metastore/src/test/resources/log4j2.properties +++ b/standalone-metastore/src/test/resources/log4j2.properties @@ -8,28 +8,64 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -name=PropertiesConfig -property.filename = logs -appenders = console +status = DEBUG +name = MetastoreLog4j2 +packages = org.apache.hadoop.hive.metastore +# list of properties +property.metastore.log.level = DEBUG +property.metastore.root.logger = DRFA +property.metastore.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} +property.metastore.log.file = metastore.log +property.hive.perflogger.log.level = DEBUG + +# list of all appenders +appenders = console, DRFA + +# console appender appender.console.type = Console -appender.console.name = STDOUT +appender.console.name = console +appender.console.target = SYSTEM_ERR appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%-5level] %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %c{1} - %msg%n +appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n + +# daily rolling file appender +appender.DRFA.type = RollingRandomAccessFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:metastore.log.dir}/${sys:metastore.log.file} +# Use %pid in the filePattern to append @ to the filename if you want separate log files for different CLI session +appender.DRFA.filePattern = ${sys:metastore.log.dir}/${sys:metastore.log.file}.%d{yyyy-MM-dd} +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = 30 + +# list of all loggers +loggers = DataNucleus, Datastore, JPOX, PerfLogger + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = DEBUG + +logger.Datastore.name = Datastore +logger.Datastore.level = DEBUG + +logger.JPOX.name = JPOX +logger.JPOX.level = DEBUG -loggers=file -logger.file.name=guru.springframework.blog.log4j2properties -logger.file.level = debug -logger.file.appenderRefs = file -logger.file.appenderRef.file.ref = LOGFILE +logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger +logger.PerfLogger.level = ${sys:hive.perflogger.log.level} -rootLogger.level = debug -rootLogger.appenderRefs = stdout -rootLogger.appenderRef.stdout.ref = STDOUT +# root logger +rootLogger.level = ${sys:metastore.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:metastore.root.logger}