Index: standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java (revision c1e5fa3d80511ac625476b0a9c454783bd6c4ad1) +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java (revision ) @@ -567,6 +567,22 @@ continue; } } + // add function in Cache + try { + List functions = rawStore.getFunctions(db.getCatalogName(), db.getName(), "*"); + List _functions = new ArrayList<>(); + if (functions != null && functions.size() > 0) { + for (String function : functions) { + Function _fun = rawStore.getFunction(db.getCatalogName(), db.getName(), function); + _functions.add(_fun); + } + } + sharedCache.populateFunctionsInCache(_functions); + } catch (MetaException e) { + LOG.warn("Failed to cache functions for database " + DatabaseName.getQualified(catName, dbName) + ", moving on"); + } + + LOG.debug("Processed database: {}. Cached {} / {} databases so far.", dbName, ++numberOfDatabasesCachedSoFar, databases.size()); } @@ -2399,26 +2415,74 @@ return rawStore.listTableColumnGrantsAll(catName, dbName, tableName, columnName); } - @Override public void createFunction(Function func) throws InvalidObjectException, MetaException { - // TODO fucntionCache + @Override + public void createFunction(Function func) throws InvalidObjectException, MetaException { + rawStore.createFunction(func); + // in case of event based cache update, cache will be updated during commit. + if (canUseEvents) { + return; - } + } + String catName = normalizeIdentifier(func.getCatName()); + String dbName = normalizeIdentifier(func.getDbName()); + String funcName = normalizeIdentifier(func.getFunctionName()); + sharedCache.addFunctionToCache(catName, dbName, funcName, func); + } - @Override public void alterFunction(String catName, String dbName, String funcName, Function newFunction) + @Override + public void alterFunction(String catName, String dbName, String funcName, Function newFunction) - throws InvalidObjectException, MetaException { + throws InvalidObjectException, MetaException { - // TODO fucntionCache + rawStore.alterFunction(catName, dbName, funcName, newFunction); + + // in case of event based cache update, cache will be updated during commit. + if (canUseEvents) { + return; - } + } - @Override public void dropFunction(String catName, String dbName, String funcName) + catName = normalizeIdentifier(catName); + dbName = normalizeIdentifier(dbName); + funcName = normalizeIdentifier(funcName); + String newFuncName = normalizeIdentifier(newFunction.getFunctionName()); + + Function oldFunction = sharedCache.getFunctionFromCache(catName, dbName, funcName); + if (oldFunction == null) { + return; + } + sharedCache.removeFunctionFromCache(catName, dbName, funcName); + sharedCache.alterFunctionInCache(catName, dbName, newFuncName, newFunction); + } + + @Override + public void dropFunction(String catName, String dbName, String funcName) - throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { + throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { // TODO fucntionCache rawStore.dropFunction(catName, dbName, funcName); + // in case of event based cache update, cache will be updated during commit. + if (!canUseEvents) { + catName = normalizeIdentifier(catName); + dbName = normalizeIdentifier(dbName); + funcName = normalizeIdentifier(funcName); + sharedCache.removeFunctionFromCache(catName, dbName, funcName); - } + } + } @Override public Function getFunction(String catName, String dbName, String funcName) throws MetaException { - // TODO fucntionCache - return rawStore.getFunction(catName, dbName, funcName); + + catName = normalizeIdentifier(catName); + dbName = StringUtils.normalizeIdentifier(dbName); + funcName = StringUtils.normalizeIdentifier(funcName); + Function func = sharedCache.getFunctionFromCache(catName, dbName, funcName); + + if (func == null) { + Function _func = rawStore.getFunction(catName, dbName, funcName); + if (_func != null) { + sharedCache.addFunctionToCache(catName, dbName, funcName, _func); + func = _func; + } + } + + return func; } @Override public List getAllFunctions(String catName) throws MetaException { Index: standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java (revision c1e5fa3d80511ac625476b0a9c454783bd6c4ad1) +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java (revision ) @@ -145,4 +145,8 @@ } return false; } + + public static String buildFunctionKey(String catName, String dbName, String functionName) { + return buildKey(catName.toLowerCase(), dbName.toLowerCase(), functionName.toLowerCase()); + } } Index: standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java (revision c1e5fa3d80511ac625476b0a9c454783bd6c4ad1) +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java (revision ) @@ -53,18 +53,7 @@ import org.apache.hadoop.hive.metastore.StatObjectConverter; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.AggrStats; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; -import org.apache.hadoop.hive.metastore.api.Catalog; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.TableMeta; +import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; @@ -111,6 +100,10 @@ private ScheduledExecutorService executor = null; private Map tableSizeMap = null; + private HashSet functionsDeletedDuringPrewarm = new HashSet<>(); + private Map functionCache = new TreeMap<>(); + private boolean isFunctionCachePrewarmed = false; + enum StatsType { ALL(0), ALLBUTDEFAULT(1), PARTIAL(2); @@ -2115,4 +2108,75 @@ public void incrementUpdateCount() { cacheUpdateCount.incrementAndGet(); } + + public void populateFunctionsInCache(Collection functions) { + for (Function func : functions) { + Function funcCopy = func.deepCopy(); + funcCopy.setFunctionName(funcCopy.getFunctionName().toLowerCase()); + try { + cacheLock.writeLock().lock(); + String key = CacheUtils.buildFunctionKey(funcCopy.getCatName().toLowerCase(), funcCopy.getDbName().toLowerCase(), funcCopy.getFunctionName().toLowerCase()); + if (functionsDeletedDuringPrewarm.contains(key)) { + continue; + } + functionCache.putIfAbsent(key, funcCopy); + functionsDeletedDuringPrewarm.clear(); + isFunctionCachePrewarmed = true; + } finally { + cacheLock.writeLock().unlock(); + } + } + } + + public Function getFunctionFromCache(String catName, String dbName, String funcName) { + Function func = null; + try { + cacheLock.readLock().lock(); + Function _func = functionCache.get(CacheUtils.buildTableKey(catName, dbName, funcName)); + if (_func != null) { + func = _func; + } + } finally { + cacheLock.readLock().unlock(); + } + return func; + } + + public void addFunctionToCache(String catName, String dbName, String funcName, Function func) { + Function funcCopy = func.deepCopy(); + try { + cacheLock.writeLock().lock(); + String key = CacheUtils.buildFunctionKey(catName.toLowerCase(), dbName.toLowerCase(), funcName.toLowerCase()); + functionCache.putIfAbsent(key, funcCopy); + } finally { + cacheLock.writeLock().unlock(); + } + } + + public void alterFunctionInCache(String catName, String dbName, String funcName, Function func) { + Function funcCopy = func.deepCopy(); + try { + cacheLock.writeLock().lock(); + String key = CacheUtils.buildFunctionKey(catName, dbName, funcName); + functionCache.putIfAbsent(key, funcCopy); + } finally { + cacheLock.writeLock().unlock(); + } + } + + public void removeFunctionFromCache(String catName, String dbName, String funcName) { + try { + cacheLock.writeLock().lock(); + String key = CacheUtils.buildFunctionKey(catName, dbName, funcName); + Function func = functionCache.get(key); + if (func == null) { + return; + } + functionCache.remove(key); + } finally { + cacheLock.writeLock().unlock(); + } + } + + }