Index: standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java (revision c1e5fa3d80511ac625476b0a9c454783bd6c4ad1) +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java (revision ) @@ -35,6 +35,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; @@ -567,6 +569,23 @@ continue; } } + // add function in Cache + try { + List functions = rawStore.getFunctions(db.getCatalogName(), db.getName(), "*"); + List funcList = new ArrayList<>(); + if (functions != null && functions.size() > 0) { + for (String function : functions) { + Function funNew = rawStore.getFunction(db.getCatalogName(), db.getName(), function); + funcList.add(funNew); + } + } + sharedCache.populateFunctionsInCache(funcList); + } catch (MetaException e) { + LOG.warn("Failed to cache functions for database " + + DatabaseName.getQualified(catName, dbName) + ", moving on"); + } + + LOG.debug("Processed database: {}. Cached {} / {} databases so far.", dbName, ++numberOfDatabasesCachedSoFar, databases.size()); } @@ -2400,30 +2419,83 @@ } @Override public void createFunction(Function func) throws InvalidObjectException, MetaException { - // TODO fucntionCache rawStore.createFunction(func); + // in case of event based cache update, cache will be updated during commit. + if (canUseEvents) { + return; - } + } + String catName = normalizeIdentifier(func.getCatName()); + String dbName = normalizeIdentifier(func.getDbName()); + String funcName = normalizeIdentifier(func.getFunctionName()); + sharedCache.addFunctionToCache(catName, dbName, funcName, func); + } @Override public void alterFunction(String catName, String dbName, String funcName, Function newFunction) - throws InvalidObjectException, MetaException { + throws InvalidObjectException, MetaException { - // TODO fucntionCache rawStore.alterFunction(catName, dbName, funcName, newFunction); + // in case of event based cache update, cache will be updated during commit. + if (canUseEvents) { + return; - } + } + catName = normalizeIdentifier(catName); + dbName = normalizeIdentifier(dbName); + funcName = normalizeIdentifier(funcName); + String newFuncName = normalizeIdentifier(newFunction.getFunctionName()); + + Function oldFunction = sharedCache.getFunctionFromCache(catName, dbName, funcName); + if (oldFunction == null) { + return; + } + sharedCache.removeFunctionFromCache(catName, dbName, funcName); + sharedCache.alterFunctionInCache(catName, dbName, newFuncName, newFunction); + } + @Override public void dropFunction(String catName, String dbName, String funcName) - throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { + throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { - // TODO fucntionCache rawStore.dropFunction(catName, dbName, funcName); + // in case of event based cache update, cache will be updated during commit. + if (!canUseEvents) { + catName = normalizeIdentifier(catName); + dbName = normalizeIdentifier(dbName); + funcName = normalizeIdentifier(funcName); + sharedCache.removeFunctionFromCache(catName, dbName, funcName); - } + } + } @Override public Function getFunction(String catName, String dbName, String funcName) throws MetaException { - // TODO fucntionCache - return rawStore.getFunction(catName, dbName, funcName); + + catName = normalizeIdentifier(catName); + dbName = StringUtils.normalizeIdentifier(dbName); + funcName = StringUtils.normalizeIdentifier(funcName); + Function func = sharedCache.getFunctionFromCache(catName, dbName, funcName); + + if (func == null) { + Function funcRawStore = rawStore.getFunction(catName, dbName, funcName); + if (funcRawStore != null) { + sharedCache.addFunctionToCache(catName, dbName, funcName, funcRawStore); + func = funcRawStore; - } + } + } + return func; + } + @Override public List getAllFunctions(String catName) throws MetaException { - // TODO fucntionCache - return rawStore.getAllFunctions(catName); + List funcList = null; + if(!isCachePrewarmed.get()){ + funcList = rawStore.getAllFunctions(catName); + } + else{ + List tmpFunctionList = sharedCache.getAllFunctionsFromCache(catName); + if(tmpFunctionList!=null && tmpFunctionList.size() > 0){ + funcList = tmpFunctionList; + } + } + if(CollectionUtils.isEmpty(funcList)){ + funcList = Lists.newArrayList(); + } + return funcList; } @Override public List getFunctions(String catName, String dbName, String pattern) throws MetaException { Index: standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java (revision c1e5fa3d80511ac625476b0a9c454783bd6c4ad1) +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CacheUtils.java (revision ) @@ -145,4 +145,9 @@ } return false; } + + public static String buildFunctionKey(String catName, String dbName, String functionName) { + return buildKey(catName.toLowerCase(), + dbName.toLowerCase(), functionName.toLowerCase()); + } } Index: standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java (revision c1e5fa3d80511ac625476b0a9c454783bd6c4ad1) +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java (revision ) @@ -65,6 +65,7 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TableMeta; +import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; @@ -111,6 +112,9 @@ private ScheduledExecutorService executor = null; private Map tableSizeMap = null; + private Map functionCache = new TreeMap<>(); + private boolean isFunctionCachePrewarmed = false; + enum StatsType { ALL(0), ALLBUTDEFAULT(1), PARTIAL(2); @@ -2114,5 +2118,90 @@ public void incrementUpdateCount() { cacheUpdateCount.incrementAndGet(); + } + + public void populateFunctionsInCache(Collection functions) { + for (Function func : functions) { + Function funcCopy = func.deepCopy(); + funcCopy.setFunctionName(funcCopy.getFunctionName().toLowerCase()); + try { + cacheLock.writeLock().lock(); + String key = CacheUtils.buildFunctionKey(funcCopy.getCatName().toLowerCase(), + funcCopy.getDbName().toLowerCase(), funcCopy.getFunctionName().toLowerCase()); + functionCache.putIfAbsent(key, funcCopy); + isFunctionCachePrewarmed = true; + } finally { + cacheLock.writeLock().unlock(); + } + } + } + + public Function getFunctionFromCache(String catName, String dbName, String funcName) { + Function func = null; + try { + cacheLock.readLock().lock(); + Function funcQuery = functionCache.get(CacheUtils.buildTableKey(catName, dbName, funcName)); + if (funcQuery != null) { + func = funcQuery; + } + } finally { + cacheLock.readLock().unlock(); + } + return func; + } + + public void addFunctionToCache(String catName, String dbName, String funcName, Function func) { + Function funcCopy = func.deepCopy(); + try { + cacheLock.writeLock().lock(); + String key = CacheUtils.buildFunctionKey(catName.toLowerCase(), dbName.toLowerCase(), funcName.toLowerCase()); + functionCache.putIfAbsent(key, funcCopy); + } finally { + cacheLock.writeLock().unlock(); + } + } + + public void alterFunctionInCache(String catName, String dbName, String funcName, Function func) { + Function funcCopy = func.deepCopy(); + try { + cacheLock.writeLock().lock(); + String key = CacheUtils.buildFunctionKey(catName, dbName, funcName); + functionCache.putIfAbsent(key, funcCopy); + } finally { + cacheLock.writeLock().unlock(); + } + } + + public void removeFunctionFromCache(String catName, String dbName, String funcName) { + try { + cacheLock.writeLock().lock(); + String key = CacheUtils.buildFunctionKey(catName, dbName, funcName); + Function func = functionCache.get(key); + if (func == null) { + return; + } + functionCache.remove(key); + } finally { + cacheLock.writeLock().unlock(); + } + } + + public List getAllFunctionsFromCache(String catName) { + catName = catName.toLowerCase(); + List funcs = new ArrayList(); + try { + cacheLock.readLock().lock(); + if (functionCache != null && functionCache.size() > 0) { + for (Function func : funcs) { + if(org.apache.commons.lang.StringUtils.isNotBlank(func.getCatName()) && + org.apache.commons.lang.StringUtils.equals(func.getCatName(),catName)){ + funcs.add(func); + } + } + } + } finally { + cacheLock.readLock().unlock(); + } + return funcs; } }