diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 0445cbf909..8f5137315a 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -24,9 +24,11 @@ import java.util.Collection; import java.util.EmptyStackException; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.Stack; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -853,9 +855,7 @@ public Table getTable(String catName, String dbName, String tblName) throws Meta } @Override - public Table getTable(String catName, String dbName, String tblName, - String validWriteIds) - throws MetaException { + public Table getTable(String catName, String dbName, String tblName, String validWriteIds) throws MetaException { catName = normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); @@ -872,12 +872,27 @@ public Table getTable(String catName, String dbName, String tblName, return rawStore.getTable(catName, dbName, tblName, validWriteIds); } if (validWriteIds != null) { - tbl.setParameters(adjustStatsParamsForGet(tbl.getParameters(), - tbl.getParameters(), tbl.getWriteId(), validWriteIds)); + tbl.setParameters( + adjustStatsParamsForGet(tbl.getParameters(), tbl.getParameters(), tbl.getWriteId(), validWriteIds)); } - tbl.unsetPrivileges(); tbl.setRewriteEnabled(tbl.isRewriteEnabled()); + if (tbl.getPartitionKeys() == null) { + // getTable call from ObjectStore returns an empty list + tbl.setPartitionKeys(new ArrayList<>()); + } + String tableType = tbl.getTableType(); + if (tableType == null) { + // for backwards compatibility with old metastore persistence + if (tbl.getViewOriginalText() != null) { + tableType = TableType.VIRTUAL_VIEW.toString(); + } else if ("TRUE".equals(tbl.getParameters().get("EXTERNAL"))) { + tableType = TableType.EXTERNAL_TABLE.toString(); + } else { + tableType = TableType.MANAGED_TABLE.toString(); + } + } + tbl.setTableType(tableType); return tbl; } @@ -1045,15 +1060,14 @@ public void dropPartitions(String catName, String dbName, String tblName, List tables = new ArrayList<>(); for (String tblName : tblNames) { tblName = normalizeIdentifier(tblName); @@ -1140,7 +1158,9 @@ public void updateCreationMetadata(String catName, String dbname, String tablena if (tbl == null) { tbl = rawStore.getTable(catName, dbName, tblName); } - tables.add(tbl); + if (tbl != null) { + tables.add(tbl); + } } return tables; } @@ -1155,14 +1175,10 @@ public void updateCreationMetadata(String catName, String dbname, String tablena } @Override - public List listTableNamesByFilter(String catName, String dbName, String filter, - short max_tables) + // TODO: implement using SharedCache + public List listTableNamesByFilter(String catName, String dbName, String filter, short max_tables) throws MetaException, UnknownDBException { - if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed.get()) { - return rawStore.listTableNamesByFilter(catName, dbName, filter, max_tables); - } - return sharedCache.listCachedTableNames(StringUtils.normalizeIdentifier(catName), - StringUtils.normalizeIdentifier(dbName), filter, max_tables); + return rawStore.listTableNamesByFilter(catName, dbName, filter, max_tables); } @Override @@ -1246,6 +1262,7 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, } @Override + // TODO: implement using SharedCache public List getPartitionsByFilter(String catName, String dbName, String tblName, String filter, short maxParts) throws MetaException, NoSuchObjectException { @@ -1558,84 +1575,72 @@ public Partition getPartitionWithAuth(String catName, String dbName, String tblN } @Override - public List listPartitionNamesPs(String catName, String dbName, String tblName, - List partVals, short maxParts) - throws MetaException, NoSuchObjectException { + public List listPartitionNamesPs(String catName, String dbName, String tblName, List partSpecs, + short maxParts) throws MetaException, NoSuchObjectException { catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); if (!shouldCacheTable(catName, dbName, tblName)) { - return rawStore.listPartitionNamesPs(catName, dbName, tblName, partVals, maxParts); + return rawStore.listPartitionNamesPs(catName, dbName, tblName, partSpecs, maxParts); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); if (table == null) { // The table is not yet loaded in cache - return rawStore.listPartitionNamesPs(catName, dbName, tblName, partVals, maxParts); + return rawStore.listPartitionNamesPs(catName, dbName, tblName, partSpecs, maxParts); } - List partNames = new ArrayList<>(); + Set partNames = new HashSet<>(); int count = 0; for (Partition part : sharedCache.listCachedPartitions(catName, dbName, tblName, maxParts)) { - boolean psMatch = true; - for (int i=0;i partVals = part.getValues(); + for (String partVal : partVals) { + if (partSpec.equalsIgnoreCase(partVal) && (maxParts == -1 || count < maxParts)) { + partNames.add(Warehouse.makePartName(table.getPartitionKeys(), part.getValues())); + count++; + } + } } } - if (!psMatch) { - continue; - } - if (maxParts == -1 || count < maxParts) { - partNames.add(Warehouse.makePartName(table.getPartitionKeys(), part.getValues())); - count++; - } } - return partNames; + return new ArrayList<>(partNames); } @Override - public List listPartitionsPsWithAuth(String catName, String dbName, String tblName, - List partVals, short maxParts, String userName, List groupNames) + public List listPartitionsPsWithAuth(String catName, String dbName, String tblName, List partSpecs, + short maxParts, String userName, List groupNames) throws MetaException, InvalidObjectException, NoSuchObjectException { catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); if (!shouldCacheTable(catName, dbName, tblName)) { - return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partVals, maxParts, userName, - groupNames); + return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partSpecs, maxParts, userName, groupNames); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); if (table == null) { // The table is not yet loaded in cache - return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partVals, maxParts, userName, - groupNames); + return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partSpecs, maxParts, userName, groupNames); } - List partitions = new ArrayList<>(); + Set partitions = new HashSet<>(); int count = 0; for (Partition part : sharedCache.listCachedPartitions(catName, dbName, tblName, maxParts)) { - boolean psMatch = true; - for (int i = 0; i < partVals.size(); i++) { - String psVal = partVals.get(i); - String partVal = part.getValues().get(i); - if (psVal != null && !psVal.isEmpty() && !psVal.equals(partVal)) { - psMatch = false; - break; + for (String partSpec : partSpecs) { + if (partSpec != null && !partSpec.isEmpty()) { + List partVals = part.getValues(); + for (String partVal : partVals) { + if (partSpec.equalsIgnoreCase(partVal) && (maxParts == -1 || count < maxParts)) { + String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); + PrincipalPrivilegeSet privs = + getPartitionPrivilegeSet(catName, dbName, tblName, partName, userName, groupNames); + part.setPrivileges(privs); + partitions.add(part); + count++; + } + } } } - if (!psMatch) { - continue; - } - if (maxParts == -1 || count < maxParts) { - String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); - PrincipalPrivilegeSet privs = - getPartitionPrivilegeSet(catName, dbName, tblName, partName, userName, groupNames); - part.setPrivileges(privs); - partitions.add(part); - } } - return partitions; + return new ArrayList<>(partitions); } // Note: ideally this should be above both CachedStore and ObjectStore. diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java index 37c300e882..c24e7160ac 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java @@ -293,10 +293,10 @@ public Partition removePartition(List partVal, SharedCache sharedCache) PartitionWrapper wrapper = partitionCache.remove(CacheUtils.buildPartitionCacheKey(partVal)); isPartitionCacheDirty.set(true); + part = CacheUtils.assemble(wrapper, sharedCache); if (wrapper.getSdHash() != null) { sharedCache.decrSd(wrapper.getSdHash()); } - part = CacheUtils.assemble(wrapper, sharedCache); // Remove col stats String partialKey = CacheUtils.buildPartitionCacheKey(partVal); Iterator> iterator = @@ -461,7 +461,11 @@ public void refreshTableColStats(List colStatsForTable) { public void removeTableColStats(String colName) { try { tableLock.writeLock().lock(); - tableColStatsCache.remove(colName); + if (colName == null) { + tableColStatsCache.clear(); + } else { + tableColStatsCache.remove(colName); + } isTableColStatsCacheDirty.set(true); } finally { tableLock.writeLock().unlock();