diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java index abf7198adb..c7cd4ad3f6 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java @@ -394,6 +394,7 @@ public void testFetchOperatorContextQuoting() throws Exception { // ensure that both of the partitions are in the complete list. String[] dirs = job.get("hive.complete.dir.list").split("\t"); assertEquals(2, dirs.length); + Arrays.sort(dirs); assertEquals(true, dirs[0].endsWith("/state=CA")); assertEquals(true, dirs[1].endsWith("/state=OR")); return super.getSplits(job, splits); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 0445cbf909..b9a545872f 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -853,9 +853,7 @@ public Table getTable(String catName, String dbName, String tblName) throws Meta } @Override - public Table getTable(String catName, String dbName, String tblName, - String validWriteIds) - throws MetaException { + public Table getTable(String catName, String dbName, String tblName, String validWriteIds) throws MetaException { catName = normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); @@ -872,12 +870,28 @@ public Table getTable(String catName, String dbName, String tblName, return rawStore.getTable(catName, dbName, tblName, validWriteIds); } if (validWriteIds != null) { - tbl.setParameters(adjustStatsParamsForGet(tbl.getParameters(), - tbl.getParameters(), tbl.getWriteId(), validWriteIds)); + tbl.setParameters( + adjustStatsParamsForGet(tbl.getParameters(), tbl.getParameters(), tbl.getWriteId(), validWriteIds)); } tbl.unsetPrivileges(); tbl.setRewriteEnabled(tbl.isRewriteEnabled()); + if (tbl.getPartitionKeys() == null) { + // getTable call from ObjectStore returns an empty list + tbl.setPartitionKeys(new ArrayList<>()); + } + String tableType = tbl.getTableType(); + if (tableType == null) { + // for backwards compatibility with old metastore persistence + if (tbl.getViewOriginalText() != null) { + tableType = TableType.VIRTUAL_VIEW.toString(); + } else if ("TRUE".equals(tbl.getParameters().get("EXTERNAL"))) { + tableType = TableType.EXTERNAL_TABLE.toString(); + } else { + tableType = TableType.MANAGED_TABLE.toString(); + } + } + tbl.setTableType(tableType); return tbl; } @@ -1133,6 +1147,10 @@ public void updateCreationMetadata(String catName, String dbname, String tablena if (!isCachePrewarmed.get() || missSomeInCache) { return rawStore.getTableObjectsByName(catName, dbName, tblNames); } + Database db = sharedCache.getDatabaseFromCache(catName, dbName); + if (db == null) { + throw new UnknownDBException("Could not find database " + dbName); + } List tables = new ArrayList<>(); for (String tblName : tblNames) { tblName = normalizeIdentifier(tblName); @@ -1140,6 +1158,9 @@ public void updateCreationMetadata(String catName, String dbname, String tablena if (tbl == null) { tbl = rawStore.getTable(catName, dbName, tblName); } + if (tbl != null) { + tables.add(tbl); + } tables.add(tbl); } return tables; @@ -1155,14 +1176,10 @@ public void updateCreationMetadata(String catName, String dbname, String tablena } @Override - public List listTableNamesByFilter(String catName, String dbName, String filter, - short max_tables) + // TODO: implement using SharedCache + public List listTableNamesByFilter(String catName, String dbName, String filter, short max_tables) throws MetaException, UnknownDBException { - if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed.get()) { - return rawStore.listTableNamesByFilter(catName, dbName, filter, max_tables); - } - return sharedCache.listCachedTableNames(StringUtils.normalizeIdentifier(catName), - StringUtils.normalizeIdentifier(dbName), filter, max_tables); + return rawStore.listTableNamesByFilter(catName, dbName, filter, max_tables); } @Override @@ -1246,6 +1263,7 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, } @Override + // TODO: implement using SharedCache public List getPartitionsByFilter(String catName, String dbName, String tblName, String filter, short maxParts) throws MetaException, NoSuchObjectException { @@ -1558,85 +1576,85 @@ public Partition getPartitionWithAuth(String catName, String dbName, String tblN } @Override - public List listPartitionNamesPs(String catName, String dbName, String tblName, - List partVals, short maxParts) - throws MetaException, NoSuchObjectException { + public List listPartitionNamesPs(String catName, String dbName, String tblName, List partSpecs, + short maxParts) throws MetaException, NoSuchObjectException { catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); if (!shouldCacheTable(catName, dbName, tblName)) { - return rawStore.listPartitionNamesPs(catName, dbName, tblName, partVals, maxParts); + return rawStore.listPartitionNamesPs(catName, dbName, tblName, partSpecs, maxParts); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); if (table == null) { // The table is not yet loaded in cache - return rawStore.listPartitionNamesPs(catName, dbName, tblName, partVals, maxParts); + return rawStore.listPartitionNamesPs(catName, dbName, tblName, partSpecs, maxParts); } - List partNames = new ArrayList<>(); + String partNameMatcher = getPartNameMatcher(table, partSpecs); + List partitionNames = new ArrayList<>(); + List allPartitions = sharedCache.listCachedPartitions(catName, dbName, tblName, maxParts); int count = 0; - for (Partition part : sharedCache.listCachedPartitions(catName, dbName, tblName, maxParts)) { - boolean psMatch = true; - for (int i=0;i listPartitionsPsWithAuth(String catName, String dbName, String tblName, - List partVals, short maxParts, String userName, List groupNames) + public List listPartitionsPsWithAuth(String catName, String dbName, String tblName, List partSpecs, + short maxParts, String userName, List groupNames) throws MetaException, InvalidObjectException, NoSuchObjectException { catName = StringUtils.normalizeIdentifier(catName); dbName = StringUtils.normalizeIdentifier(dbName); tblName = StringUtils.normalizeIdentifier(tblName); if (!shouldCacheTable(catName, dbName, tblName)) { - return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partVals, maxParts, userName, - groupNames); + return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partSpecs, maxParts, userName, groupNames); } Table table = sharedCache.getTableFromCache(catName, dbName, tblName); if (table == null) { // The table is not yet loaded in cache - return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partVals, maxParts, userName, - groupNames); + return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partSpecs, maxParts, userName, groupNames); } + String partNameMatcher = getPartNameMatcher(table, partSpecs); List partitions = new ArrayList<>(); + List allPartitions = sharedCache.listCachedPartitions(catName, dbName, tblName, maxParts); int count = 0; - for (Partition part : sharedCache.listCachedPartitions(catName, dbName, tblName, maxParts)) { - boolean psMatch = true; - for (int i = 0; i < partVals.size(); i++) { - String psVal = partVals.get(i); - String partVal = part.getValues().get(i); - if (psVal != null && !psVal.isEmpty() && !psVal.equals(partVal)) { - psMatch = false; - break; - } - } - if (!psMatch) { - continue; - } - if (maxParts == -1 || count < maxParts) { - String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); + for (Partition part : allPartitions) { + String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); + if (partName.matches(partNameMatcher) && (maxParts == -1 || count < maxParts)) { PrincipalPrivilegeSet privs = getPartitionPrivilegeSet(catName, dbName, tblName, partName, userName, groupNames); part.setPrivileges(privs); partitions.add(part); + count++; } } return partitions; } + + private String getPartNameMatcher(Table table, List partSpecs) throws MetaException { + List partCols = table.getPartitionKeys(); + int numPartKeys = partCols.size(); + if (partSpecs.size() > numPartKeys) { + throw new MetaException( + "Incorrect number of partition values." + " numPartKeys=" + numPartKeys + ", partSpecs=" + partSpecs.size()); + } + partCols = partCols.subList(0, partSpecs.size()); + // Construct a pattern of the form: partKey=partVal/partKey2=partVal2/... + // where partVal is either the escaped partition value given as input, + // or a regex of the form ".*" + // This works because the "=" and "/" separating key names and partition key/values + // are not escaped. + String partNameMatcher = Warehouse.makePartName(partCols, partSpecs, ".*"); + // add ".*" to the regex to match anything else afterwards the partial spec. + if (partSpecs.size() < numPartKeys) { + partNameMatcher += ".*"; + } + return partNameMatcher; + } // Note: ideally this should be above both CachedStore and ObjectStore. private Map adjustStatsParamsForGet(Map tableParams, @@ -1893,7 +1911,7 @@ private MergedColumnStatsForPartitions mergeColStatsForPartitions( colStatsMap.put(colStatsAggregator, colStatsWithPartInfoList); } if (partsFoundForColumn == partNames.size()) { - partsFound++; + partsFound = partsFoundForColumn; } if (colStatsMap.size() < 1) { LOG.debug("No stats data found for: dbName={} tblName= {} partNames= {} colNames= ", dbName, diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java index 37c300e882..c24e7160ac 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java @@ -293,10 +293,10 @@ public Partition removePartition(List partVal, SharedCache sharedCache) PartitionWrapper wrapper = partitionCache.remove(CacheUtils.buildPartitionCacheKey(partVal)); isPartitionCacheDirty.set(true); + part = CacheUtils.assemble(wrapper, sharedCache); if (wrapper.getSdHash() != null) { sharedCache.decrSd(wrapper.getSdHash()); } - part = CacheUtils.assemble(wrapper, sharedCache); // Remove col stats String partialKey = CacheUtils.buildPartitionCacheKey(partVal); Iterator> iterator = @@ -461,7 +461,11 @@ public void refreshTableColStats(List colStatsForTable) { public void removeTableColStats(String colName) { try { tableLock.writeLock().lock(); - tableColStatsCache.remove(colName); + if (colName == null) { + tableColStatsCache.clear(); + } else { + tableColStatsCache.remove(colName); + } isTableColStatsCacheDirty.set(true); } finally { tableLock.writeLock().unlock();