diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
index abf7198adb..c7cd4ad3f6 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
@@ -394,6 +394,7 @@ public void testFetchOperatorContextQuoting() throws Exception {
// ensure that both of the partitions are in the complete list.
String[] dirs = job.get("hive.complete.dir.list").split("\t");
assertEquals(2, dirs.length);
+ Arrays.sort(dirs);
assertEquals(true, dirs[0].endsWith("/state=CA"));
assertEquals(true, dirs[1].endsWith("/state=OR"));
return super.getSplits(job, splits);
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index 0445cbf909..b9a545872f 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -853,9 +853,7 @@ public Table getTable(String catName, String dbName, String tblName) throws Meta
}
@Override
- public Table getTable(String catName, String dbName, String tblName,
- String validWriteIds)
- throws MetaException {
+ public Table getTable(String catName, String dbName, String tblName, String validWriteIds) throws MetaException {
catName = normalizeIdentifier(catName);
dbName = StringUtils.normalizeIdentifier(dbName);
tblName = StringUtils.normalizeIdentifier(tblName);
@@ -872,12 +870,28 @@ public Table getTable(String catName, String dbName, String tblName,
return rawStore.getTable(catName, dbName, tblName, validWriteIds);
}
if (validWriteIds != null) {
- tbl.setParameters(adjustStatsParamsForGet(tbl.getParameters(),
- tbl.getParameters(), tbl.getWriteId(), validWriteIds));
+ tbl.setParameters(
+ adjustStatsParamsForGet(tbl.getParameters(), tbl.getParameters(), tbl.getWriteId(), validWriteIds));
}
tbl.unsetPrivileges();
tbl.setRewriteEnabled(tbl.isRewriteEnabled());
+ if (tbl.getPartitionKeys() == null) {
+ // getTable call from ObjectStore returns an empty list
+ tbl.setPartitionKeys(new ArrayList<>());
+ }
+ String tableType = tbl.getTableType();
+ if (tableType == null) {
+ // for backwards compatibility with old metastore persistence
+ if (tbl.getViewOriginalText() != null) {
+ tableType = TableType.VIRTUAL_VIEW.toString();
+ } else if ("TRUE".equals(tbl.getParameters().get("EXTERNAL"))) {
+ tableType = TableType.EXTERNAL_TABLE.toString();
+ } else {
+ tableType = TableType.MANAGED_TABLE.toString();
+ }
+ }
+ tbl.setTableType(tableType);
return tbl;
}
@@ -1133,6 +1147,10 @@ public void updateCreationMetadata(String catName, String dbname, String tablena
if (!isCachePrewarmed.get() || missSomeInCache) {
return rawStore.getTableObjectsByName(catName, dbName, tblNames);
}
+ Database db = sharedCache.getDatabaseFromCache(catName, dbName);
+ if (db == null) {
+ throw new UnknownDBException("Could not find database " + dbName);
+ }
List
tables = new ArrayList<>();
for (String tblName : tblNames) {
tblName = normalizeIdentifier(tblName);
@@ -1140,6 +1158,9 @@ public void updateCreationMetadata(String catName, String dbname, String tablena
if (tbl == null) {
tbl = rawStore.getTable(catName, dbName, tblName);
}
+ if (tbl != null) {
+ tables.add(tbl);
+ }
tables.add(tbl);
}
return tables;
@@ -1155,14 +1176,10 @@ public void updateCreationMetadata(String catName, String dbname, String tablena
}
@Override
- public List listTableNamesByFilter(String catName, String dbName, String filter,
- short max_tables)
+ // TODO: implement using SharedCache
+ public List listTableNamesByFilter(String catName, String dbName, String filter, short max_tables)
throws MetaException, UnknownDBException {
- if (!isBlacklistWhitelistEmpty(conf) || !isCachePrewarmed.get()) {
- return rawStore.listTableNamesByFilter(catName, dbName, filter, max_tables);
- }
- return sharedCache.listCachedTableNames(StringUtils.normalizeIdentifier(catName),
- StringUtils.normalizeIdentifier(dbName), filter, max_tables);
+ return rawStore.listTableNamesByFilter(catName, dbName, filter, max_tables);
}
@Override
@@ -1246,6 +1263,7 @@ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr,
}
@Override
+ // TODO: implement using SharedCache
public List getPartitionsByFilter(String catName, String dbName, String tblName,
String filter, short maxParts)
throws MetaException, NoSuchObjectException {
@@ -1558,85 +1576,85 @@ public Partition getPartitionWithAuth(String catName, String dbName, String tblN
}
@Override
- public List listPartitionNamesPs(String catName, String dbName, String tblName,
- List partVals, short maxParts)
- throws MetaException, NoSuchObjectException {
+ public List listPartitionNamesPs(String catName, String dbName, String tblName, List partSpecs,
+ short maxParts) throws MetaException, NoSuchObjectException {
catName = StringUtils.normalizeIdentifier(catName);
dbName = StringUtils.normalizeIdentifier(dbName);
tblName = StringUtils.normalizeIdentifier(tblName);
if (!shouldCacheTable(catName, dbName, tblName)) {
- return rawStore.listPartitionNamesPs(catName, dbName, tblName, partVals, maxParts);
+ return rawStore.listPartitionNamesPs(catName, dbName, tblName, partSpecs, maxParts);
}
Table table = sharedCache.getTableFromCache(catName, dbName, tblName);
if (table == null) {
// The table is not yet loaded in cache
- return rawStore.listPartitionNamesPs(catName, dbName, tblName, partVals, maxParts);
+ return rawStore.listPartitionNamesPs(catName, dbName, tblName, partSpecs, maxParts);
}
- List partNames = new ArrayList<>();
+ String partNameMatcher = getPartNameMatcher(table, partSpecs);
+ List partitionNames = new ArrayList<>();
+ List allPartitions = sharedCache.listCachedPartitions(catName, dbName, tblName, maxParts);
int count = 0;
- for (Partition part : sharedCache.listCachedPartitions(catName, dbName, tblName, maxParts)) {
- boolean psMatch = true;
- for (int i=0;i listPartitionsPsWithAuth(String catName, String dbName, String tblName,
- List partVals, short maxParts, String userName, List groupNames)
+ public List listPartitionsPsWithAuth(String catName, String dbName, String tblName, List partSpecs,
+ short maxParts, String userName, List groupNames)
throws MetaException, InvalidObjectException, NoSuchObjectException {
catName = StringUtils.normalizeIdentifier(catName);
dbName = StringUtils.normalizeIdentifier(dbName);
tblName = StringUtils.normalizeIdentifier(tblName);
if (!shouldCacheTable(catName, dbName, tblName)) {
- return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partVals, maxParts, userName,
- groupNames);
+ return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partSpecs, maxParts, userName, groupNames);
}
Table table = sharedCache.getTableFromCache(catName, dbName, tblName);
if (table == null) {
// The table is not yet loaded in cache
- return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partVals, maxParts, userName,
- groupNames);
+ return rawStore.listPartitionsPsWithAuth(catName, dbName, tblName, partSpecs, maxParts, userName, groupNames);
}
+ String partNameMatcher = getPartNameMatcher(table, partSpecs);
List partitions = new ArrayList<>();
+ List allPartitions = sharedCache.listCachedPartitions(catName, dbName, tblName, maxParts);
int count = 0;
- for (Partition part : sharedCache.listCachedPartitions(catName, dbName, tblName, maxParts)) {
- boolean psMatch = true;
- for (int i = 0; i < partVals.size(); i++) {
- String psVal = partVals.get(i);
- String partVal = part.getValues().get(i);
- if (psVal != null && !psVal.isEmpty() && !psVal.equals(partVal)) {
- psMatch = false;
- break;
- }
- }
- if (!psMatch) {
- continue;
- }
- if (maxParts == -1 || count < maxParts) {
- String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues());
+ for (Partition part : allPartitions) {
+ String partName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues());
+ if (partName.matches(partNameMatcher) && (maxParts == -1 || count < maxParts)) {
PrincipalPrivilegeSet privs =
getPartitionPrivilegeSet(catName, dbName, tblName, partName, userName, groupNames);
part.setPrivileges(privs);
partitions.add(part);
+ count++;
}
}
return partitions;
}
+
+ private String getPartNameMatcher(Table table, List partSpecs) throws MetaException {
+ List partCols = table.getPartitionKeys();
+ int numPartKeys = partCols.size();
+ if (partSpecs.size() > numPartKeys) {
+ throw new MetaException(
+ "Incorrect number of partition values." + " numPartKeys=" + numPartKeys + ", partSpecs=" + partSpecs.size());
+ }
+ partCols = partCols.subList(0, partSpecs.size());
+ // Construct a pattern of the form: partKey=partVal/partKey2=partVal2/...
+ // where partVal is either the escaped partition value given as input,
+ // or a regex of the form ".*"
+ // This works because the "=" and "/" separating key names and partition key/values
+ // are not escaped.
+ String partNameMatcher = Warehouse.makePartName(partCols, partSpecs, ".*");
+ // add ".*" to the regex to match anything else afterwards the partial spec.
+ if (partSpecs.size() < numPartKeys) {
+ partNameMatcher += ".*";
+ }
+ return partNameMatcher;
+ }
// Note: ideally this should be above both CachedStore and ObjectStore.
private Map adjustStatsParamsForGet(Map tableParams,
@@ -1893,7 +1911,7 @@ private MergedColumnStatsForPartitions mergeColStatsForPartitions(
colStatsMap.put(colStatsAggregator, colStatsWithPartInfoList);
}
if (partsFoundForColumn == partNames.size()) {
- partsFound++;
+ partsFound = partsFoundForColumn;
}
if (colStatsMap.size() < 1) {
LOG.debug("No stats data found for: dbName={} tblName= {} partNames= {} colNames= ", dbName,
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
index 37c300e882..c24e7160ac 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
@@ -293,10 +293,10 @@ public Partition removePartition(List partVal, SharedCache sharedCache)
PartitionWrapper wrapper =
partitionCache.remove(CacheUtils.buildPartitionCacheKey(partVal));
isPartitionCacheDirty.set(true);
+ part = CacheUtils.assemble(wrapper, sharedCache);
if (wrapper.getSdHash() != null) {
sharedCache.decrSd(wrapper.getSdHash());
}
- part = CacheUtils.assemble(wrapper, sharedCache);
// Remove col stats
String partialKey = CacheUtils.buildPartitionCacheKey(partVal);
Iterator> iterator =
@@ -461,7 +461,11 @@ public void refreshTableColStats(List colStatsForTable) {
public void removeTableColStats(String colName) {
try {
tableLock.writeLock().lock();
- tableColStatsCache.remove(colName);
+ if (colName == null) {
+ tableColStatsCache.clear();
+ } else {
+ tableColStatsCache.remove(colName);
+ }
isTableColStatsCacheDirty.set(true);
} finally {
tableLock.writeLock().unlock();