diff --git a/standalone-metastore/metastore-server/pom.xml b/standalone-metastore/metastore-server/pom.xml
index 895abfc..ad2fdef 100644
--- a/standalone-metastore/metastore-server/pom.xml
+++ b/standalone-metastore/metastore-server/pom.xml
@@ -313,7 +313,7 @@
${test.tmp.dir}/derby.log
true
${test.tmp.dir}
- jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db;create=true
+ jdbc:derby:${test.tmp.dir}/junit_metastore_db;create=true
false
${test.tmp.dir}
${test.warehouse.scheme}${test.warehouse.dir}
@@ -548,7 +548,7 @@
true
${test.tmp.dir}
- jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db;create=true
+ jdbc:derby:${test.tmp.dir}/junit_metastore_db;create=true
false
${test.tmp.dir}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index e4ef46f..5b830c2 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -239,7 +239,7 @@ static void prewarm(RawStore rawStore) {
List partitionColStats = null;
AggrStats aggrStatsAllPartitions = null;
AggrStats aggrStatsAllButDefaultPartition = null;
- if (table.isSetPartitionKeys()) {
+ if (!table.getPartitionKeys().isEmpty()) {
Deadline.startTimer("getPartitions");
partitions = rawStore.getPartitions(catName, dbName, tblName, Integer.MAX_VALUE);
Deadline.stopTimer();
@@ -310,6 +310,7 @@ static void prewarm(RawStore rawStore) {
LOG.debug("Processed database: {}. Cached {} / {} databases so far.", dbName,
++numberOfDatabasesCachedSoFar, databases.size());
}
+ sharedCache.clearDirtyFlags();
completePrewarm(startTime);
}
}
@@ -549,7 +550,7 @@ private void updateTableColStats(RawStore rawStore, String catName, String dbNam
rawStore.openTransaction();
try {
Table table = rawStore.getTable(catName, dbName, tblName);
- if (!table.isSetPartitionKeys()) {
+ if (table.getPartitionKeys().isEmpty()) {
List colNames = MetaStoreUtils.getColumnNamesForTable(table);
Deadline.startTimer("getTableColumnStatistics");
@@ -600,12 +601,6 @@ private void updateTablePartitionColStats(RawStore rawStore, String catName, Str
List partitionColStats =
rawStore.getPartitionColumnStatistics(catName, dbName, tblName, partNames, colNames);
Deadline.stopTimer();
- sharedCache.refreshPartitionColStatsInCache(catName, dbName, tblName, partitionColStats);
- List parts = rawStore.getPartitionsByNames(catName, dbName, tblName, partNames);
- // Also save partitions for consistency as they have the stats state.
- for (Partition part : parts) {
- sharedCache.alterPartitionInCache(catName, dbName, tblName, part.getValues(), part);
- }
committed = rawStore.commitTransaction();
} catch (MetaException | NoSuchObjectException e) {
LOG.info("Updating CachedStore: unable to read partitions of table: " + tblName, e);
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
index c24e716..5d6a7d8 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
@@ -1057,7 +1057,7 @@ public boolean populateTableInCache(Table table, ColumnStatistics tableColStats,
}
LOG.debug("Current cache size: {} bytes", currentCacheSizeInBytes);
}
- if (!table.isSetPartitionKeys() && (tableColStats != null)) {
+ if (table.getPartitionKeys().isEmpty() && (tableColStats != null)) {
if (!tblWrapper.updateTableColStats(tableColStats.getStatsObj())) {
return false;
}
@@ -1085,6 +1085,10 @@ public boolean populateTableInCache(Table table, ColumnStatistics tableColStats,
tblWrapper.cacheAggrPartitionColStats(aggrStatsAllPartitions,
aggrStatsAllButDefaultPartition);
}
+ tblWrapper.isPartitionCacheDirty.set(false);
+ tblWrapper.isTableColStatsCacheDirty.set(false);
+ tblWrapper.isPartitionColStatsCacheDirty.set(false);
+ tblWrapper.isAggrPartitionColStatsCacheDirty.set(false);
try {
cacheLock.writeLock().lock();
// 2. Skip overwriting exisiting table object
@@ -1288,7 +1292,9 @@ public void refreshTablesInCache(String catName, String dbName, List tabl
tableCache.clear();
tableCache = newTableCache;
} finally {
- cacheLock.writeLock().unlock();
+ if (cacheLock.writeLock().isHeldByCurrentThread()) {
+ cacheLock.writeLock().unlock();
+ }
}
}
@@ -1692,6 +1698,12 @@ void resetCatalogCache() {
isCatalogCacheDirty.set(false);
}
+ void clearDirtyFlags() {
+ isCatalogCacheDirty.set(false);
+ isDatabaseCacheDirty.set(false);
+ isTableCacheDirty.set(false);
+ }
+
public long getUpdateCount() {
return cacheUpdateCount.get();
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java
index bb20d9f..b0cc1fc 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java
@@ -374,10 +374,10 @@ public void testPartitionOps() throws Exception {
sharedCache.getSdCache().clear();
}
- //@Test
+ @Test
public void testTableColStatsOps() throws Exception {
// Add a db via ObjectStore
- String dbName = "testTableColStatsOps";
+ String dbName = "testTableColStatsOps".toLowerCase();
String dbOwner = "user1";
Database db = createTestDb(dbName, dbOwner);
objectStore.createDatabase(db);
@@ -407,9 +407,7 @@ public void testTableColStatsOps() throws Exception {
cols.add(col1);
cols.add(col2);
cols.add(col3);
- FieldSchema ptnCol1 = new FieldSchema("part1", "string", "string partition column");
List ptnCols = new ArrayList();
- ptnCols.add(ptnCol1);
Table tbl = createTestTbl(dbName, tblName, tblOwner, cols, ptnCols);
objectStore.createTable(tbl);
tbl = objectStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName);
@@ -465,6 +463,21 @@ public void testTableColStatsOps() throws Exception {
ColumnStatistics newStats =
cachedStore.getTableColumnStatistics(DEFAULT_CATALOG_NAME, dbName, tblName,
Arrays.asList(col1.getName(), col2.getName(), col3.getName()));
+ newStats.setIsStatsCompliantIsSet(false);
+ Assert.assertEquals(stats, newStats);
+
+ boolStats.setNumNulls(col3Nulls-1);
+ // Save to DB
+ objectStore.updateTableColumnStatistics(stats, null, -1);
+
+ // We update twice to accurately detect if cache is dirty or not
+ updateCache(cachedStore);
+ updateCache(cachedStore);
+
+ newStats =
+ cachedStore.getTableColumnStatistics(DEFAULT_CATALOG_NAME, dbName, tblName,
+ Arrays.asList(col1.getName(), col2.getName(), col3.getName()));
+ newStats.setIsStatsCompliantIsSet(false);
Assert.assertEquals(stats, newStats);
// Clean up
@@ -664,7 +677,7 @@ public void testSharedStorePartition() {
@Test
public void testAggrStatsRepeatedRead() throws Exception {
- String dbName = "testTableColStatsOps";
+ String dbName = "testAggrStatsRepeatedRead";
String tblName = "tbl";
String colName = "f1";
@@ -736,7 +749,7 @@ public void testAggrStatsRepeatedRead() throws Exception {
@Test
public void testPartitionAggrStats() throws Exception {
- String dbName = "testTableColStatsOps1";
+ String dbName = "testPartitionAggrStats";
String tblName = "tbl1";
String colName = "f1";