diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index a97a623235..3a0ff19013 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2909,25 +2909,25 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal new RangeValidator(0, 100), "Determines how many attempted compaction records will be " + "retained in compaction history for a given table/partition."), /** - * @deprecated Use MetastoreConf.COMPACTOR_HISTORY_REAPER_INTERVAL + * @deprecated Use MetastoreConf.ACID_HOUSEKEEPER_SERVICE_INTERVAL */ @Deprecated COMPACTOR_HISTORY_REAPER_INTERVAL("hive.compactor.history.reaper.interval", "2m", new TimeValidator(TimeUnit.MILLISECONDS), "Determines how often compaction history reaper runs"), /** - * @deprecated Use MetastoreConf.TIMEDOUT_TXN_REAPER_START + * @deprecated Use MetastoreConf.ACID_HOUSEKEEPER_SERVICE_START */ @Deprecated HIVE_TIMEDOUT_TXN_REAPER_START("hive.timedout.txn.reaper.start", "100s", new TimeValidator(TimeUnit.MILLISECONDS), "Time delay of 1st reaper run after metastore start"), /** - * @deprecated Use MetastoreConf.TIMEDOUT_TXN_REAPER_INTERVAL + * @deprecated Use MetastoreConf.ACID_HOUSEKEEPER_SERVICE_INTERVAL */ @Deprecated HIVE_TIMEDOUT_TXN_REAPER_INTERVAL("hive.timedout.txn.reaper.interval", "180s", new TimeValidator(TimeUnit.MILLISECONDS), "Time interval describing how often the reaper runs"), /** - * @deprecated Use MetastoreConf.WRITE_SET_REAPER_INTERVAL + * @deprecated Use MetastoreConf.ACID_HOUSEKEEPER_SERVICE_INTERVAL */ @Deprecated WRITE_SET_REAPER_INTERVAL("hive.writeset.reaper.interval", "60s", diff --git ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java index 37a5862791..02020b4efc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java +++ ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java @@ -109,8 +109,9 @@ public void run() { //todo: add method to only get current i.e. skip history - more efficient ShowCompactResponse currentCompactions = txnHandler.showCompact(new ShowCompactRequest()); - Set potentials = txnHandler.findPotentialCompactions(abortedThreshold, compactionInterval) - .stream().filter(ci -> checkCompactionElig(ci, currentCompactions)).collect(Collectors.toSet()); + Set potentials = txnHandler.findPotentialCompactions(abortedThreshold, compactionInterval).stream() + .filter(ci -> isEligibleForCompaction(ci, currentCompactions)) + .collect(Collectors.toSet()); LOG.debug("Found " + potentials.size() + " potential compactions, " + "checking to see if we should compact any of them"); @@ -149,12 +150,6 @@ public void run() { // Check for timed out remote workers. recoverFailedCompactions(true); - - // Clean anything from the txns table that has no components left in txn_components. - txnHandler.cleanEmptyAbortedTxns(); - - // Clean TXN_TO_WRITE_ID table for entries under min_uncommitted_txn referred by any open txns. - txnHandler.cleanTxnToWriteIdTable(); } catch (Throwable t) { LOG.error("Initiator loop caught unexpected exception this time through the loop: " + StringUtils.stringifyException(t)); @@ -423,7 +418,7 @@ private static boolean checkDynPartitioning(Table t, CompactionInfo ci){ return false; } - private boolean checkCompactionElig(CompactionInfo ci, ShowCompactResponse currentCompactions) { + private boolean isEligibleForCompaction(CompactionInfo ci, ShowCompactResponse currentCompactions) { LOG.info("Checking to see if we should compact " + ci.getFullPartitionName()); // Check if we already have initiated or are working on a compaction for this partition diff --git ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java index 2c13e8dd03..fa8f66b652 100644 --- ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java +++ ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.ValidTxnWriteIdList; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.MetastoreTaskThread; import org.apache.hadoop.hive.metastore.api.CommitTxnRequest; import org.apache.hadoop.hive.metastore.api.CompactionRequest; import org.apache.hadoop.hive.metastore.api.CompactionType; @@ -47,7 +48,7 @@ import org.apache.hadoop.hive.metastore.api.ShowCompactRequest; import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement; -import org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService; +import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; @@ -1047,9 +1048,9 @@ void testInitiatorWithMultipleFailedCompactionsForVariousTblProperties(String tb checkCompactionState(new CompactionsByState(numAttemptedCompactions,numFailedCompactions,0,0,0,0,numFailedCompactions + numAttemptedCompactions), countCompacts(txnHandler)); hiveConf.setTimeVar(HiveConf.ConfVars.COMPACTOR_HISTORY_REAPER_INTERVAL, 10, TimeUnit.MILLISECONDS); - AcidCompactionHistoryService compactionHistoryService = new AcidCompactionHistoryService(); - compactionHistoryService.setConf(hiveConf); - compactionHistoryService.run(); + MetastoreTaskThread houseKeeper = new AcidHouseKeeperService(); + houseKeeper.setConf(hiveConf); + houseKeeper.run(); checkCompactionState(new CompactionsByState(numAttemptedCompactions,numFailedCompactions,0,0,0,0,numFailedCompactions + numAttemptedCompactions), countCompacts(txnHandler)); txnHandler.compact(new CompactionRequest("default", tblName, CompactionType.MAJOR)); @@ -1062,7 +1063,7 @@ void testInitiatorWithMultipleFailedCompactionsForVariousTblProperties(String tb numAttemptedCompactions++; checkCompactionState(new CompactionsByState(numAttemptedCompactions,numFailedCompactions + 2,0,0,0,0,numFailedCompactions + 2 + numAttemptedCompactions), countCompacts(txnHandler)); - compactionHistoryService.run(); + houseKeeper.run(); //COMPACTOR_HISTORY_RETENTION_FAILED failed compacts left (and no other since we only have failed ones here) checkCompactionState(new CompactionsByState( hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED), @@ -1086,7 +1087,7 @@ void testInitiatorWithMultipleFailedCompactionsForVariousTblProperties(String tb hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED)+ 1), countCompacts(txnHandler)); runCleaner(hiveConf); // transition to Success state - compactionHistoryService.run(); + houseKeeper.run(); checkCompactionState(new CompactionsByState( hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED), hiveConf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED),0,0,1,0, diff --git ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java index f90396b2a3..5c8db55d0c 100644 --- ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java +++ ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hive.metastore.api.ShowLocksResponse; import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.txn.AcidWriteSetService; +import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.ql.TestTxnCommands2; @@ -1215,7 +1215,7 @@ public void testWriteSetTracking4() throws Exception { Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from \"WRITE_SET\""), 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); - AcidWriteSetService houseKeeper = new AcidWriteSetService(); + MetastoreTaskThread houseKeeper = new AcidHouseKeeperService(); houseKeeper.setConf(conf); houseKeeper.run(); //since T3 overlaps with Long Running (still open) GC does nothing @@ -1311,7 +1311,7 @@ public void testWriteSetTracking6() throws Exception { Assert.assertEquals("Unexpected lock count", 1, locks.size()); checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB2", null, locks); txnMgr.commitTxn(); - MetastoreTaskThread writeSetService = new AcidWriteSetService(); + MetastoreTaskThread writeSetService = new AcidHouseKeeperService(); writeSetService.setConf(conf); writeSetService.run(); Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from \"WRITE_SET\"")); diff --git standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index a874121e12..e367457b46 100644 --- standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -86,17 +86,11 @@ static final String METASTORE_DELEGATION_MANAGER_CLASS = "org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager"; @VisibleForTesting - static final String ACID_COMPACTION_HISTORY_SERVICE_CLASS = - "org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService"; - @VisibleForTesting static final String ACID_HOUSE_KEEPER_SERVICE_CLASS = "org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService"; @VisibleForTesting static final String ACID_OPEN_TXNS_COUNTER_SERVICE_CLASS = "org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService"; - @VisibleForTesting - static final String ACID_WRITE_SET_SERVICE_CLASS = - "org.apache.hadoop.hive.metastore.txn.AcidWriteSetService"; public static final String METASTORE_AUTHENTICATION_LDAP_USERMEMBERSHIPKEY_NAME = "metastore.authentication.ldap.userMembershipKey"; @@ -271,6 +265,12 @@ public static ConfVars getMetaConf(String name) { public enum ConfVars { // alpha order, PLEASE! + ACID_HOUSEKEEPER_SERVICE_START("metastore.acid.housekeeper.start", + "hive.metastore.acid.housekeeper.start", 60, TimeUnit.SECONDS, + "Time delay of 1st acid housekeeper run after metastore has started."), + ACID_HOUSEKEEPER_SERVICE_INTERVAL("metastore.acid.housekeeper.interval", + "hive.metastore.acid.housekeeper.interval", 60, TimeUnit.SECONDS, + "Time interval describing how often the acid housekeeper runs."), ADDED_JARS("metastore.added.jars.path", "hive.added.jars.path", "", "This an internal parameter."), AGGREGATE_STATS_CACHE_CLEAN_UNTIL("metastore.aggregate.stats.cache.clean.until", @@ -382,9 +382,6 @@ public static ConfVars getMetaConf(String name) { "has an infinite lifetime."), CLIENT_SOCKET_TIMEOUT("metastore.client.socket.timeout", "hive.metastore.client.socket.timeout", 600, TimeUnit.SECONDS, "MetaStore Client socket timeout in seconds"), - COMPACTOR_HISTORY_REAPER_INTERVAL("metastore.compactor.history.reaper.interval", - "hive.compactor.history.reaper.interval", 2, TimeUnit.MINUTES, - "Determines how often compaction history reaper runs"), COMPACTOR_HISTORY_RETENTION_ATTEMPTED("metastore.compactor.history.retention.attempted", "hive.compactor.history.retention.attempted", 2, new RangeValidator(0, 100), "Determines how many attempted compaction records will be " + @@ -1059,8 +1056,6 @@ public static ConfVars getMetaConf(String name) { TASK_THREADS_REMOTE_ONLY("metastore.task.threads.remote", "metastore.task.threads.remote", ACID_HOUSE_KEEPER_SERVICE_CLASS + "," + ACID_OPEN_TXNS_COUNTER_SERVICE_CLASS + "," + - ACID_COMPACTION_HISTORY_SERVICE_CLASS + "," + - ACID_WRITE_SET_SERVICE_CLASS + "," + MATERIALZIATIONS_REBUILD_LOCK_CLEANER_TASK_CLASS + "," + PARTITION_MANAGEMENT_TASK_CLASS, "Command separated list of tasks that will be started in separate threads. These will be" + @@ -1149,12 +1144,6 @@ public static ConfVars getMetaConf(String name) { "metastore. SEQUENTIAL implies that the first valid metastore from the URIs specified " + "through hive.metastore.uris will be picked. RANDOM implies that the metastore " + "will be picked randomly"), - TIMEDOUT_TXN_REAPER_START("metastore.timedout.txn.reaper.start", - "hive.timedout.txn.reaper.start", 100, TimeUnit.SECONDS, - "Time delay of 1st reaper run after metastore start"), - TIMEDOUT_TXN_REAPER_INTERVAL("metastore.timedout.txn.reaper.interval", - "hive.timedout.txn.reaper.interval", 180, TimeUnit.SECONDS, - "Time interval describing how often the reaper runs"), TOKEN_SIGNATURE("metastore.token.signature", "hive.metastore.token.signature", "", "The delegation token service name to match when selecting a token from the current user's tokens."), METASTORE_CACHE_CAN_USE_EVENT("metastore.cache.can.use.event", "hive.metastore.cache.can.use.event", false, @@ -1249,9 +1238,6 @@ public static ConfVars getMetaConf(String name) { "hive.metastore.warehouse.external.dir", "", "Default location for external tables created in the warehouse. " + "If not set or null, then the normal warehouse location will be used as the default location."), - WRITE_SET_REAPER_INTERVAL("metastore.writeset.reaper.interval", - "hive.writeset.reaper.interval", 60, TimeUnit.SECONDS, - "Frequency of WriteSet reaper runs"), WM_DEFAULT_POOL_SIZE("metastore.wm.default.pool.size", "hive.metastore.wm.default.pool.size", 4, "The size of a default pool to create when creating an empty resource plan;\n" + diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 32494ae257..2fdb42fdf2 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -10266,8 +10266,6 @@ private static void logCompactionParameters(Configuration conf) { MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_WORKER_THREADS)); HMSHandler.LOG .info("hive.metastore.runworker.in = {}", MetastoreConf.getVar(conf, ConfVars.HIVE_METASTORE_RUNWORKER_IN)); - HMSHandler.LOG.info("metastore.compactor.history.reaper.interval = {}", - MetastoreConf.getTimeVar(conf, ConfVars.COMPACTOR_HISTORY_REAPER_INTERVAL, TimeUnit.MINUTES)); HMSHandler.LOG.info("metastore.compactor.history.retention.attempted = {}", MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED)); HMSHandler.LOG.info("metastore.compactor.history.retention.failed = {}", diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java index d35c9602a6..4c2d5e31b3 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java @@ -71,10 +71,10 @@ public void run() { LOG.debug("Number of materialization locks deleted: " + removedCnt); } } - } catch(Throwable t) { - LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); + } catch (Throwable t) { + LOG.error("Unexpected error in thread: {}, message: {}", Thread.currentThread().getName(), t.getMessage(), t); } finally { - if(handle != null) { + if (handle != null) { handle.releaseLocks(); } } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidCompactionHistoryService.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidCompactionHistoryService.java deleted file mode 100644 index e96a7ba289..0000000000 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidCompactionHistoryService.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore.txn; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.MetastoreTaskThread; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.TimeUnit; - -/** - * Purges obsolete items from compaction history data - */ -public class AcidCompactionHistoryService implements MetastoreTaskThread { - private static final Logger LOG = LoggerFactory.getLogger(AcidCompactionHistoryService.class); - - private Configuration conf; - private TxnStore txnHandler; - - @Override - public void setConf(Configuration configuration) { - this.conf = configuration; - txnHandler = TxnUtils.getTxnStore(conf); - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public long runFrequency(TimeUnit unit) { - return MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.COMPACTOR_HISTORY_REAPER_INTERVAL, - unit); - } - - @Override - public void run() { - TxnStore.MutexAPI.LockHandle handle = null; - try { - handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.CompactionHistory.name()); - long startTime = System.currentTimeMillis(); - txnHandler.purgeCompactionHistory(); - LOG.debug("History reaper reaper ran for " + (System.currentTimeMillis() - startTime)/1000 + - "seconds."); - } catch(Throwable t) { - LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); - } finally { - if(handle != null) { - handle.releaseLocks(); - } - } - } -} diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java index c4a488bac0..29e333bd57 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidHouseKeeperService.java @@ -17,14 +17,18 @@ */ package org.apache.hadoop.hive.metastore.txn; +import org.apache.commons.lang3.Functions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.MetastoreTaskThread; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.concurrent.TimeUnit; +import static org.apache.commons.lang3.Functions.FailableRunnable; + /** * Performs background tasks for Transaction management in Hive. * Runs inside Hive Metastore Service. @@ -48,7 +52,7 @@ public Configuration getConf() { @Override public long runFrequency(TimeUnit unit) { - return MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.TIMEDOUT_TXN_REAPER_INTERVAL, unit); + return MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.ACID_HOUSEKEEPER_SERVICE_INTERVAL, unit); } @Override @@ -56,16 +60,33 @@ public void run() { TxnStore.MutexAPI.LockHandle handle = null; try { handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.HouseKeeper.name()); - long startTime = System.currentTimeMillis(); - txnHandler.performTimeOuts(); - LOG.debug("timeout reaper ran for " + (System.currentTimeMillis() - startTime)/1000 + - "seconds."); - } catch(Throwable t) { - LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); + long houseKeeperStart = System.currentTimeMillis(); + performHouseKeeping(); + LOG.debug("Total time housekeeper thread took: {} seconds.", elapsedSince(houseKeeperStart)); + } catch (Throwable t) { + LOG.error("Unexpected error in thread: {}, message: {}", Thread.currentThread().getName(), t.getMessage(), t); } finally { - if(handle != null) { + if (handle != null) { handle.releaseLocks(); } } } + + private void performHouseKeeping() { + performTask(txnHandler::performTimeOuts, "Cleaning timed out txns and locks"); + performTask(txnHandler::performWriteSetGC, "Cleaning obsolete write set entries"); + performTask(txnHandler::purgeCompactionHistory, "Cleaning obsolete compaction history entries"); + performTask(txnHandler::cleanEmptyAbortedTxns, "Cleaning empty aborted txns"); + performTask(txnHandler::cleanTxnToWriteIdTable, "Cleaning obsolete TXN_TO_WRITE_ID entries"); + } + + private void performTask(FailableRunnable task, String description) { + long start = System.currentTimeMillis(); + Functions.run(task); + LOG.debug("{} took {} seconds.", description, elapsedSince(start)); + } + + private long elapsedSince(long start) { + return (System.currentTimeMillis() - start) / 1000; + } } diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidOpenTxnsCounterService.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidOpenTxnsCounterService.java index 2ad5a89f03..7ee27f294e 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidOpenTxnsCounterService.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidOpenTxnsCounterService.java @@ -30,6 +30,7 @@ */ public class AcidOpenTxnsCounterService implements MetastoreTaskThread { private static final Logger LOG = LoggerFactory.getLogger(AcidOpenTxnsCounterService.class); + private static final int ONE_MINUTE = 60 * 1000; private Configuration conf; private int isAliveCounter = 0; @@ -47,18 +48,21 @@ public void run() { long startTime = System.currentTimeMillis(); isAliveCounter++; txnHandler.countOpenTxns(); - if (System.currentTimeMillis() - lastLogTime > 60 * 1000) { - LOG.info("AcidOpenTxnsCounterService ran for " + - ((System.currentTimeMillis() - startTime) / 1000) + - " seconds. isAliveCounter = " + isAliveCounter); + if (shouldLog()) { + LOG.info("Open txn counter ran for {} seconds. isAliveCounter: {}", + (System.currentTimeMillis() - startTime) / 1000, isAliveCounter); lastLogTime = System.currentTimeMillis(); } } - catch(Throwable t) { - LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); + catch (Throwable t) { + LOG.error("Unexpected error in thread: {}, message: {}", Thread.currentThread().getName(), t.getMessage(), t); } } + private boolean shouldLog() { + return System.currentTimeMillis() - lastLogTime > ONE_MINUTE; + } + @Override public void setConf(Configuration configuration) { conf = configuration; diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidWriteSetService.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidWriteSetService.java deleted file mode 100644 index 5ec513dfd4..0000000000 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidWriteSetService.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore.txn; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.MetastoreTaskThread; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.TimeUnit; - -/** - * Periodically cleans WriteSet tracking information used in Transaction management - */ -public class AcidWriteSetService implements MetastoreTaskThread { - private static final Logger LOG = LoggerFactory.getLogger(AcidWriteSetService.class); - - private Configuration conf; - private TxnStore txnHandler; - - @Override - public void setConf(Configuration configuration) { - this.conf = configuration; - txnHandler = TxnUtils.getTxnStore(conf); - } - - @Override - public Configuration getConf() { - return conf; - } - - @Override - public long runFrequency(TimeUnit unit) { - return MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.WRITE_SET_REAPER_INTERVAL, unit); - } - - @Override - public void run() { - TxnStore.MutexAPI.LockHandle handle = null; - try { - handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.WriteSetCleaner.name()); - long startTime = System.currentTimeMillis(); - txnHandler.performWriteSetGC(); - LOG.debug("cleaner ran for " + (System.currentTimeMillis() - startTime)/1000 + "seconds."); - } catch(Throwable t) { - LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t); - } finally { - if(handle != null) { - handle.releaseLocks(); - } - } - } -} diff --git standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java index 87130a519d..b083087f4e 100644 --- standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java +++ standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java @@ -47,8 +47,8 @@ public static final String TXN_KEY_START = "_meta"; enum MUTEX_KEY { - Initiator, Cleaner, HouseKeeper, CompactionHistory, CheckLock, - WriteSetCleaner, CompactionScheduler, WriteIdAllocator, MaterializationRebuild + Initiator, Cleaner, HouseKeeper, CheckLock, + CompactionScheduler, WriteIdAllocator, MaterializationRebuild } // Compactor states (Should really be enum) String INITIATED_RESPONSE = "initiated"; diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java index 9905a14983..7554d7deef 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java @@ -47,10 +47,8 @@ import org.apache.hadoop.hive.metastore.RuntimeStatsCleanerTask; import org.apache.hadoop.hive.metastore.events.EventCleanerTask; import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; -import org.apache.hadoop.hive.metastore.txn.AcidCompactionHistoryService; import org.apache.hadoop.hive.metastore.txn.AcidHouseKeeperService; import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService; -import org.apache.hadoop.hive.metastore.txn.AcidWriteSetService; @Category(MetastoreUnitTest.class) public class TestMetastoreConf { @@ -465,13 +463,9 @@ public void testClassNames() { EventCleanerTask.class.getName()); Assert.assertEquals(MetastoreConf.METASTORE_DELEGATION_MANAGER_CLASS, MetastoreDelegationTokenManager.class.getName()); - Assert.assertEquals(MetastoreConf.ACID_COMPACTION_HISTORY_SERVICE_CLASS, - AcidCompactionHistoryService.class.getName()); Assert.assertEquals(MetastoreConf.ACID_HOUSE_KEEPER_SERVICE_CLASS, AcidHouseKeeperService.class.getName()); Assert.assertEquals(MetastoreConf.ACID_OPEN_TXNS_COUNTER_SERVICE_CLASS, AcidOpenTxnsCounterService.class.getName()); - Assert.assertEquals(MetastoreConf.ACID_WRITE_SET_SERVICE_CLASS, - AcidWriteSetService.class.getName()); } }