From b48a73004d0bdfaaa6ee193028ac408d8123fb17 Mon Sep 17 00:00:00 2001 From: Umesh Agashe Date: Tue, 23 May 2017 11:08:33 -0700 Subject: [PATCH] HBASE-18091 Added API for getting who currently holds a lock on namespace/ table/ region/ server and log messages when procedure needs to wait to acquire lock --- .../hbase/procedure2/ProcedureScheduler.java | 4 ++ .../hbase/procedure2/SimpleProcedureScheduler.java | 5 ++ .../master/procedure/MasterProcedureScheduler.java | 56 +++++++++++++++++++++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureScheduler.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureScheduler.java index b5295e7fb0708a932cafd8f1cc5ae27e5539aa12..93d0d5d8e515ad4e0b7207f1aeb4cbf767a36bff 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureScheduler.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureScheduler.java @@ -128,6 +128,10 @@ public interface ProcedureScheduler { List listLocks(); /** + * @return {@link LockInfo} for resource of specified type & name. null if resource is not locked. + */ + LockInfo getLockInfoForResource(LockInfo.ResourceType resourceType, String resourceName); + /** * Returns the number of elements in this queue. * @return the number of elements in this queue. */ diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SimpleProcedureScheduler.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SimpleProcedureScheduler.java index 176a9003f0af8161817c5ddcc58b7c5142440880..f3523ec0ac1f329f9b23364b8c39ea7d6d315084 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SimpleProcedureScheduler.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SimpleProcedureScheduler.java @@ -81,4 +81,9 @@ public class SimpleProcedureScheduler extends AbstractProcedureScheduler { public List listLocks() { return Collections.emptyList(); } + + @Override + public LockInfo getLockInfoForResource(LockInfo.ResourceType resourceType, String resourceName) { + return null; + } } \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java index b0baf858cd35cb2909e357e6e3463728e65eaf5e..15b557afaae71beb505aca7dd0722a983f049cb3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureScheduler.java @@ -339,6 +339,32 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { } @Override + public LockInfo getLockInfoForResource(LockInfo.ResourceType resourceType, String resourceName) { + LockAndQueue queue = null; + schedLock(); + try { + switch (resourceType) { + case SERVER: + queue = locking.serverLocks.get(ServerName.valueOf(resourceName)); + break; + case NAMESPACE: + queue = locking.namespaceLocks.get(resourceName); + break; + case TABLE: + queue = locking.tableLocks.get(TableName.valueOf(resourceName)); + break; + case REGION: + queue = locking.regionLocks.get(resourceName); + break; + } + + return queue != null ? createLockInfo(resourceType, resourceName, queue) : null; + } finally { + schedUnlock(); + } + } + + @Override public void clear() { schedLock(); try { @@ -586,6 +612,27 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { } /** + * Get lock info for a resource of specified type and name and log details + */ + protected void logLockInfoForResource(LockInfo.ResourceType resourceType, String resourceName) { + if (!LOG.isDebugEnabled()) { + return; + } + + LockInfo lockInfo = getLockInfoForResource(resourceType, resourceName); + if (lockInfo != null) { + String msg = resourceType.toString() + " '" + resourceName + "', shared lock count=" + + lockInfo.getSharedLockCount(); + + ProcedureInfo proc = lockInfo.getExclusiveLockOwnerProcedure(); + if (proc != null) { + msg += ", exclusively locked by procId=" + proc.getProcId(); + } + LOG.debug(msg); + } + } + + /** * Suspend the procedure if the specified table is already locked. * Other operations in the table-queue will be executed after the lock is released. * @param procedure the procedure trying to acquire the lock @@ -595,15 +642,18 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { public boolean waitTableExclusiveLock(final Procedure procedure, final TableName table) { schedLock(); try { - final LockAndQueue namespaceLock = locking.getNamespaceLock(table.getNamespaceAsString()); + final String namespace = table.getNamespaceAsString(); + final LockAndQueue namespaceLock = locking.getNamespaceLock(namespace); final LockAndQueue tableLock = locking.getTableLock(table); if (!namespaceLock.trySharedLock()) { waitProcedure(namespaceLock, procedure); + logLockInfoForResource(LockInfo.ResourceType.NAMESPACE, namespace); return true; } if (!tableLock.tryExclusiveLock(procedure)) { namespaceLock.releaseSharedLock(); waitProcedure(tableLock, procedure); + logLockInfoForResource(LockInfo.ResourceType.TABLE, table.getNameAsString()); return true; } removeFromRunQueue(tableRunQueue, getTableQueue(table)); @@ -856,6 +906,8 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { locking.getTableLock(TableName.NAMESPACE_TABLE_NAME); if (!systemNamespaceTableLock.trySharedLock()) { waitProcedure(systemNamespaceTableLock, procedure); + logLockInfoForResource(LockInfo.ResourceType.TABLE, + TableName.NAMESPACE_TABLE_NAME.getNameAsString()); return true; } @@ -863,6 +915,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { if (!namespaceLock.tryExclusiveLock(procedure)) { systemNamespaceTableLock.releaseSharedLock(); waitProcedure(namespaceLock, procedure); + logLockInfoForResource(LockInfo.ResourceType.NAMESPACE, namespace); return true; } return false; @@ -915,6 +968,7 @@ public class MasterProcedureScheduler extends AbstractProcedureScheduler { return false; } waitProcedure(lock, procedure); + logLockInfoForResource(LockInfo.ResourceType.SERVER, serverName.getServerName()); return true; } finally { schedUnlock(); -- 2.10.1 (Apple Git-78)