From 2514bf0ccc64b09d9afb665307581a03efcc9e8b Mon Sep 17 00:00:00 2001 From: khemani Date: Mon, 30 Apr 2012 15:58:38 -0700 Subject: [PATCH] HBASE-5860 splitlogmanager should not unnecessarily resubmit tasks when zk unavailable --- .../hadoop/hbase/master/SplitLogManager.java | 23 +++++++++++++++++-- .../apache/hadoop/hbase/zookeeper/ZKSplitLog.java | 4 +++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java index 919c65f..3ec8320 100644 --- src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java +++ src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java @@ -626,7 +626,8 @@ public class SplitLogManager extends ZooKeeperListener { create(ZKSplitLog.getRescanNode(watcher), TaskState.TASK_DONE.get(serverName), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL, - new CreateRescanAsyncCallback(), Long.valueOf(retries)); + new CreateRescanAsyncCallback(), Long.valueOf(0)); + tot_mgr_node_create_rescan_queued.incrementAndGet(); } private void createRescanSuccess(String path) { @@ -924,7 +925,11 @@ public class SplitLogManager extends ZooKeeperListener { LOG.info("resubmitted " + resubmitted + " out of " + tot + " tasks"); } // If there are pending tasks and all of them have been unassigned for - // some time then put up a RESCAN node to ping the workers. + // some time then put up a RESCAN node to ping the workers. (Skip this + // processing if znode create is pending. Creation of any znode has + // the same effect as creation of a RESCAN node. They both ping the + // workers to look for new tasks) + // // ZKSplitlog.DEFAULT_UNASSIGNED_TIMEOUT is of the order of minutes // because a. it is very unlikely that every worker had a // transient error when trying to grab the task b. if there are no @@ -934,7 +939,7 @@ public class SplitLogManager extends ZooKeeperListener { // that there is always one worker in the system if (tot > 0 && !found_assigned_task && ((EnvironmentEdgeManager.currentTimeMillis() - lastNodeCreateTime) > - unassignedTimeout)) { + unassignedTimeout) && !isAnyCreateZNodePending()) { for (Map.Entry e : tasks.entrySet()) { String path = e.getKey(); Task task = e.getValue(); @@ -993,6 +998,17 @@ public class SplitLogManager extends ZooKeeperListener { } /** + * @return true if there is a pending create of a task node or a RESCAN node + * in zookeeper. It doesn't bother about being strict - there are small + * windows when it can return wrong results. + */ + static boolean isAnyCreateZNodePending() { + return ((tot_mgr_node_create_queued.get() > + tot_mgr_node_create_result.get()) || + (tot_mgr_node_create_rescan_queued.get() > + tot_mgr_node_create_rescan_result.get())); + } + /** * Asynchronous handler for zk get-data-set-watch on node results. * Retries on failures. */ @@ -1088,6 +1104,7 @@ public class SplitLogManager extends ZooKeeperListener { @Override public void processResult(int rc, String path, Object ctx, String name) { + tot_mgr_node_create_rescan_result.incrementAndGet(); if (rc != 0) { if (rc == KeeperException.Code.SESSIONEXPIRED.intValue()) { LOG.error("ZK session expired. Master is expected to shut down. Abandoning retries."); diff --git src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java index 30d7fe9..45b738e 100644 --- src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java +++ src/main/java/org/apache/hadoop/hbase/zookeeper/ZKSplitLog.java @@ -212,6 +212,10 @@ public class ZKSplitLog { public static AtomicLong tot_mgr_log_split_err = new AtomicLong(0); public static AtomicLong tot_mgr_node_create_queued = new AtomicLong(0); public static AtomicLong tot_mgr_node_create_result = new AtomicLong(0); + public static AtomicLong tot_mgr_node_create_rescan_queued = + new AtomicLong(0); + public static AtomicLong tot_mgr_node_create_rescan_result = + new AtomicLong(0); public static AtomicLong tot_mgr_node_already_exists = new AtomicLong(0); public static AtomicLong tot_mgr_node_create_err = new AtomicLong(0); public static AtomicLong tot_mgr_node_create_retry = new AtomicLong(0); -- 1.7.7.2