Index: src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java (revision 1148622) +++ src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java (working copy) @@ -73,6 +73,7 @@ import org.apache.hadoop.hbase.filter.WhileMatchFilter; import org.apache.hadoop.hbase.filter.WritableByteArrayComparable; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.hbase.regionserver.wal.HLogKey; import org.apache.hadoop.hbase.util.Bytes; @@ -204,6 +205,8 @@ addToMap(KeyOnlyFilter.class, code++); + addToMap(RegionOpeningState.class, code++); + } private Class declaredClass; Index: src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java (revision 1148622) +++ src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java (working copy) @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.ipc.RemoteException; @@ -311,10 +312,17 @@ /** * Opens the specified region. - * @param region region to open + * + * @param region + * region to open + * @return RegionOpeningState + * OPENED - if region opened succesfully. + * ALREADY_OPENED - if the region was already opened. + * FAILED_OPENING - if region opening failed. + * * @throws IOException */ - public void openRegion(final HRegionInfo region) throws IOException; + public RegionOpeningState openRegion(final HRegionInfo region) throws IOException; /** * Opens the specified regions. Index: src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (revision 1148622) +++ src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (working copy) @@ -50,6 +50,7 @@ import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.Stoppable; +import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.catalog.CatalogTracker; import org.apache.hadoop.hbase.catalog.MetaReader; import org.apache.hadoop.hbase.catalog.RootLocationEditor; @@ -57,9 +58,11 @@ import org.apache.hadoop.hbase.executor.EventHandler.EventType; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.executor.RegionTransitionData; +import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.master.LoadBalancer.RegionPlan; import org.apache.hadoop.hbase.master.handler.ClosedRegionHandler; import org.apache.hadoop.hbase.master.handler.DisableTableHandler; +import org.apache.hadoop.hbase.master.handler.EnableTableHandler; import org.apache.hadoop.hbase.master.handler.OpenedRegionHandler; import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler; import org.apache.hadoop.hbase.util.Bytes; @@ -1040,7 +1043,34 @@ // Transition RegionState to PENDING_OPEN state.update(RegionState.State.PENDING_OPEN); // Send OPEN RPC. This can fail if the server on other end is is not up. - serverManager.sendRegionOpen(plan.getDestination(), state.getRegion()); + RegionOpeningState regionOpenState = serverManager.sendRegionOpen(plan + .getDestination(), state.getRegion()); + if (regionOpenState == RegionOpeningState.ALREADY_OPENED) { + // Remove region from in-memory transition and unassigned node from ZK + // While trying to enable the table the regions of the table were + // already enabled. + String encodedRegionName = state.getRegion().getEncodedName(); + try { + ZKAssign.deleteOfflineNode(master.getZooKeeper(), state.getRegion() + .getEncodedName()); + } catch (KeeperException.NoNodeException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("The unassigned node " + encodedRegionName + + " doesnot exist."); + } + } catch (KeeperException e) { + master.abort( + "Error deleting OFFLINED node in ZK for transition ZK node (" + + encodedRegionName + ")", e); + } + synchronized (this.regionsInTransition) { + this.regionsInTransition.remove(plan.getRegionInfo() + .getEncodedName()); + } + synchronized (this.regions) { + this.regions.put(plan.getRegionInfo(), plan.getDestination()); + } + } break; } catch (Throwable t) { LOG.warn("Failed assignment of " + @@ -1544,6 +1574,8 @@ new TreeMap>>(); // store all the disabling state table names Set disablingTables = new HashSet(1); + // store all the enabling state tablenames. + Set enablingTables = new HashSet(1); // Iterate regions in META for (Result result : results) { Pair region = @@ -1551,17 +1583,17 @@ if (region == null) continue; HServerInfo regionLocation = region.getSecond(); HRegionInfo regionInfo = region.getFirst(); - String disablingTableName = regionInfo.getTableDesc().getNameAsString(); + String tableName = regionInfo.getTableDesc().getNameAsString(); if (regionLocation == null) { // Region not being served, add to region map with no assignment // If this needs to be assigned out, it will also be in ZK as RIT - // add if the table is not in disabled state - if (false == checkIfRegionBelongsToDisabled(regionInfo)) { - this.regions.put(regionInfo, null); + // add if the table is not in disabled and enabling state + if (false == checkIfRegionBelongsToDisabled(regionInfo) + && false == checkIfRegionsBelongsToEnabling(regionInfo)) { + regions.put(regionInfo, regionLocation); } - if (checkIfRegionBelongsToDisabling(regionInfo)) { - disablingTables.add(disablingTableName); - } + addTheTablesInPartialState(disablingTables, enablingTables, regionInfo, + tableName); } else if (!serverManager.isServerOnline(regionLocation.getServerName())) { // Region is located on a server that isn't online List> offlineRegions = @@ -1573,23 +1605,51 @@ offlineRegions.add(new Pair(regionInfo, result)); } else { // Region is being served and on an active server - // add only if region not in disabled table - if (false == checkIfRegionBelongsToDisabled(regionInfo)) { + // add only if region not in disabled and enabling table + if (false == checkIfRegionBelongsToDisabled(regionInfo) + && false == checkIfRegionsBelongsToEnabling(regionInfo)) { regions.put(regionInfo, regionLocation); addToServers(regionLocation, regionInfo); } - if (checkIfRegionBelongsToDisabling(regionInfo)) { - disablingTables.add(disablingTableName); - } + addTheTablesInPartialState(disablingTables, enablingTables, regionInfo, + tableName); } } // Recover the tables that were not fully moved to DISABLED state. - // These tables are in DISABLING state when the master - // restarted/switched. + // These tables are in DISABLING state when the master restarted/switched. + boolean isWatcherCreated = recoverTableInDisablingState(disablingTables); + recoverTableInEnablingState(enablingTables, isWatcherCreated); + return offlineServers; + } + + private void addTheTablesInPartialState(Set disablingTables, + Set enablingTables, HRegionInfo regionInfo, + String disablingTableName) { + if (checkIfRegionBelongsToDisabling(regionInfo)) { + disablingTables.add(disablingTableName); + } else if (checkIfRegionsBelongsToEnabling(regionInfo)) { + enablingTables.add(disablingTableName); + } + } + + /** + * Recover the tables that were not fully moved to DISABLED state. These + * tables are in DISABLING state when the master restarted/switched. + * + * @param disablingTables + * @return + * @throws KeeperException + * @throws TableNotFoundException + * @throws IOException + */ + private boolean recoverTableInDisablingState(Set disablingTables) + throws KeeperException, TableNotFoundException, IOException { + boolean isWatcherCreated = false; if (disablingTables.size() != 0) { // Create a watcher on the zookeeper node ZKUtil.listChildrenAndWatchForNewChildren(watcher, watcher.assignmentZNode); + isWatcherCreated = true; for (String tableName : disablingTables) { // Recover by calling DisableTableHandler LOG.info("The table " + tableName @@ -1599,9 +1659,43 @@ catalogTracker, this).process(); } } - return offlineServers; + return isWatcherCreated; } - + + /** + * Recover the tables that are not fully moved to ENABLED state. These tables + * are in ENABLING state when the master restarted/switched + * + * @param enablingTables + * @param isWatcherCreated + * @throws KeeperException + * @throws TableNotFoundException + * @throws IOException + */ + private void recoverTableInEnablingState(Set enablingTables, + boolean isWatcherCreated) throws KeeperException, TableNotFoundException, + IOException { + if (enablingTables.size() != 0) { + if (false == isWatcherCreated) { + ZKUtil.listChildrenAndWatchForNewChildren(watcher, + watcher.assignmentZNode); + } + for (String tableName : enablingTables) { + // Recover by calling DisableTableHandler + LOG.info("The table " + tableName + + " is in ENABLING state. Hence recovering by moving the table" + + " to ENABLED state."); + new EnableTableHandler(this.master, tableName.getBytes(), + catalogTracker, this).process(); + } + } + } + + private boolean checkIfRegionsBelongsToEnabling(HRegionInfo regionInfo) { + String tableName = regionInfo.getTableDesc().getNameAsString(); + return getZKTable().isEnablingTable(tableName); + } + private boolean checkIfRegionBelongsToDisabled(HRegionInfo regionInfo) { String tableName = regionInfo.getTableDesc().getNameAsString(); return getZKTable().isDisabledTable(tableName); Index: src/main/java/org/apache/hadoop/hbase/master/ServerManager.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (revision 1148622) +++ src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (working copy) @@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.RetriesExhaustedException; import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler; import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler; import org.apache.hadoop.hbase.master.metrics.MasterMetrics; @@ -550,15 +551,15 @@ * @param server server to open a region * @param region region to open */ - public void sendRegionOpen(HServerInfo server, HRegionInfo region) + public RegionOpeningState sendRegionOpen(HServerInfo server, HRegionInfo region) throws IOException { HRegionInterface hri = getServerConnection(server); if (hri == null) { LOG.warn("Attempting to send OPEN RPC to server " + server.getServerName() + " failed because no RPC connection found to this server"); - return; + return RegionOpeningState.FAILED_OPENING; } - hri.openRegion(region); + return hri.openRegion(region); } /** Index: src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java (revision 1148622) +++ src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java (working copy) @@ -78,13 +78,7 @@ } final String encodedName = regionInfo.getEncodedName(); - // Check that this region is not already online HRegion region = this.rsServices.getFromOnlineRegions(encodedName); - if (region != null) { - LOG.warn("Attempted open of " + name + - " but already online on this server"); - return; - } // If fails, just return. Someone stole the region from under us. // Calling transitionZookeeperOfflineToOpening initalizes this.version. Index: src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (revision 1148622) +++ src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (working copy) @@ -105,6 +105,7 @@ import org.apache.hadoop.hbase.ipc.HBaseServer; import org.apache.hadoop.hbase.ipc.HMasterRegionInterface; import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.ipc.ServerNotRunningException; import org.apache.hadoop.hbase.regionserver.Leases.LeaseStillHeldException; import org.apache.hadoop.hbase.regionserver.handler.CloseMetaHandler; @@ -2102,8 +2103,15 @@ @Override @QosPriority(priority=HIGH_QOS) - public void openRegion(HRegionInfo region) + public RegionOpeningState openRegion(HRegionInfo region) throws IOException { + // Check that this region is not already online + HRegion onlineRegion = this.getFromOnlineRegions(region.getEncodedName()); + if (null != onlineRegion) { + LOG.warn("Attempted open of " + region.getEncodedName() + + " but already online on this server"); + return RegionOpeningState.ALREADY_OPENED; + } if (this.regionsInTransitionInRS.contains(region.getEncodedNameAsBytes())) { throw new RegionAlreadyInTransitionException("open", region.getEncodedName()); } @@ -2117,6 +2125,7 @@ } else { this.service.submit(new OpenRegionHandler(this, this, region)); } + return RegionOpeningState.OPENED; } @Override Index: src/main/java/org/apache/hadoop/hbase/regionserver/RegionOpeningState.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/RegionOpeningState.java (revision 0) +++ src/main/java/org/apache/hadoop/hbase/regionserver/RegionOpeningState.java (revision 0) @@ -0,0 +1,30 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +public enum RegionOpeningState { + + OPENED, + + ALREADY_OPENED, + + FAILED_OPENING; +} +