Index: src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (revision 1234687) +++ src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (working copy) @@ -168,6 +168,12 @@ private List ignoreStatesRSOffline = Arrays.asList(new EventType[]{ EventType.RS_ZK_REGION_FAILED_OPEN, EventType.RS_ZK_REGION_CLOSED }); + boolean regionsToProcess = false; + + //Map holding all the regions which got processed while RIT is not + //populated during master failover. + private Map regionsProcessed = new HashMap(1); + /** * Constructs a new assignment manager. @@ -359,7 +365,6 @@ watcher.assignmentZNode); // Run through all regions. If they are not assigned and not in RIT, then // its a clean cluster startup, else its a failover. - boolean regionsToProcess = false; for (Map.Entry e: this.regions.entrySet()) { if (!e.getKey().isMetaTable() && e.getValue() != null) { @@ -381,7 +386,8 @@ // Process list of dead servers and regions in RIT. // See HBASE-4580 for more information. processDeadServersAndRecoverLostRegions(deadServers, nodes); - + regionsToProcess = false; + regionsProcessed.clear(); } else { // Fresh cluster startup. LOG.info("Clean cluster startup. Assigning userregions"); @@ -455,79 +461,101 @@ LOG.info("Processing region " + regionInfo.getRegionNameAsString() + " in state " + data.getEventType()); synchronized (regionsInTransition) { - switch (data.getEventType()) { - case M_ZK_REGION_CLOSING: - // If zk node of the region was updated by a live server skip this - // region and just add it into RIT. - if (isOnDeadServer(regionInfo, deadServers) && - (data.getOrigin() == null || !serverManager.isServerOnline(data.getOrigin()))) { - // If was on dead server, its closed now. Force to OFFLINE and this - // will get it reassigned if appropriate - forceOffline(regionInfo, data); - } else { - // Just insert region into RIT. - // If this never updates the timeout will trigger new assignment - regionsInTransition.put(encodedRegionName, new RegionState( - regionInfo, RegionState.State.CLOSING, - data.getStamp(), data.getOrigin())); - } - break; + RegionState regionState = regionsInTransition.get(encodedRegionName); + if (regionState == null + && regionsProcessed.get(encodedRegionName) == null) { + switch (data.getEventType()) { + case M_ZK_REGION_CLOSING: + // If zk node of the region was updated by a live server skip this + // region and just add it into RIT. + if (isOnDeadServer(regionInfo, deadServers) + && (data.getOrigin() == null || !serverManager + .isServerOnline(data.getOrigin()))) { + // If was on dead server, its closed now. Force to OFFLINE and this + // will get it reassigned if appropriate + forceOffline(regionInfo, data); + } else { + // Just insert region into RIT. + // If this never updates the timeout will trigger new assignment + regionsInTransition.put(encodedRegionName, new RegionState( + regionInfo, RegionState.State.CLOSING, data.getStamp(), data + .getOrigin())); + } + regionsProcessed.put(encodedRegionName, true); + break; - case RS_ZK_REGION_CLOSED: - case RS_ZK_REGION_FAILED_OPEN: - // Region is closed, insert into RIT and handle it - addToRITandCallClose(regionInfo, RegionState.State.CLOSED, data); - break; + case RS_ZK_REGION_CLOSED: + case RS_ZK_REGION_FAILED_OPEN: + // Region is closed, insert into RIT and handle it + addToRITandCallClose(regionInfo, RegionState.State.CLOSED, data); + regionsProcessed.put(encodedRegionName, true); + break; - case M_ZK_REGION_OFFLINE: - // Region is offline, insert into RIT and handle it like a closed - addToRITandCallClose(regionInfo, RegionState.State.OFFLINE, data); - break; + case M_ZK_REGION_OFFLINE: + // If zk node of the region was updated by a live server skip this + // region and just add it into RIT. + if (isOnDeadServer(regionInfo, deadServers) + && (data.getOrigin() == null || !serverManager + .isServerOnline(data.getOrigin()))) { + // Region is offline, insert into RIT and handle it like a closed + addToRITandCallClose(regionInfo, RegionState.State.OFFLINE, data); + } else { + regionsInTransition.put(encodedRegionName, new RegionState( + regionInfo, RegionState.State.PENDING_OPEN, data.getStamp(), + data.getOrigin())); + } + regionsProcessed.put(encodedRegionName, true); + break; - case RS_ZK_REGION_OPENING: - // TODO: Could check if it was on deadServers. If it was, then we could - // do what happens in TimeoutMonitor when it sees this condition. + case RS_ZK_REGION_OPENING: + // TODO: Could check if it was on deadServers. If it was, then we + // could + // do what happens in TimeoutMonitor when it sees this condition. - // Just insert region into RIT - // If this never updates the timeout will trigger new assignment - if (regionInfo.isMetaTable()) { + // Just insert region into RIT + // If this never updates the timeout will trigger new assignment + if (regionInfo.isMetaTable()) { + regionsInTransition.put(encodedRegionName, new RegionState( + regionInfo, RegionState.State.OPENING, data.getStamp(), data + .getOrigin())); + // If ROOT or .META. table is waiting for timeout monitor to assign + // it may take lot of time when the assignment.timeout.period is + // the default value which may be very long. We will not be able + // to serve any request during this time. + // So we will assign the ROOT and .META. region immediately. + processOpeningState(regionInfo); + break; + } regionsInTransition.put(encodedRegionName, new RegionState( regionInfo, RegionState.State.OPENING, data.getStamp(), data .getOrigin())); - // If ROOT or .META. table is waiting for timeout monitor to assign - // it may take lot of time when the assignment.timeout.period is - // the default value which may be very long. We will not be able - // to serve any request during this time. - // So we will assign the ROOT and .META. region immediately. - processOpeningState(regionInfo); + regionsProcessed.put(encodedRegionName, true); break; - } - regionsInTransition.put(encodedRegionName, new RegionState(regionInfo, - RegionState.State.OPENING, data.getStamp(), data.getOrigin())); - break; - case RS_ZK_REGION_OPENED: - // Region is opened, insert into RIT and handle it - regionsInTransition.put(encodedRegionName, new RegionState( - regionInfo, RegionState.State.OPEN, - data.getStamp(), data.getOrigin())); - ServerName sn = data.getOrigin() == null? null: data.getOrigin(); - // sn could be null if this server is no longer online. If - // that is the case, just let this RIT timeout; it'll be assigned - // to new server then. - if (sn == null) { - LOG.warn("Region in transition " + regionInfo.getEncodedName() + - " references a null server; letting RIT timeout so will be " + - "assigned elsewhere"); - } else if (!serverManager.isServerOnline(sn) - && (isOnDeadServer(regionInfo, deadServers) - || regionInfo.isMetaRegion() || regionInfo.isRootRegion())) { - forceOffline(regionInfo, data); - } else { - new OpenedRegionHandler(master, this, regionInfo, sn, expectedVersion) - .process(); + case RS_ZK_REGION_OPENED: + // Region is opened, insert into RIT and handle it + regionsInTransition.put(encodedRegionName, new RegionState( + regionInfo, RegionState.State.OPEN, data.getStamp(), data + .getOrigin())); + ServerName sn = data.getOrigin() == null ? null : data.getOrigin(); + // sn could be null if this server is no longer online. If + // that is the case, just let this RIT timeout; it'll be assigned + // to new server then. + if (sn == null) { + LOG.warn("Region in transition " + regionInfo.getEncodedName() + + " references a null server; letting RIT timeout so will be " + + "assigned elsewhere"); + } else if (!serverManager.isServerOnline(sn) + && (isOnDeadServer(regionInfo, deadServers) + || regionInfo.isMetaRegion() || regionInfo.isRootRegion())) { + forceOffline(regionInfo, data); + } else { + new OpenedRegionHandler(master, this, regionInfo, sn, + expectedVersion).process(); + } + regionsProcessed.put(encodedRegionName, true); + break; } - break; } } } @@ -606,6 +634,7 @@ */ private void handleRegion(final RegionTransitionData data, int expectedVersion) { synchronized(regionsInTransition) { + HRegionInfo hri = null; if (data == null || data.getOrigin() == null) { LOG.warn("Unexpected NULL input " + data); return; @@ -683,6 +712,15 @@ break; case M_ZK_REGION_CLOSING: + hri = handleRegionWhileFailOverInProgress(regionState, encodedName, + data); + if (hri != null ) { + regionState = new RegionState(hri, RegionState.State.CLOSING, data + .getStamp(), data.getOrigin()); + regionsInTransition.put(encodedName, regionState); + regionsProcessed.put(encodedName, true); + break; + } // Should see CLOSING after we have asked it to CLOSE or additional // times after already being in state of CLOSING if (regionState == null || @@ -699,6 +737,18 @@ break; case RS_ZK_REGION_CLOSED: + hri = handleRegionWhileFailOverInProgress(regionState, encodedName, + data); + if (hri != null) { + regionState = new RegionState(hri, RegionState.State.CLOSED, data + .getStamp(), data.getOrigin()); + regionsInTransition.put(encodedName, regionState); + removeClosedRegion(regionState.getRegion()); + this.executorService.submit(new ClosedRegionHandler(master, this, + regionState.getRegion())); + regionsProcessed.put(encodedName, true); + break; + } // Should see CLOSED after CLOSING but possible after PENDING_CLOSE if (regionState == null || (!regionState.isPendingClose() && !regionState.isClosing())) { @@ -713,12 +763,23 @@ // what follows will fail because not in expected state. regionState.update(RegionState.State.CLOSED, data.getStamp(), data.getOrigin()); - removeClosedRegion(regionState.getRegion()); + removeClosedRegion(regionState.getRegion()); this.executorService.submit(new ClosedRegionHandler(master, this, regionState.getRegion())); break; case RS_ZK_REGION_FAILED_OPEN: + hri = handleRegionWhileFailOverInProgress(regionState, encodedName, + data); + if (hri != null) { + regionState = new RegionState(hri, RegionState.State.CLOSED, data + .getStamp(), data.getOrigin()); + regionsInTransition.put(encodedName, regionState); + this.executorService.submit(new ClosedRegionHandler(master, this, + regionState.getRegion())); + regionsProcessed.put(encodedName, true); + break; + } if (regionState == null || (!regionState.isPendingOpen() && !regionState.isOpening())) { LOG.warn("Received FAILED_OPEN for region " + prettyPrintedRegionName + @@ -734,6 +795,15 @@ break; case RS_ZK_REGION_OPENING: + hri = handleRegionWhileFailOverInProgress(regionState, encodedName, + data); + if (hri != null) { + regionState = new RegionState(hri, RegionState.State.OPENING, data + .getStamp(), data.getOrigin()); + regionsInTransition.put(encodedName, regionState); + regionsProcessed.put(encodedName, true); + break; + } // Should see OPENING after we have asked it to OPEN or additional // times after already being in state of OPENING if (regionState == null || @@ -751,6 +821,17 @@ break; case RS_ZK_REGION_OPENED: + hri = handleRegionWhileFailOverInProgress(regionState, encodedName, + data); + if (hri != null) { + regionState = new RegionState(hri, RegionState.State.OPEN, data + .getStamp(), data.getOrigin()); + regionsInTransition.put(encodedName, regionState); + this.executorService.submit(new OpenedRegionHandler(master, this, + regionState.getRegion(), data.getOrigin(), expectedVersion)); + regionsProcessed.put(encodedName, true); + break; + } // Should see OPENED after OPENING but possible after PENDING_OPEN if (regionState == null || (!regionState.isPendingOpen() && !regionState.isOpening())) { @@ -771,6 +852,29 @@ } } } + + private HRegionInfo handleRegionWhileFailOverInProgress( + RegionState regionState, String encodedName, RegionTransitionData data) { + if (regionState == null && regionsToProcess + && regionsProcessed.get(encodedName) == null) { + HRegionInfo hri = getHRegionInfo(data); + return hri; + } + return null; + } + + private HRegionInfo getHRegionInfo(RegionTransitionData data) { + HRegionInfo hri = null; + Pair p = null; + try { + p = MetaReader.getRegion(catalogTracker, data.getRegionName()); + hri = p.getFirst(); + return hri; + } catch (IOException e) { + master.abort("Aborting because error occoured while reading META.", e); + return null; + } + } /** * @return Returns true if this RegionState is splittable; i.e. the