Index: src/test/org/apache/hadoop/hbase/TestEmptyMetaInfo.java =================================================================== --- src/test/org/apache/hadoop/hbase/TestEmptyMetaInfo.java (revision 0) +++ src/test/org/apache/hadoop/hbase/TestEmptyMetaInfo.java (revision 0) @@ -0,0 +1,76 @@ +/** + * Copyright 2008 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase; + +import java.io.IOException; + +import java.util.SortedMap; +import java.util.TreeMap; + +import org.apache.hadoop.io.Text; + +/** + * Tests master cleanup of rows in meta table where there is no HRegionInfo + */ +public class TestEmptyMetaInfo extends HBaseClusterTestCase { + /** + * Insert some bogus rows in meta. Master should clean them up. + * @throws IOException + */ + public void testEmptyMetaInfo() throws IOException { + HTable t = new HTable(conf, HConstants.META_TABLE_NAME); + for (int i = 0; i < 5; i++) { + Text regionName = new Text("tablename," + (i == 0 ? "" : i) + + System.currentTimeMillis()); + long lockid = t.startUpdate(regionName); + t.put(lockid, HConstants.COL_SERVER, + "localhost:1234".getBytes(HConstants.UTF8_ENCODING)); + t.commit(lockid); + } + long sleepTime = + conf.getLong("hbase.master.meta.thread.rescanfrequency", 10000); + int tries = conf.getInt("hbase.client.retries.number", 5); + int count = 0; + do { + tries -= 1; + try { + Thread.sleep(sleepTime); + } catch (InterruptedException e) { + // ignore + } + HScannerInterface scanner = + t.obtainScanner(HConstants.ALL_META_COLUMNS, new Text("tablename")); + + try { + count = 0; + HStoreKey key = new HStoreKey(); + SortedMap results = new TreeMap(); + while (scanner.next(key, results)) { + count += 1; + } + } finally { + scanner.close(); + } + } while (count != 0 && tries >= 0); + assertTrue(tries >= 0); + assertEquals(0, count); + } +} Index: src/java/org/apache/hadoop/hbase/HMaster.java =================================================================== --- src/java/org/apache/hadoop/hbase/HMaster.java (revision 635588) +++ src/java/org/apache/hadoop/hbase/HMaster.java (working copy) @@ -215,6 +215,7 @@ // scan we go check if parents can be removed. Map> splitParents = new HashMap>(); + List emptyRows = new ArrayList(); try { regionServer = connection.getHRegionConnection(region.getServer()); scannerId = @@ -229,10 +230,12 @@ } // TODO: Why does this have to be a sorted map? - SortedMap results = toRowMap(values).getMap(); + RowMap m = toRowMap(values); + SortedMap results = m.getMap(); HRegionInfo info = getHRegionInfo(results); if (info == null) { + emptyRows.add(m.getRow()); continue; } @@ -272,12 +275,20 @@ } } catch (IOException e) { LOG.error("Closing scanner", - RemoteExceptionHandler.checkIOException(e)); + RemoteExceptionHandler.checkIOException(e)); } } - // Scan is finished. Take a look at split parents to see if any we can - // clean up. + // Scan is finished. + + // First clean up any meta region rows which had null HRegionInfo's + + if (emptyRows.size() > 0) { + deleteEmptyMetaRows(regionServer, region.getRegionName(), emptyRows); + } + + // Take a look at split parents to see if any we can clean up. + if (splitParents.size() > 0) { for (Map.Entry> e: splitParents.entrySet()) { @@ -1263,6 +1274,7 @@ * HMasterRegionInterface */ + /** {@inheritDoc} */ @SuppressWarnings("unused") public HbaseMapWritable regionServerStartup(HServerInfo serverInfo) throws IOException { @@ -2002,8 +2014,9 @@ private void scanMetaRegion(HRegionInterface server, long scannerId, Text regionName) throws IOException { - ArrayList toDoList = new ArrayList(); - HashSet regions = new HashSet(); + List toDoList = new ArrayList(); + Set regions = new HashSet(); + List emptyRows = new ArrayList(); try { while (true) { HbaseMapWritable values = null; @@ -2040,6 +2053,7 @@ // Bingo! Found it. HRegionInfo info = getHRegionInfo(map); if (info == null) { + emptyRows.add(row); continue; } LOG.info(info.getRegionName() + " was on shutdown server <" + @@ -2099,6 +2113,12 @@ } } + // Scan complete. Remove any rows which had empty HRegionInfo + + if (emptyRows.size() > 0) { + deleteEmptyMetaRows(server, regionName, emptyRows); + } + // Update server in root/meta entries for (ToDoEntry e: toDoList) { if (e.deleteRegion) { @@ -2697,6 +2717,7 @@ server.openScanner(m.getRegionName(), COLUMN_FAMILY_ARRAY, tableName, System.currentTimeMillis(), null); + List emptyRows = new ArrayList(); try { while (true) { HbaseMapWritable values = server.next(scannerId); @@ -2707,6 +2728,7 @@ SortedMap map = rm.getMap(); HRegionInfo info = getHRegionInfo(map); if (info == null) { + emptyRows.add(rm.getRow()); throw new IOException(COL_REGIONINFO + " not found on " + rm.getRow()); } @@ -2734,6 +2756,12 @@ scannerId = -1L; } + // Get rid of any rows that have a null HRegionInfo + + if (emptyRows.size() > 0) { + deleteEmptyMetaRows(server, m.getRegionName(), emptyRows); + } + if (!tableExists) { throw new IOException(tableName + " does not exist"); } @@ -3120,20 +3148,28 @@ /* * Data structure used to return results out of the toRowMap method. */ - private class RowMap { + private static class RowMap { final Text row; final SortedMap map; - private RowMap(final Text r, final SortedMap m) { + /** + * Constructor + * + * @param r the row + * @param m the map of column names to values + */ + public RowMap(final Text r, final SortedMap m) { this.row = r; this.map = m; } - private Text getRow() { + /** @return the row */ + public Text getRow() { return this.row; } - private SortedMap getMap() { + /** @return the column value map */ + public SortedMap getMap() { return this.map; } } @@ -3185,6 +3221,29 @@ } return (HRegionInfo)Writables.getWritable(bytes, new HRegionInfo()); } + + /* + * When we find rows in a meta region that has an empty HRegionInfo, we + * clean them up here. + * + * @param server connection to server serving meta region + * @param metaRegionName name of the meta region we scanned + * @param emptyRows the row keys that had empty HRegionInfos + */ + protected void deleteEmptyMetaRows(HRegionInterface server, + Text metaRegionName, + List emptyRows) { + for (Text regionName: emptyRows) { + try { + HRegion.removeRegionFromMETA(server, metaRegionName, regionName); + LOG.warn("Removed region: " + regionName + " from meta region: " + + metaRegionName + " because HRegionInfo was empty"); + } catch (IOException e) { + LOG.error("deleting region: " + regionName + " from meta region: " + + metaRegionName, e); + } + } + } /* * Main program Index: src/java/org/apache/hadoop/hbase/HRegion.java =================================================================== --- src/java/org/apache/hadoop/hbase/HRegion.java (revision 635588) +++ src/java/org/apache/hadoop/hbase/HRegion.java (working copy) @@ -1844,22 +1844,17 @@ * Delete a region's meta information from the passed * meta region. * - * @param srvr META server to be updated + * @param server META server to be updated * @param metaRegionName Meta region name * @param regionNmae HRegion to remove from meta * * @throws IOException * @see {@link #addRegionToMETA(HRegion, HRegion)} */ - static void removeRegionFromMETA(final HRegionInterface srvr, + static void removeRegionFromMETA(final HRegionInterface server, final Text metaRegionName, final Text regionName) throws IOException { - BatchUpdate b = new BatchUpdate(rand.nextLong()); - long lockid = b.startUpdate(regionName); - for (int i = 0; i < ALL_META_COLUMNS.length; i++) { - b.delete(lockid, ALL_META_COLUMNS[i]); - } - srvr.batchUpdate(metaRegionName, System.currentTimeMillis(), b); + server.deleteAll(metaRegionName, regionName, LATEST_TIMESTAMP); } /**