Index: src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java (revision 0) +++ src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java (revision 0) @@ -0,0 +1,125 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.catalog; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.ServerConnection; +import org.apache.hadoop.hbase.client.ServerConnectionManager; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Test {@link MetaReader}, {@link MetaEditor}, and {@link RootLocationEditor}. + */ +public class TestMetaReaderEditor { + private static final Log LOG = LogFactory.getLog(TestMetaReaderEditor.class); + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static ZooKeeperWatcher ZKW; + private static CatalogTracker CT; + private final static Abortable ABORTABLE = new Abortable() { + private final AtomicBoolean abort = new AtomicBoolean(false); + + @Override + public void abort(String why, Throwable e) { + LOG.info(why, e); + abort.set(true); + } + }; + + @BeforeClass public static void beforeClass() throws Exception { + UTIL.startMiniCluster(); + ZKW = new ZooKeeperWatcher(UTIL.getConfiguration(), + "TestMetaReaderEditor", ABORTABLE); + ServerConnection connection = + ServerConnectionManager.getConnection(UTIL.getConfiguration()); + CT = new CatalogTracker(ZKW, connection, ABORTABLE); + CT.start(); + } + + @AfterClass public static void afterClass() throws IOException { + UTIL.shutdownMiniCluster(); + } + + @Test public void testGetRegionsCatalogTables() + throws IOException, InterruptedException { + List regions = + MetaReader.getTableRegions(CT, HConstants.META_TABLE_NAME); + assertTrue(regions.size() >= 1); + assertTrue(MetaReader.getTableRegionsAndLocations(CT, + Bytes.toString(HConstants.META_TABLE_NAME)).size() >= 1); + assertTrue(MetaReader.getTableRegionsAndLocations(CT, + Bytes.toString(HConstants.ROOT_TABLE_NAME)).size() == 1); + } + + @Test public void testTableExists() throws IOException { + final String name = "testTableExists"; + final byte [] nameBytes = Bytes.toBytes(name); + assertFalse(MetaReader.tableExists(CT, name)); + UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY); + assertTrue(MetaReader.tableExists(CT, name)); + UTIL.getHBaseAdmin().disableTable(name); + UTIL.getHBaseAdmin().deleteTable(name); + assertFalse(MetaReader.tableExists(CT, name)); + assertTrue(MetaReader.tableExists(CT, + Bytes.toString(HConstants.META_TABLE_NAME))); + assertTrue(MetaReader.tableExists(CT, + Bytes.toString(HConstants.ROOT_TABLE_NAME))); + } + + @Test public void testGetRegion() throws IOException, InterruptedException { + final String name = "testGetRegion"; + final byte [] nameBytes = Bytes.toBytes(name); + HTable t = UTIL.createTable(nameBytes, HConstants.CATALOG_FAMILY); + int regionCount = UTIL.createMultiRegions(t, HConstants.CATALOG_FAMILY); + + // Test it works getting a region from user table. + List regions = MetaReader.getTableRegions(CT, nameBytes); + assertEquals(regionCount, regions.size()); + Pair pair = + MetaReader.getRegion(CT, regions.get(0).getRegionName()); + assertEquals(regions.get(0).getEncodedName(), + pair.getFirst().getEncodedName()); + // Test get on non-existent region. + pair = MetaReader.getRegion(CT, Bytes.toBytes("nonexistent-region")); + assertNull(pair); + // Test it works getting a region from meta/root. + pair = + MetaReader.getRegion(CT, HRegionInfo.FIRST_META_REGIONINFO.getRegionName()); + assertEquals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName(), + pair.getFirst().getEncodedName()); + } +} Index: src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java (revision 992025) +++ src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java (working copy) @@ -222,6 +222,8 @@ public void compact(byte[] tableNameOrRegionName) throws IOError { try{ admin.compact(tableNameOrRegionName); + } catch (InterruptedException e) { + throw new IOError(e.getMessage()); } catch (IOException e) { throw new IOError(e.getMessage()); } @@ -230,6 +232,8 @@ public void majorCompact(byte[] tableNameOrRegionName) throws IOError { try{ admin.majorCompact(tableNameOrRegionName); + } catch (InterruptedException e) { + throw new IOError(e.getMessage()); } catch (IOException e) { throw new IOError(e.getMessage()); } Index: src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java (revision 992025) +++ src/main/java/org/apache/hadoop/hbase/avro/AvroServer.java (working copy) @@ -312,6 +312,10 @@ try { admin.flush(Bytes.toBytes(table)); return null; + } catch (InterruptedException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; } catch (IOException e) { AIOError ioe = new AIOError(); ioe.message = new Utf8(e.getMessage()); @@ -324,6 +328,10 @@ try { admin.split(Bytes.toBytes(table)); return null; + } catch (InterruptedException e) { + AIOError ioe = new AIOError(); + ioe.message = new Utf8(e.getMessage()); + throw ioe; } catch (IOException e) { AIOError ioe = new AIOError(); ioe.message = new Utf8(e.getMessage()); Index: src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java (revision 992025) +++ src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java (working copy) @@ -104,7 +104,7 @@ } NavigableMap hris = - MetaReader.getServerRegions(this.server.getCatalogTracker(), this.hsi); + MetaReader.getServerUserRegions(this.server.getCatalogTracker(), this.hsi); LOG.info("Reassigning the " + hris.size() + " region(s) that " + serverName + " was carrying."); Index: src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java (revision 992025) +++ src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java (working copy) @@ -48,6 +48,7 @@ * be explicitly set. Instead, ZooKeeper is used to learn of the availability * and location of ROOT. ROOT is used to learn of the location of META. If not * available in ROOT, ZooKeeper is used to monitor for a new location of META. + *

Call {@link #start()} to start up operation. */ public class CatalogTracker { private static final Log LOG = LogFactory.getLog(CatalogTracker.class); @@ -80,6 +81,21 @@ * @throws IOException */ public CatalogTracker(final ZooKeeperWatcher zk, + final ServerConnection connection, final Abortable abortable) + throws IOException { + this(zk, connection, abortable, 0); + } + + /** + * Constructs the catalog tracker. Find current state of catalog tables and + * begin active tracking by executing {@link #start()}. + * @param zk + * @param connection server connection + * @param abortable if fatal exception + * @param defaultTimeout Timeout to use. + * @throws IOException + */ + public CatalogTracker(final ZooKeeperWatcher zk, final ServerConnection connection, final Abortable abortable, final int defaultTimeout) throws IOException { @@ -285,7 +301,7 @@ } if(getMetaServerConnection(true) == null) { throw new NotAllMetaRegionsOnlineException( - "Timed out (" + timeout + "ms"); + "Timed out (" + timeout + "ms)"); } return metaLocation; } Index: src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java (revision 992025) +++ src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java (working copy) @@ -24,14 +24,14 @@ import java.util.List; import java.util.Map; import java.util.NavigableMap; -import java.util.NavigableSet; import java.util.TreeMap; -import java.util.TreeSet; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; import org.apache.hadoop.hbase.HServerInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; @@ -47,7 +47,74 @@ * catalogs. */ public class MetaReader { + public static final byte [] META_REGION_PREFIX; + static { + // Copy the prefix from FIRST_META_REGIONINFO into META_REGION_PREFIX. + // FIRST_META_REGIONINFO == '.META.,,1'. META_REGION_PREFIX == '.META.,' + int len = HRegionInfo.FIRST_META_REGIONINFO.getRegionName().length - 2; + META_REGION_PREFIX = new byte [len]; + System.arraycopy(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), 0, + META_REGION_PREFIX, 0, len); + } + /** + * @param ct + * @param tableName A user tablename or a .META. table name. + * @return Interface on to server hosting the -ROOT- or + * .META. regions. + * @throws NotAllMetaRegionsOnlineException + * @throws IOException + */ + private static HRegionInterface getCatalogRegionInterface(final CatalogTracker ct, + final byte [] tableName) + throws NotAllMetaRegionsOnlineException, IOException { + return Bytes.equals(HConstants.META_TABLE_NAME, tableName)? + ct.waitForRootServerConnectionDefault(): + ct.waitForMetaServerConnectionDefault(); + } + + /** + * @param tableName + * @return Returns region name to look in for regions for tableName; + * e.g. if we are looking for .META. regions, we need to look + * in the -ROOT- region, else if a user table, we need to look + * in the .META. region. + */ + private static byte [] getCatalogRegionNameForTable(final byte [] tableName) { + return Bytes.equals(HConstants.META_TABLE_NAME, tableName)? + HRegionInfo.ROOT_REGIONINFO.getRegionName(): + HRegionInfo.FIRST_META_REGIONINFO.getRegionName(); + } + + /** + * @param regionName + * @return Returns region name to look in for regionName; + * e.g. if we are looking for .META.,,1 region, we need to look + * in -ROOT- region, else if a user region, we need to look + * in the .META.,,1 region. + */ + private static byte [] getCatalogRegionNameForRegion(final byte [] regionName) { + return isMetaRegion(regionName)? + HRegionInfo.ROOT_REGIONINFO.getRegionName(): + HRegionInfo.FIRST_META_REGIONINFO.getRegionName(); + } + + /** + * @param regionName + * @return True if regionName is from .META. table. + */ + private static boolean isMetaRegion(final byte [] regionName) { + if (regionName.length < META_REGION_PREFIX.length + 2 /* ',', + '1' */) { + // Can't be meta table region. + return false; + } + // Compare the prefix of regionName. If it matches META_REGION_PREFIX prefix, + // then this is region from .META. table. + return Bytes.compareTo(regionName, 0, META_REGION_PREFIX.length, + META_REGION_PREFIX, 0, META_REGION_PREFIX.length) == 0; + } + + /** * Performs a full scan of .META.. *

* Returns a map of every region to it's currently assigned server, according @@ -57,8 +124,7 @@ * @return map of regions to their currently assigned server * @throws IOException */ - public static Map fullScan( - CatalogTracker catalogTracker) + public static Map fullScan(CatalogTracker catalogTracker) throws IOException { HRegionInterface metaServer = catalogTracker.waitForMetaServerConnectionDefault(); @@ -105,6 +171,7 @@ public static HServerAddress readRegionLocation(CatalogTracker catalogTracker, byte [] regionName) throws IOException { + if (isMetaRegion(regionName)) throw new IllegalArgumentException("See readMetaLocation"); return readLocation(catalogTracker.waitForMetaServerConnectionDefault(), CatalogTracker.META_REGION, regionName); } @@ -155,14 +222,19 @@ throws IOException { Get get = new Get(regionName); get.addFamily(HConstants.CATALOG_FAMILY); - Result r = catalogTracker.waitForMetaServerConnectionDefault().get( - CatalogTracker.META_REGION, get); + byte [] meta = getCatalogRegionNameForRegion(regionName); + Result r = catalogTracker.waitForMetaServerConnectionDefault().get(meta, get); if(r == null || r.isEmpty()) { return null; } return metaRowToRegionPair(r); } + /** + * @param data A .META. table row. + * @return A pair of the regioninfo and the server address from data. + * @throws IOException + */ public static Pair metaRowToRegionPair( Result data) throws IOException { HRegionInfo info = Writables.getHRegionInfo( @@ -188,6 +260,11 @@ public static boolean tableExists(CatalogTracker catalogTracker, String tableName) throws IOException { + if (tableName.equals(HTableDescriptor.ROOT_TABLEDESC.getNameAsString()) || + tableName.equals(HTableDescriptor.META_TABLEDESC.getNameAsString())) { + // Catalog tables always exist. + return true; + } HRegionInterface metaServer = catalogTracker.waitForMetaServerConnectionDefault(); byte[] firstRowInTable = Bytes.toBytes(tableName + ",,"); @@ -213,24 +290,37 @@ } /** - * Gets all of the regions of the specified table from META. + * Gets all of the regions of the specified table. * @param catalogTracker * @param tableName - * @return + * @return Ordered list of {@link HRegionInfo}. * @throws IOException */ public static List getTableRegions(CatalogTracker catalogTracker, byte [] tableName) throws IOException { + if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) { + // If root, do a bit of special handling. + List list = new ArrayList(); + list.add(HRegionInfo.ROOT_REGIONINFO); + return list; + } else if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) { + // Same for .META. table + List list = new ArrayList(); + list.add(HRegionInfo.FIRST_META_REGIONINFO); + return list; + } + + // Its a user table. HRegionInterface metaServer = - catalogTracker.waitForMetaServerConnectionDefault(); + getCatalogRegionInterface(catalogTracker, tableName); List regions = new ArrayList(); String tableString = Bytes.toString(tableName); byte[] firstRowInTable = Bytes.toBytes(tableString + ",,"); Scan scan = new Scan(firstRowInTable); scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); - long scannerid = metaServer.openScanner( - HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan); + long scannerid = + metaServer.openScanner(getCatalogRegionNameForTable(tableName), scan); try { Result data; while((data = metaServer.next(scannerid)) != null) { @@ -253,16 +343,25 @@ public static List> getTableRegionsAndLocations(CatalogTracker catalogTracker, String tableName) - throws IOException { + throws IOException, InterruptedException { + byte [] tableNameBytes = Bytes.toBytes(tableName); + if (Bytes.equals(tableNameBytes, HConstants.ROOT_TABLE_NAME)) { + // If root, do a bit of special handling. + HServerAddress hsa = catalogTracker.getRootLocation(); + List> list = + new ArrayList>(); + list.add(new Pair(HRegionInfo.ROOT_REGIONINFO, hsa)); + return list; + } HRegionInterface metaServer = - catalogTracker.waitForMetaServerConnectionDefault(); + getCatalogRegionInterface(catalogTracker, tableNameBytes); List> regions = new ArrayList>(); byte[] firstRowInTable = Bytes.toBytes(tableName + ",,"); Scan scan = new Scan(firstRowInTable); scan.addFamily(HConstants.CATALOG_FAMILY); - long scannerid = metaServer.openScanner( - HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan); + long scannerid = + metaServer.openScanner(getCatalogRegionNameForTable(tableNameBytes), scan); try { Result data; while((data = metaServer.next(scannerid)) != null) { @@ -282,8 +381,15 @@ } } + /** + * @param catalogTracker + * @param hsi Server specification + * @return List of user regions installed on this server (does not include + * catalog regions). + * @throws IOException + */ public static NavigableMap - getServerRegions(CatalogTracker catalogTracker, final HServerInfo hsi) + getServerUserRegions(CatalogTracker catalogTracker, final HServerInfo hsi) throws IOException { HRegionInterface metaServer = catalogTracker.waitForMetaServerConnectionDefault(); Index: src/main/java/org/apache/hadoop/hbase/client/HConnection.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/client/HConnection.java (revision 992025) +++ src/main/java/org/apache/hadoop/hbase/client/HConnection.java (working copy) @@ -108,7 +108,7 @@ * lives in. * @param tableName name of the table row is in * @param row row key you're trying to find the region of - * @return HRegionLocation that describes where to find the reigon in + * @return HRegionLocation that describes where to find the region in * question * @throws IOException if a remote or network exception occurs */ Index: src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (revision 992025) +++ src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (working copy) @@ -740,8 +740,10 @@ * * @param tableNameOrRegionName table or region to flush * @throws IOException if a remote or network exception occurs + * @throws InterruptedException */ - public void flush(final String tableNameOrRegionName) throws IOException { + public void flush(final String tableNameOrRegionName) + throws IOException, InterruptedException { flush(Bytes.toBytes(tableNameOrRegionName)); } @@ -751,8 +753,10 @@ * * @param tableNameOrRegionName table or region to flush * @throws IOException if a remote or network exception occurs + * @throws InterruptedException */ - public void flush(final byte [] tableNameOrRegionName) throws IOException { + public void flush(final byte [] tableNameOrRegionName) + throws IOException, InterruptedException { boolean isRegionName = isRegionName(tableNameOrRegionName); if (isRegionName) { Pair pair = @@ -780,8 +784,10 @@ * * @param tableNameOrRegionName table or region to compact * @throws IOException if a remote or network exception occurs + * @throws InterruptedException */ - public void compact(final String tableNameOrRegionName) throws IOException { + public void compact(final String tableNameOrRegionName) + throws IOException, InterruptedException { compact(Bytes.toBytes(tableNameOrRegionName)); } @@ -791,8 +797,10 @@ * * @param tableNameOrRegionName table or region to compact * @throws IOException if a remote or network exception occurs + * @throws InterruptedException */ - public void compact(final byte [] tableNameOrRegionName) throws IOException { + public void compact(final byte [] tableNameOrRegionName) + throws IOException, InterruptedException { compact(tableNameOrRegionName, false); } @@ -802,9 +810,10 @@ * * @param tableNameOrRegionName table or region to major compact * @throws IOException if a remote or network exception occurs + * @throws InterruptedException */ public void majorCompact(final String tableNameOrRegionName) - throws IOException { + throws IOException, InterruptedException { majorCompact(Bytes.toBytes(tableNameOrRegionName)); } @@ -814,9 +823,10 @@ * * @param tableNameOrRegionName table or region to major compact * @throws IOException if a remote or network exception occurs + * @throws InterruptedException */ public void majorCompact(final byte [] tableNameOrRegionName) - throws IOException { + throws IOException, InterruptedException { compact(tableNameOrRegionName, true); } @@ -827,9 +837,10 @@ * @param tableNameOrRegionName table or region to compact * @param major True if we are to do a major compaction. * @throws IOException if a remote or network exception occurs + * @throws InterruptedException */ private void compact(final byte [] tableNameOrRegionName, final boolean major) - throws IOException { + throws IOException, InterruptedException { if (isRegionName(tableNameOrRegionName)) { Pair pair = MetaReader.getRegion(getCatalogTracker(), tableNameOrRegionName); @@ -882,8 +893,10 @@ * * @param tableNameOrRegionName table or region to split * @throws IOException if a remote or network exception occurs + * @throws InterruptedException */ - public void split(final String tableNameOrRegionName) throws IOException { + public void split(final String tableNameOrRegionName) + throws IOException, InterruptedException { split(Bytes.toBytes(tableNameOrRegionName)); } @@ -893,8 +906,9 @@ * * @param tableNameOrRegionName table to region to split * @throws IOException if a remote or network exception occurs + * @throws InterruptedException */ - public void split(final byte [] tableNameOrRegionName) throws IOException { + public void split(final byte [] tableNameOrRegionName) throws IOException, InterruptedException { if (isRegionName(tableNameOrRegionName)) { // Its a possible region name. Pair pair =