From 45928500a0da8745d7cbde79110ffb9adb0dbd3b Mon Sep 17 00:00:00 2001 From: stack Date: Sun, 16 Nov 2014 21:27:29 -0800 Subject: [PATCH] v7 edited --- .../java/org/apache/hadoop/hbase/client/Admin.java | 4 +- .../apache/hadoop/hbase/client/AsyncProcess.java | 22 +++--- .../hadoop/hbase/client/ConnectionManager.java | 2 +- .../hadoop/hbase/client/ConnectionUtils.java | 4 +- .../org/apache/hadoop/hbase/client/HBaseAdmin.java | 75 +++++++------------ .../org/apache/hadoop/hbase/client/MetaCache.java | 1 + .../apache/hadoop/hbase/client/RegionLocator.java | 2 +- .../apache/hadoop/hbase/zookeeper/ZKConfig.java | 21 +++--- .../hadoop/hbase/DistributedHBaseCluster.java | 39 +++++----- .../apache/hadoop/hbase/mapreduce/TableMapper.java | 2 +- .../hadoop/hbase/mapreduce/TableOutputFormat.java | 1 - .../hbase/master/RegionPlacementMaintainer.java | 45 ++++++----- .../apache/hadoop/hbase/master/ServerManager.java | 10 +-- .../hadoop/hbase/regionserver/HRegionServer.java | 3 +- .../hadoop/hbase/regionserver/SplitLogWorker.java | 4 - .../hbase/regionserver/wal/HLogSplitter.java | 5 +- .../java/org/apache/hadoop/hbase/HBaseCluster.java | 7 +- .../org/apache/hadoop/hbase/MiniHBaseCluster.java | 5 +- .../org/apache/hadoop/hbase/client/TestAdmin1.java | 1 - .../hbase/client/TestHBaseAdminNoCluster.java | 14 ++-- .../org/apache/hadoop/hbase/client/TestHCM.java | 5 +- .../hadoop/hbase/client/TestMultiParallel.java | 25 +++---- .../TestLoadIncrementalHFilesSplitRecovery.java | 86 ++++++++++++++++------ .../hbase/master/TestDistributedLogSplitting.java | 7 +- .../regionserver/TestScannerWithBulkload.java | 3 +- .../TestSplitTransactionOnCluster.java | 6 +- .../security/access/TestAccessController2.java | 5 +- 27 files changed, 210 insertions(+), 194 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index ae2b827..69b33d0 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -595,7 +595,7 @@ public interface Admin extends Abortable, Closeable { /** * Compact all regions on the region server - * @param regionserver the region server name + * @param sn the region server name * @param major if it's major compaction * @throws IOException * @throws InterruptedException @@ -1289,7 +1289,7 @@ public interface Admin extends Abortable, Closeable { * @return A RegionServerCoprocessorRpcChannel instance */ CoprocessorRpcChannel coprocessorService(ServerName sn); - + /** * Update the configuration and trigger an online config change diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java index 3806115..fe1906d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncProcess.java @@ -161,7 +161,7 @@ class AsyncProcess { // TODO: many of the fields should be made private protected final long id; - protected final ClusterConnection hConnection; + protected final ClusterConnection connection; protected final RpcRetryingCallerFactory rpcCallerFactory; protected final RpcControllerFactory rpcFactory; protected final BatchErrors globalErrors; @@ -244,7 +244,7 @@ class AsyncProcess { throw new IllegalArgumentException("HConnection cannot be null."); } - this.hConnection = hc; + this.connection = hc; this.pool = pool; this.globalErrors = useGlobalErrors ? new BatchErrors() : null; @@ -338,7 +338,7 @@ class AsyncProcess { new HashMap>(); List> retainedActions = new ArrayList>(rows.size()); - NonceGenerator ng = this.hConnection.getNonceGenerator(); + NonceGenerator ng = this.connection.getNonceGenerator(); long nonceGroup = ng.getNonceGroup(); // Currently, nonce group is per entire client. // Location errors that happen before we decide what requests to take. @@ -361,7 +361,7 @@ class AsyncProcess { try { if (r == null) throw new IllegalArgumentException("#" + id + ", row cannot be null"); // Make sure we get 0-s replica. - RegionLocations locs = hConnection.locateRegion( + RegionLocations locs = connection.locateRegion( tableName, r.getRow(), true, true, RegionReplicaUtil.DEFAULT_REPLICA_ID); if (locs == null || locs.isEmpty() || locs.getDefaultRegionLocation() == null) { throw new IOException("#" + id + ", no location found, aborting submit for" @@ -533,7 +533,7 @@ class AsyncProcess { // The position will be used by the processBatch to match the object array returned. int posInList = -1; - NonceGenerator ng = this.hConnection.getNonceGenerator(); + NonceGenerator ng = this.connection.getNonceGenerator(); for (Row r : rows) { posInList++; if (r instanceof Put) { @@ -908,7 +908,7 @@ class AsyncProcess { ", row cannot be null"); RegionLocations loc = null; try { - loc = hConnection.locateRegion( + loc = connection.locateRegion( tableName, action.getAction().getRow(), useCache, true, action.getReplicaId()); } catch (IOException ex) { manageLocationError(action, ex); @@ -1023,7 +1023,7 @@ class AsyncProcess { if (tableName == null) { // tableName is null when we made a cross-table RPC call. - hConnection.clearCaches(server); + connection.clearCaches(server); } int failed = 0, stopped = 0; List> toReplay = new ArrayList>(); @@ -1034,7 +1034,7 @@ class AsyncProcess { // any of the regions in the MultiAction. // TODO: depending on type of exception we might not want to update cache at all? if (tableName != null) { - hConnection.updateCachedLocations(tableName, regionName, row, null, server); + connection.updateCachedLocations(tableName, regionName, row, null, server); } for (Action action : e.getValue()) { Retry retry = manageError( @@ -1148,7 +1148,7 @@ class AsyncProcess { // Register corresponding failures once per server/once per region. if (!regionFailureRegistered) { regionFailureRegistered = true; - hConnection.updateCachedLocations( + connection.updateCachedLocations( tableName, regionName, row.getRow(), result, server); } if (failureCount == 0) { @@ -1197,7 +1197,7 @@ class AsyncProcess { errorsByServer.reportServerError(server); canRetry = errorsByServer.canRetryMore(numAttempt); } - hConnection.updateCachedLocations( + connection.updateCachedLocations( tableName, region, actions.get(0).getAction().getRow(), throwable, server); failureCount += actions.size(); @@ -1512,7 +1512,7 @@ class AsyncProcess { @VisibleForTesting protected MultiServerCallable createCallable(final ServerName server, TableName tableName, final MultiAction multi) { - return new MultiServerCallable(hConnection, tableName, server, this.rpcFactory, multi); + return new MultiServerCallable(connection, tableName, server, this.rpcFactory, multi); } /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java index b3a6295..5deef45 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionManager.java @@ -249,7 +249,7 @@ class ConnectionManager { */ @VisibleForTesting static NonceGenerator injectNonceGeneratorForTesting( - HConnection conn, NonceGenerator cnm) { + ClusterConnection conn, NonceGenerator cnm) { HConnectionImplementation connImpl = (HConnectionImplementation)conn; NonceGenerator ng = connImpl.getNonceGenerator(); LOG.warn("Nonce generator is being replaced by test code for " + cnm.getClass().getName()); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java index e26ae48..72b447a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionUtils.java @@ -79,7 +79,7 @@ public class ConnectionUtils { * @return old nonce generator. */ public static NonceGenerator injectNonceGeneratorForTesting( - HConnection conn, NonceGenerator cnm) { + ClusterConnection conn, NonceGenerator cnm) { return ConnectionManager.injectNonceGeneratorForTesting(conn, cnm); } @@ -111,7 +111,7 @@ public class ConnectionUtils { * @param client the client interface of the local server * @return an adapted/decorated HConnection */ - public static HConnection createShortCircuitHConnection(final HConnection conn, + public static HConnection createShortCircuitHConnection(final Connection conn, final ServerName serverName, final AdminService.BlockingInterface admin, final ClientService.BlockingInterface client) { return new ConnectionAdapter(conn) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index aec5ae8..3c808e8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -83,7 +83,6 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterReque import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.StopServerRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfigurationRequest; -import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription; @@ -175,8 +174,6 @@ public class HBaseAdmin implements Admin { private static final String ZK_IDENTIFIER_PREFIX = "hbase-admin-on-"; - // We use the implementation class rather then the interface because we - // need the package protected functions to get the connection to master private ClusterConnection connection; private volatile Configuration conf; @@ -1450,8 +1447,7 @@ public class HBaseAdmin implements Admin { * Get all the online regions on a region server. */ @Override - public List getOnlineRegions( - final ServerName sn) throws IOException { + public List getOnlineRegions(final ServerName sn) throws IOException { AdminService.BlockingInterface admin = this.connection.getAdmin(sn); return ProtobufUtil.getOnlineRegions(admin); } @@ -2340,12 +2336,6 @@ public class HBaseAdmin implements Admin { }); } - private HRegionLocation getFirstMetaServerForTable(final TableName tableName) - throws IOException { - return connection.locateRegion(TableName.META_TABLE_NAME, - HRegionInfo.createRegionName(tableName, null, HConstants.NINES, false)); - } - /** * @return Configuration used by the instance. */ @@ -2502,52 +2492,40 @@ public class HBaseAdmin implements Admin { /** * Check to see if HBase is running. Throw an exception if not. - * We consider that HBase is running if ZooKeeper and Master are running. - * * @param conf system configuration * @throws MasterNotRunningException if the master is not running * @throws ZooKeeperConnectionException if unable to connect to zookeeper */ + // Used by tests and by the Merge tool. Merge tool uses it to figure if HBase is up or not. public static void checkHBaseAvailable(Configuration conf) - throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException, IOException { + throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException, IOException { Configuration copyOfConf = HBaseConfiguration.create(conf); - // We set it to make it fail as soon as possible if HBase is not available copyOfConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1); copyOfConf.setInt("zookeeper.recovery.retry", 0); - - ConnectionManager.HConnectionImplementation connection - = (ConnectionManager.HConnectionImplementation) - HConnectionManager.getConnection(copyOfConf); - - try { - // Check ZK first. - // If the connection exists, we may have a connection to ZK that does - // not work anymore - ZooKeeperKeepAliveConnection zkw = null; - try { - zkw = connection.getKeepAliveZooKeeperWatcher(); - zkw.getRecoverableZooKeeper().getZooKeeper().exists( - zkw.baseZNode, false); - - } catch (IOException e) { - throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e); - } catch (InterruptedException e) { - throw (InterruptedIOException) + try (ClusterConnection connection = + (ClusterConnection)ConnectionFactory.createConnection(copyOfConf)) { + // Check ZK first. + // If the connection exists, we may have a connection to ZK that does not work anymore + ZooKeeperKeepAliveConnection zkw = null; + try { + // This is NASTY. FIX!!!! Dependent on internal implementation! TODO + zkw = ((ConnectionManager.HConnectionImplementation)connection). + getKeepAliveZooKeeperWatcher(); + zkw.getRecoverableZooKeeper().getZooKeeper().exists(zkw.baseZNode, false); + } catch (IOException e) { + throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e); + } catch (InterruptedException e) { + throw (InterruptedIOException) new InterruptedIOException("Can't connect to ZooKeeper").initCause(e); - } catch (KeeperException e) { - throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e); - } finally { - if (zkw != null) { - zkw.close(); + } catch (KeeperException e) { + throw new ZooKeeperConnectionException("Can't connect to ZooKeeper", e); + } finally { + if (zkw != null) { + zkw.close(); + } } - } - - // Check Master connection.isMasterRunning(); - - } finally { - connection.close(); } } @@ -3734,7 +3712,7 @@ public class HBaseAdmin implements Admin { public CoprocessorRpcChannel coprocessorService(ServerName sn) { return new RegionServerCoprocessorRpcChannel(connection, sn); } - + @Override public void updateConfiguration(ServerName server) throws IOException { try { @@ -3754,8 +3732,9 @@ public class HBaseAdmin implements Admin { @Override public int getMasterInfoPort() throws IOException { + // TODO: Fix! Reaching into internal implementation!!!! ConnectionManager.HConnectionImplementation connection = - (ConnectionManager.HConnectionImplementation) HConnectionManager.getConnection(conf); + (ConnectionManager.HConnectionImplementation)this.connection; ZooKeeperKeepAliveConnection zkw = connection.getKeepAliveZooKeeperWatcher(); try { return MasterAddressTracker.getMasterInfoPort(zkw); @@ -3763,4 +3742,4 @@ public class HBaseAdmin implements Admin { throw new IOException("Failed to get master info port from MasterAddressTracker", e); } } -} \ No newline at end of file +} diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaCache.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaCache.java index 4ecddb5..a49f95c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaCache.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetaCache.java @@ -63,6 +63,7 @@ public class MetaCache { * Search the cache for a location that fits our table and row key. * Return null if no suitable region is located. * + * * @param tableName * @param row * @return Null or region location found in cache. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java index 8168fe1..39518a6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionLocator.java @@ -30,7 +30,7 @@ import org.apache.hadoop.hbase.util.Pair; /** * Used to view region location information for a single HBase table. - * Obtain an instance from an {@link HConnection}. + * Obtain an instance from an {@link Connection}. * * @see ConnectionFactory * @see Connection diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java index 75b0175..3dc9aa6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKConfig.java @@ -84,16 +84,19 @@ public class ZKConfig { Properties zkProperties = new Properties(); // Directly map all of the hbase.zookeeper.property.KEY properties. - for (Entry entry : new Configuration(conf)) { // copy for mt safety - String key = entry.getKey(); - if (key.startsWith(HConstants.ZK_CFG_PROPERTY_PREFIX)) { - String zkKey = key.substring(HConstants.ZK_CFG_PROPERTY_PREFIX_LEN); - String value = entry.getValue(); - // If the value has variables substitutions, need to do a get. - if (value.contains(VARIABLE_START)) { - value = conf.get(key); + // Synchronize on conf so no loading of configs while we iterate + synchronized (conf) { + for (Entry entry : conf) { + String key = entry.getKey(); + if (key.startsWith(HConstants.ZK_CFG_PROPERTY_PREFIX)) { + String zkKey = key.substring(HConstants.ZK_CFG_PROPERTY_PREFIX_LEN); + String value = entry.getValue(); + // If the value has variables substitutions, need to do a get. + if (value.contains(VARIABLE_START)) { + value = conf.get(key); + } + zkProperties.put(zkKey, value); } - zkProperties.put(zkKey, value); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java index d97862d..ac20535 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/DistributedHBaseCluster.java @@ -25,9 +25,11 @@ import java.util.List; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterManager.ServiceType; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo; @@ -44,8 +46,8 @@ import com.google.common.collect.Sets; */ @InterfaceAudience.Private public class DistributedHBaseCluster extends HBaseCluster { - - private HBaseAdmin admin; + private Admin admin; + private final Connection connection; private ClusterManager clusterManager; @@ -53,7 +55,8 @@ public class DistributedHBaseCluster extends HBaseCluster { throws IOException { super(conf); this.clusterManager = clusterManager; - this.admin = new HBaseAdmin(conf); + this.connection = ConnectionFactory.createConnection(conf); + this.admin = this.connection.getAdmin(); this.initialClusterStatus = getClusterStatus(); } @@ -89,13 +92,13 @@ public class DistributedHBaseCluster extends HBaseCluster { @Override public AdminProtos.AdminService.BlockingInterface getAdminProtocol(ServerName serverName) throws IOException { - return admin.getConnection().getAdmin(serverName); + return ((ClusterConnection)this.connection).getAdmin(serverName); } @Override public ClientProtos.ClientService.BlockingInterface getClientProtocol(ServerName serverName) throws IOException { - return admin.getConnection().getClient(serverName); + return ((ClusterConnection)this.connection).getClient(serverName); } @Override @@ -138,8 +141,7 @@ public class DistributedHBaseCluster extends HBaseCluster { @Override public MasterService.BlockingInterface getMasterAdminService() throws IOException { - HConnection conn = HConnectionManager.getConnection(conf); - return conn.getMaster(); + return ((ClusterConnection)this.connection).getMaster(); } @Override @@ -183,18 +185,19 @@ public class DistributedHBaseCluster extends HBaseCluster { } @Override - public ServerName getServerHoldingRegion(byte[] regionName) throws IOException { - HConnection connection = admin.getConnection(); - HRegionLocation regionLoc = connection.locateRegion(regionName); + public ServerName getServerHoldingRegion(TableName tn, byte[] regionName) throws IOException { + HRegionLocation regionLoc = null; + try (RegionLocator locator = connection.getRegionLocator(tn)) { + regionLoc = locator.getRegionLocation(regionName); + } if (regionLoc == null) { - LOG.warn("Cannot find region server holding region " + Bytes.toString(regionName) - + " for table " + HRegionInfo.getTableName(regionName) + ", start key [" + - Bytes.toString(HRegionInfo.getStartKey(regionName)) + "]"); + LOG.warn("Cannot find region server holding region " + Bytes.toString(regionName) + + ", start key [" + Bytes.toString(HRegionInfo.getStartKey(regionName)) + "]"); return null; } AdminProtos.AdminService.BlockingInterface client = - connection.getAdmin(regionLoc.getServerName()); + ((ClusterConnection)this.connection).getAdmin(regionLoc.getServerName()); ServerInfo info = ProtobufUtil.getServerInfo(client); return ProtobufUtil.toServerName(info.getServerName()); } @@ -374,7 +377,7 @@ public class DistributedHBaseCluster extends HBaseCluster { } catch (IOException ioe) { LOG.warn("While closing the old connection", ioe); } - this.admin = new HBaseAdmin(conf); + this.admin = this.connection.getAdmin(); LOG.info("Added new HBaseAdmin"); return true; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapper.java index 0d9b65a..cde94fe 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapper.java @@ -37,4 +37,4 @@ import org.apache.hadoop.mapreduce.Mapper; public abstract class TableMapper extends Mapper { -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java index da40b2e..91bfaa2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java @@ -48,7 +48,6 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; * while the output value must be either a {@link Put} or a * {@link Delete} instance. * - *

is the type of the key. Ignored in this class. */ @InterfaceAudience.Public @InterfaceStability.Stable diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java index 1d16bdf..e7d52a2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionPlacementMaintainer.java @@ -48,9 +48,9 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper; import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; @@ -67,9 +67,9 @@ import org.apache.log4j.Logger; /** * A tool that is used for manipulating and viewing favored nodes information * for regions. Run with -h to get a list of the options - * */ @InterfaceAudience.Private +// TODO: Remove? Unused. Partially implemented only. public class RegionPlacementMaintainer { private static final Log LOG = LogFactory.getLog(RegionPlacementMaintainer.class .getName()); @@ -93,9 +93,9 @@ public class RegionPlacementMaintainer { private Configuration conf; private final boolean enforceLocality; private final boolean enforceMinAssignmentMove; - private HBaseAdmin admin; private RackManager rackManager; private Set targetTableSet; + private final Connection connection; public RegionPlacementMaintainer(Configuration conf) { this(conf, true, true); @@ -108,7 +108,13 @@ public class RegionPlacementMaintainer { this.enforceMinAssignmentMove = enforceMinAssignmentMove; this.targetTableSet = new HashSet(); this.rackManager = new RackManager(conf); + try { + this.connection = ConnectionFactory.createConnection(this.conf); + } catch (IOException e) { + throw new RuntimeException(e); + } } + private static void printHelp(Options opt) { new HelpFormatter().printHelp( "RegionPlacement < -w | -u | -n | -v | -t | -h | -overwrite -r regionName -f favoredNodes " + @@ -125,24 +131,13 @@ public class RegionPlacementMaintainer { } /** - * @return the cached HBaseAdmin - * @throws IOException - */ - private HBaseAdmin getHBaseAdmin() throws IOException { - if (this.admin == null) { - this.admin = new HBaseAdmin(this.conf); - } - return this.admin; - } - - /** * @return the new RegionAssignmentSnapshot * @throws IOException */ public SnapshotOfRegionAssignmentFromMeta getRegionAssignmentSnapshot() throws IOException { SnapshotOfRegionAssignmentFromMeta currentAssignmentShapshot = - new SnapshotOfRegionAssignmentFromMeta(HConnectionManager.getConnection(conf)); + new SnapshotOfRegionAssignmentFromMeta(ConnectionFactory.createConnection(conf)); currentAssignmentShapshot.initialize(); return currentAssignmentShapshot; } @@ -210,8 +205,10 @@ public class RegionPlacementMaintainer { // Get the all the region servers List servers = new ArrayList(); - servers.addAll(getHBaseAdmin().getClusterStatus().getServers()); - + try (Admin admin = this.connection.getAdmin()) { + servers.addAll(admin.getClusterStatus().getServers()); + } + LOG.info("Start to generate assignment plan for " + numRegions + " regions from table " + tableName + " with " + servers.size() + " region servers"); @@ -619,9 +616,9 @@ public class RegionPlacementMaintainer { // sort the map based on region info Map> assignmentMap = new TreeMap>(plan.getAssignmentMap()); - + for (Map.Entry> entry : assignmentMap.entrySet()) { - + String serverList = FavoredNodeAssignmentHelper.getFavoredNodesAsString(entry.getValue()); String regionName = entry.getKey().getRegionNameAsString(); LOG.info("Region: " + regionName ); @@ -660,7 +657,6 @@ public class RegionPlacementMaintainer { // Get the region to region server map Map> currentAssignment = this.getRegionAssignmentSnapshot().getRegionServerToRegionMap(); - HConnection connection = this.getHBaseAdmin().getConnection(); // track of the failed and succeeded updates int succeededNum = 0; @@ -691,10 +687,11 @@ public class RegionPlacementMaintainer { } if (singleServerPlan != null) { // Update the current region server with its updated favored nodes - BlockingInterface currentRegionServer = connection.getAdmin(entry.getKey()); + BlockingInterface currentRegionServer = + ((ClusterConnection)this.connection).getAdmin(entry.getKey()); UpdateFavoredNodesRequest request = RequestConverter.buildUpdateFavoredNodesRequest(regionUpdateInfos); - + UpdateFavoredNodesResponse updateFavoredNodesResponse = currentRegionServer.updateFavoredNodes(null, request); LOG.info("Region server " + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index cca39f5..369362b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -36,7 +36,6 @@ import java.util.concurrent.CopyOnWriteArrayList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClockOutOfSyncException; import org.apache.hadoop.hbase.HRegionInfo; @@ -46,8 +45,9 @@ import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.YouAreDeadException; import org.apache.hadoop.hbase.ZooKeeperConnectionException; -import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HConnectionManager; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.RetriesExhaustedException; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler; @@ -137,7 +137,7 @@ public class ServerManager { private final Server master; private final MasterServices services; - private final HConnection connection; + private final ClusterConnection connection; private final DeadServer deadservers = new DeadServer(); @@ -201,7 +201,7 @@ public class ServerManager { Configuration c = master.getConfiguration(); maxSkew = c.getLong("hbase.master.maxclockskew", 30000); warningSkew = c.getLong("hbase.master.warningclockskew", 10000); - this.connection = connect ? HConnectionManager.getConnection(c) : null; + this.connection = connect ? (ClusterConnection)ConnectionFactory.createConnection(c) : null; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 45e5558..7952a87 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -73,6 +73,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.YouAreDeadException; import org.apache.hadoop.hbase.ZNodeClearer; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.ConnectionUtils; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; @@ -609,7 +610,7 @@ public class HRegionServer extends HasThread implements */ protected HConnection createShortCircuitConnection() throws IOException { return ConnectionUtils.createShortCircuitHConnection( - HConnectionManager.getConnection(conf), serverName, rpcServices, rpcServices); + ConnectionFactory.createConnection(conf), serverName, rpcServices, rpcServices); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java index 94864ba..0052b00 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitLogWorker.java @@ -31,7 +31,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.Server; -import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.RetriesExhaustedException; import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager; import org.apache.hadoop.hbase.coordination.SplitLogWorkerCoordination; @@ -132,9 +131,6 @@ public class SplitLogWorker implements Runnable { try { LOG.info("SplitLogWorker " + server.getServerName() + " starting"); coordination.registerListener(); - // pre-initialize a new connection for splitlogworker configuration - HConnectionManager.getConnection(conf); - // wait for Coordination Engine is ready boolean res = false; while (!res && !coordination.isStop()) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java index 1f81874..20eb783 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java @@ -49,6 +49,7 @@ import java.util.concurrent.atomic.AtomicReference; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.protobuf.ServiceException; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -70,11 +71,11 @@ import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagRewriteCell; import org.apache.hadoop.hbase.TagType; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.ConnectionUtils; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager; @@ -1892,7 +1893,7 @@ public class HLogSplitter { synchronized (this.tableNameToHConnectionMap) { hconn = this.tableNameToHConnectionMap.get(tableName); if (hconn == null) { - hconn = HConnectionManager.getConnection(conf); + hconn = (HConnection)ConnectionFactory.createConnection(conf); this.tableNameToHConnectionMap.put(tableName, hconn); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java index 76a9566..9e7a0c4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseCluster.java @@ -250,15 +250,18 @@ public abstract class HBaseCluster implements Closeable, Configurable { * Get the ServerName of region server serving the first hbase:meta region */ public ServerName getServerHoldingMeta() throws IOException { - return getServerHoldingRegion(HRegionInfo.FIRST_META_REGIONINFO.getRegionName()); + return getServerHoldingRegion(TableName.META_TABLE_NAME, + HRegionInfo.FIRST_META_REGIONINFO.getRegionName()); } /** * Get the ServerName of region server serving the specified region * @param regionName Name of the region in bytes + * @param tn Table name that has the region. * @return ServerName that hosts the region or null */ - public abstract ServerName getServerHoldingRegion(byte[] regionName) throws IOException; + public abstract ServerName getServerHoldingRegion(final TableName tn, byte[] regionName) + throws IOException; /** * @return whether we are interacting with a distributed cluster as opposed to an diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java index a3ec004..7672ac1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService; @@ -517,7 +516,6 @@ public class MiniHBaseCluster extends HBaseCluster { if (this.hbaseCluster != null) { this.hbaseCluster.shutdown(); } - HConnectionManager.deleteAllConnections(false); } @Override @@ -657,7 +655,8 @@ public class MiniHBaseCluster extends HBaseCluster { } @Override - public ServerName getServerHoldingRegion(byte[] regionName) throws IOException { + public ServerName getServerHoldingRegion(final TableName tn, byte[] regionName) + throws IOException { // Assume there is only one master thread which is the active master. // If there are multiple master threads, the backup master threads // should hold some regions. Please refer to #countServedRegions diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java index 3d3b499..131adb6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java @@ -882,7 +882,6 @@ public class TestAdmin1 { admin.createTable(desc, splitKeys); HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName); Map regions = ht.getRegionLocations(); - ht.close(); assertEquals("Tried to create " + expectedRegions + " regions " + "but only found " + regions.size(), expectedRegions, regions.size()); // Disable table. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHBaseAdminNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHBaseAdminNoCluster.java index 7521bb7..73fffe3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHBaseAdminNoCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHBaseAdminNoCluster.java @@ -52,7 +52,7 @@ public class TestHBaseAdminNoCluster { * @throws ServiceException */ @Test - public void testMasterMonitorCollableRetries() + public void testMasterMonitorCallableRetries() throws MasterNotRunningException, ZooKeeperConnectionException, IOException, ServiceException { Configuration configuration = HBaseConfiguration.create(); // Set the pause and retry count way down. @@ -61,20 +61,18 @@ public class TestHBaseAdminNoCluster { configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, count); // Get mocked connection. Getting the connection will register it so when HBaseAdmin is // constructed with same configuration, it will find this mocked connection. - HConnection connection = HConnectionTestingUtility.getMockedConnection(configuration); + ClusterConnection connection = HConnectionTestingUtility.getMockedConnection(configuration); // Mock so we get back the master interface. Make it so when createTable is called, we throw // the PleaseHoldException. - MasterKeepAliveConnection masterAdmin = - Mockito.mock(MasterKeepAliveConnection.class); + MasterKeepAliveConnection masterAdmin = Mockito.mock(MasterKeepAliveConnection.class); Mockito.when(masterAdmin.createTable((RpcController)Mockito.any(), (CreateTableRequest)Mockito.any())). thenThrow(new ServiceException("Test fail").initCause(new PleaseHoldException("test"))); Mockito.when(connection.getKeepAliveMasterService()).thenReturn(masterAdmin); - // Mock up our admin Interfaces - Admin admin = new HBaseAdmin(configuration); + Admin admin = new HBaseAdmin(connection); try { HTableDescriptor htd = - new HTableDescriptor(TableName.valueOf("testMasterMonitorCollableRetries")); + new HTableDescriptor(TableName.valueOf("testMasterMonitorCollableRetries")); // Pass any old htable descriptor; not important try { admin.createTable(htd, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); @@ -87,7 +85,7 @@ public class TestHBaseAdminNoCluster { (CreateTableRequest)Mockito.any()); } finally { admin.close(); - if (connection != null)HConnectionManager.deleteConnection(configuration); + if (connection != null) connection.close(); } } } \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java index 3e29725..491a1a6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java @@ -132,6 +132,8 @@ public class TestHCM { @BeforeClass public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setBoolean(HConstants.STATUS_PUBLISHED, + HConstants.STATUS_PUBLISHED_DEFAULT); if (isJavaOk) { TEST_UTIL.getConfiguration().setBoolean(HConstants.STATUS_PUBLISHED, true); } @@ -1012,8 +1014,7 @@ public class TestHCM { HTable table = TEST_UTIL.createTable(TABLE_NAME3, FAM_NAM); TEST_UTIL.createMultiRegions(table, FAM_NAM); ConnectionManager.HConnectionImplementation conn = - (ConnectionManager.HConnectionImplementation) - HConnectionManager.getConnection(TEST_UTIL.getConfiguration()); + ( ConnectionManager.HConnectionImplementation)table.getConnection(); // We're now going to move the region and check that it works for the client // First a new put to add the location in the cache diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java index 20828ec..8849425 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMultiParallel.java @@ -33,7 +33,6 @@ import java.util.concurrent.ThreadPoolExecutor; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; @@ -42,14 +41,11 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.exceptions.OperationConflictException; -import org.apache.hadoop.hbase.ipc.RpcClient; -import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.testclassification.FlakeyTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Threads; -import org.apache.log4j.Level; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; @@ -71,6 +67,7 @@ public class TestMultiParallel { private static final byte [][] KEYS = makeKeys(); private static final int slaves = 5; // also used for testing HTable pool size + private static Connection CONNECTION; @BeforeClass public static void beforeClass() throws Exception { // Uncomment the following lines if more verbosity is needed for @@ -83,9 +80,11 @@ public class TestMultiParallel { UTIL.createMultiRegions(t, Bytes.toBytes(FAMILY)); UTIL.waitTableEnabled(TEST_TABLE); t.close(); + CONNECTION = ConnectionFactory.createConnection(UTIL.getConfiguration()); } @AfterClass public static void afterClass() throws Exception { + CONNECTION.close(); UTIL.shutdownMiniCluster(); } @@ -98,9 +97,6 @@ public class TestMultiParallel { // Wait until completing balance UTIL.waitFor(15 * 1000, UTIL.predicateNoRegionsInTransition()); } - HConnection conn = HConnectionManager.getConnection(UTIL.getConfiguration()); - conn.clearRegionCache(); - conn.close(); LOG.info("before done"); } @@ -330,7 +326,7 @@ public class TestMultiParallel { @Test (timeout=300000) public void testBatchWithPut() throws Exception { LOG.info("test=testBatchWithPut"); - Table table = new HTable(UTIL.getConfiguration(), TEST_TABLE); + Table table = CONNECTION.getTable(TEST_TABLE); // put multiple rows using a batch List puts = constructPutRequests(); @@ -349,9 +345,8 @@ public class TestMultiParallel { results = table.batch(puts); } catch (RetriesExhaustedWithDetailsException ree) { LOG.info(ree.getExhaustiveDescription()); - throw ree; - } finally { table.close(); + throw ree; } validateSizeAndEmpty(results, KEYS.length); } @@ -492,7 +487,8 @@ public class TestMultiParallel { @Test(timeout=300000) public void testNonceCollision() throws Exception { LOG.info("test=testNonceCollision"); - HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE); + final Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration()); + Table table = connection.getTable(TEST_TABLE); Put put = new Put(ONE_ROW); put.add(BYTES_FAMILY, QUALIFIER, Bytes.toBytes(0L)); @@ -511,8 +507,9 @@ public class TestMultiParallel { return nonce; } }; + NonceGenerator oldCnm = - ConnectionUtils.injectNonceGeneratorForTesting(table.getConnection(), cnm); + ConnectionUtils.injectNonceGeneratorForTesting((ClusterConnection)connection, cnm); // First test sequential requests. try { @@ -542,7 +539,7 @@ public class TestMultiParallel { public void run() { Table table = null; try { - table = new HTable(UTIL.getConfiguration(), TEST_TABLE); + table = connection.getTable(TEST_TABLE); } catch (IOException e) { fail("Not expected"); } @@ -575,7 +572,7 @@ public class TestMultiParallel { validateResult(result, QUALIFIER, Bytes.toBytes((numRequests / 2) + 1L)); table.close(); } finally { - ConnectionManager.injectNonceGeneratorForTesting(table.getConnection(), oldCnm); + ConnectionManager.injectNonceGeneratorForTesting((ClusterConnection)connection, oldCnm); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java index 47501d8..323d0e9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFilesSplitRecovery.java @@ -45,6 +45,8 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.HTable; @@ -232,10 +234,10 @@ public class TestLoadIncrementalHFilesSplitRecovery { * @throws IOException */ void assertExpectedTable(TableName table, int count, int value) throws IOException { + HTableDescriptor [] htds = util.getHBaseAdmin().listTables(table.getNameAsString()); + assertEquals(htds.length, 1); Table t = null; try { - assertEquals( - util.getHBaseAdmin().listTables(table.getNameAsString()).length, 1); t = new HTable(util.getConfiguration(), table); Scan s = new Scan(); ResultScanner sr = t.getScanner(s); @@ -444,30 +446,33 @@ public class TestLoadIncrementalHFilesSplitRecovery { public void testGroupOrSplitWhenRegionHoleExistsInMeta() throws Exception { TableName tableName = TableName.valueOf("testGroupOrSplitWhenRegionHoleExistsInMeta"); byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("row_00000100") }; - HTable table = new HTable(util.getConfiguration(), tableName); + // Share connection. We were failing to find the table with our new reverse scan because it + // looks for first region, not any region -- that is how it works now. The below removes first + // region in test. Was reliant on the Connection caching having first region. + Connection connection = ConnectionFactory.createConnection(util.getConfiguration()); + Table table = connection.getTable(tableName); setupTableWithSplitkeys(tableName, 10, SPLIT_KEYS); Path dir = buildBulkFiles(tableName, 2); final AtomicInteger countedLqis = new AtomicInteger(); - LoadIncrementalHFiles loader = new LoadIncrementalHFiles( - util.getConfiguration()) { - - protected List groupOrSplit( - Multimap regionGroups, - final LoadQueueItem item, final HTable htable, - final Pair startEndKeys) throws IOException { - List lqis = super.groupOrSplit(regionGroups, item, htable, startEndKeys); - if (lqis != null) { - countedLqis.addAndGet(lqis.size()); + LoadIncrementalHFiles loader = new LoadIncrementalHFiles(util.getConfiguration()) { + + protected List groupOrSplit( + Multimap regionGroups, + final LoadQueueItem item, final HTable htable, + final Pair startEndKeys) throws IOException { + List lqis = super.groupOrSplit(regionGroups, item, htable, startEndKeys); + if (lqis != null) { + countedLqis.addAndGet(lqis.size()); + } + return lqis; } - return lqis; - } - }; + }; // do bulkload when there is no region hole in hbase:meta. try { - loader.doBulkLoad(dir, table); + loader.doBulkLoad(dir, (HTable)table); } catch (Exception e) { LOG.error("exeception=", e); } @@ -477,18 +482,16 @@ public class TestLoadIncrementalHFilesSplitRecovery { dir = buildBulkFiles(tableName, 3); // Mess it up by leaving a hole in the hbase:meta - HConnection hConnection = HConnectionManager.getConnection(util.getConfiguration()); - List regionInfos = MetaTableAccessor.getTableRegions( - hConnection, tableName); + List regionInfos = MetaTableAccessor.getTableRegions(connection, tableName); for (HRegionInfo regionInfo : regionInfos) { if (Bytes.equals(regionInfo.getStartKey(), HConstants.EMPTY_BYTE_ARRAY)) { - MetaTableAccessor.deleteRegion(hConnection, regionInfo); + MetaTableAccessor.deleteRegion(connection, regionInfo); break; } } try { - loader.doBulkLoad(dir, table); + loader.doBulkLoad(dir, (HTable)table); } catch (Exception e) { LOG.error("exeception=", e); assertTrue("IOException expected", e instanceof IOException); @@ -496,7 +499,42 @@ public class TestLoadIncrementalHFilesSplitRecovery { table.close(); - this.assertExpectedTable(tableName, ROWCOUNT, 2); + // Make sure at least the one region that still exists can be found. + regionInfos = MetaTableAccessor.getTableRegions(connection, tableName); + assertTrue(regionInfos.size() >= 1); + + this.assertExpectedTable(connection, tableName, ROWCOUNT, 2); + connection.close(); } -} + /** + * Checks that all columns have the expected value and that there is the + * expected number of rows. + * @throws IOException + */ + void assertExpectedTable(final Connection connection, TableName table, int count, int value) + throws IOException { + HTableDescriptor [] htds = util.getHBaseAdmin().listTables(table.getNameAsString()); + assertEquals(htds.length, 1); + Table t = null; + try { + t = connection.getTable(table); + Scan s = new Scan(); + ResultScanner sr = t.getScanner(s); + int i = 0; + for (Result r : sr) { + i++; + for (NavigableMap nm : r.getNoVersionMap().values()) { + for (byte[] val : nm.values()) { + assertTrue(Bytes.equals(val, value(value))); + } + } + } + assertEquals(count, i); + } catch (IOException e) { + fail("Failed due to exception"); + } finally { + if (t != null) t.close(); + } + } +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java index 5321ae8..bc06521 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -67,6 +67,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.SplitLogCounters; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.client.ConnectionUtils; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; @@ -104,8 +105,6 @@ import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; @@ -332,7 +331,7 @@ public class TestDistributedLogSplitting { HTable ht = installTable(zkw, TABLE_NAME, FAMILY_NAME, NUM_REGIONS_TO_CREATE); NonceGeneratorWithDups ng = new NonceGeneratorWithDups(); NonceGenerator oldNg = - ConnectionUtils.injectNonceGeneratorForTesting(ht.getConnection(), ng); + ConnectionUtils.injectNonceGeneratorForTesting((ClusterConnection)ht.getConnection(), ng); try { List reqs = new ArrayList(); @@ -366,7 +365,7 @@ public class TestDistributedLogSplitting { } } } finally { - ConnectionUtils.injectNonceGeneratorForTesting(ht.getConnection(), oldNg); + ConnectionUtils.injectNonceGeneratorForTesting((ClusterConnection) ht.getConnection(), oldNg); ht.close(); zkw.close(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java index 5593d80..49ded21 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerWithBulkload.java @@ -179,7 +179,6 @@ public class TestScannerWithBulkload { table.put(put1); table.flushCommits(); admin.flush(tableName); - admin.close(); put0 = new Put(Bytes.toBytes("row1")); put0.add(new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("col"), Bytes.toBytes("q"), l, Bytes .toBytes("version1"))); @@ -200,7 +199,7 @@ public class TestScannerWithBulkload { @Test public void testBulkLoadWithParallelScan() throws Exception { TableName tableName = TableName.valueOf("testBulkLoadWithParallelScan"); - final long l = System.currentTimeMillis(); + final long l = System.currentTimeMillis(); HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); createTable(admin, tableName); Scan scan = createScan(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java index 29073ed..678adc4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -107,8 +107,8 @@ import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; /** - * Like {@link TestSplitTransaction} in that we're testing {@link SplitTransaction} - * only the below tests are against a running cluster where {@link TestSplitTransaction} + * Like TestSplitTransaction in that we're testing {@link SplitTransaction} + * only the below tests are against a running cluster where TestSplitTransaction * is tests against a bare {@link HRegion}. */ @Category({RegionServerTests.class, LargeTests.class}) @@ -904,7 +904,7 @@ public class TestSplitTransactionOnCluster { fail("Each table should have at least one region."); } ServerName serverName = - cluster.getServerHoldingRegion(firstTableRegions.get(0).getRegionName()); + cluster.getServerHoldingRegion(firstTable, firstTableRegions.get(0).getRegionName()); admin.move(secondTableRegions.get(0).getRegionInfo().getEncodedNameAsBytes(), Bytes.toBytes(serverName.getServerName())); Table table1 = null; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController2.java index c41e977..01e8f82 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController2.java @@ -67,6 +67,9 @@ public class TestAccessController2 extends SecureTestUtil { TEST_UTIL.startMiniCluster(); // Wait for the ACL table to become available TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME); + String ensemble = TEST_UTIL.getConfiguration().get("hbase.zookeeper.quorum"); + int port = TEST_UTIL.getConfiguration().getInt("hbase.zookeeper.property.clientPort", -1); + } @AfterClass @@ -86,7 +89,7 @@ public class TestAccessController2 extends SecureTestUtil { public Object run() throws Exception { HTableDescriptor desc = new HTableDescriptor(TEST_TABLE.getTableName()); desc.addFamily(new HColumnDescriptor(TEST_FAMILY)); - Admin admin = new HBaseAdmin(conf); + Admin admin = new HBaseAdmin(TEST_UTIL.getConfiguration()); try { admin.createTable(desc); } finally { -- 1.8.5.2 (Apple Git-48)