From 60a350aac47196c44988d73317b56311354c481e Mon Sep 17 00:00:00 2001 From: Guangxu Cheng Date: Tue, 5 Sep 2017 17:34:20 +0800 Subject: [PATCH] HBASE-18131 Add an hbase shell command to clear deadserver list in ServerManager --- .../java/org/apache/hadoop/hbase/client/Admin.java | 14 ++++++ .../hbase/client/ConnectionImplementation.java | 12 +++++ .../org/apache/hadoop/hbase/client/HBaseAdmin.java | 34 +++++++++++++ .../hbase/client/ShortCircuitMasterConnection.java | 13 +++++ .../hbase/shaded/protobuf/RequestConverter.java | 9 ++++ .../src/main/protobuf/Master.proto | 23 +++++++++ .../hadoop/hbase/coprocessor/MasterObserver.java | 24 +++++++++ .../org/apache/hadoop/hbase/master/DeadServer.java | 15 ++++++ .../hadoop/hbase/master/MasterCoprocessorHost.java | 40 +++++++++++++++ .../hadoop/hbase/master/MasterRpcServices.java | 57 ++++++++++++++++++++++ .../hbase/security/access/AccessController.java | 5 ++ .../apache/hadoop/hbase/master/TestDeadServer.java | 19 ++++++++ hbase-shell/src/main/ruby/hbase/admin.rb | 22 +++++++++ hbase-shell/src/main/ruby/shell.rb | 2 + .../main/ruby/shell/commands/clear_deadservers.rb | 42 ++++++++++++++++ .../main/ruby/shell/commands/list_deadservers.rb | 43 ++++++++++++++++ 16 files changed, 374 insertions(+) create mode 100644 hbase-shell/src/main/ruby/shell/commands/clear_deadservers.rb create mode 100644 hbase-shell/src/main/ruby/shell/commands/list_deadservers.rb diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index b19c107..e59f8a9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -2256,4 +2256,18 @@ public interface Admin extends Abortable, Closeable { */ void clearCompactionQueues(final ServerName sn, final Set queues) throws IOException, InterruptedException; + + /** + * List dead region servers. + * @return List of dead region servers. + */ + List listDeadServers() throws IOException; + + /** + * Clear dead region servers from master. + * @param servers Set of dead region servers. + * @throws IOException if a remote or network exception occurs + * @return True if the dead region server cleared. + */ + boolean clearDeadServers(final List servers) throws IOException; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java index fcd7c22..28a97f2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java @@ -1775,6 +1775,18 @@ class ConnectionImplementation implements ClusterConnection, Closeable { RpcController controller, GetQuotaStatesRequest request) throws ServiceException { return stub.getQuotaStates(controller, request); } + + @Override + public MasterProtos.ListDeadServersResponse listDeadServers(RpcController controller, + MasterProtos.ListDeadServersRequest request) throws ServiceException { + return stub.listDeadServers(controller, request); + } + + @Override + public MasterProtos.ClearDeadServersResponse clearDeadServers(RpcController controller, + MasterProtos.ClearDeadServersRequest request) throws ServiceException { + return stub.clearDeadServers(controller, request); + } }; } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index c699676..e13ba84 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -115,6 +115,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AbortProce import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AddColumnRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AddColumnResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AssignRegionRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ClearDeadServersRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ClearDeadServersResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateNamespaceRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateNamespaceResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateTableRequest; @@ -148,6 +150,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsProcedur import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsProcedureDoneResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSnapshotDoneResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDeadServersRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDrainingRegionServersRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListLocksRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListLocksResponse; @@ -4348,4 +4351,35 @@ public class HBaseAdmin implements Admin { }; ProtobufUtil.call(callable); } + + @Override + public List listDeadServers() throws IOException { + return executeCallable(new MasterCallable>(getConnection(), + getRpcControllerFactory()) { + @Override + public List rpcCall() throws ServiceException { + ListDeadServersRequest req = ListDeadServersRequest.newBuilder().build(); + List servers = new ArrayList<>(); + for (HBaseProtos.ServerName server : master.listDeadServers(null, req) + .getServerNameList()) { + servers.add(ProtobufUtil.toServerName(server)); + } + return servers; + } + }); + } + + @Override + public boolean clearDeadServers(List servers) throws IOException { + if (servers == null || servers.size() == 0) { + throw new IllegalArgumentException("servers cannot be null or empty"); + } + return executeCallable(new MasterCallable(getConnection(), getRpcControllerFactory()) { + @Override + protected Boolean rpcCall() throws Exception { + return master.clearDeadServers(getRpcController(), + RequestConverter.buildClearDeadServersRequest(servers)).getCleared(); + } + }); + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java index a8050d4..1cc8e7c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController; import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.*; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse; @@ -501,6 +502,18 @@ public class ShortCircuitMasterConnection implements MasterKeepAliveConnection { } @Override + public ClearDeadServersResponse clearDeadServers(RpcController controller, + ClearDeadServersRequest request) throws ServiceException { + return stub.clearDeadServers(controller, request); + } + + @Override + public ListDeadServersResponse listDeadServers(RpcController controller, + ListDeadServersRequest request) throws ServiceException { + return stub.listDeadServers(controller, request); + } + + @Override public SplitTableRegionResponse splitRegion(RpcController controller, SplitTableRegionRequest request) throws ServiceException { return stub.splitRegion(controller, request); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index e620a91..1af726e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -87,6 +87,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AddColumnRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.AssignRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.BalanceRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ClearDeadServersRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateNamespaceRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.CreateTableRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.DeleteColumnRequest; @@ -1811,6 +1812,14 @@ public final class RequestConverter { return builder.build(); } + public static ClearDeadServersRequest buildClearDeadServersRequest(List deadServers) { + ClearDeadServersRequest.Builder builder = ClearDeadServersRequest.newBuilder(); + for(ServerName server: deadServers) { + builder.addServerName(ProtobufUtil.toServerName(server)); + } + return builder.build(); + } + private static final GetSpaceQuotaRegionSizesRequest GET_SPACE_QUOTA_REGION_SIZES_REQUEST = GetSpaceQuotaRegionSizesRequest.newBuilder().build(); diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index 33f9bf3..58e59ef 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -622,6 +622,21 @@ message RemoveDrainFromRegionServersRequest { message RemoveDrainFromRegionServersResponse { } +message ListDeadServersRequest { +} + +message ListDeadServersResponse { + repeated ServerName server_name = 1; +} + +message ClearDeadServersRequest { + repeated ServerName server_name = 1; +} + +message ClearDeadServersResponse { + required bool cleared = 1; +} + service MasterService { /** Used by the client to get the number of regions that have received the updated schema */ rpc GetSchemaAlterStatus(GetSchemaAlterStatusRequest) @@ -970,4 +985,12 @@ service MasterService { /** Fetches the Master's view of quotas */ rpc GetQuotaStates(GetQuotaStatesRequest) returns(GetQuotaStatesResponse); + + /** clear dead servers from master*/ + rpc ClearDeadServers(ClearDeadServersRequest) + returns(ClearDeadServersResponse); + + /** Returns a list of Dead Servers. */ + rpc ListDeadServers(ListDeadServersRequest) + returns(ListDeadServersResponse); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java index 8e368ba..1347034 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java @@ -1916,4 +1916,28 @@ public interface MasterObserver extends Coprocessor { */ default void postLockHeartbeat(ObserverContext ctx, LockProcedure proc, boolean keepAlive) throws IOException {} + + /** + * Called before list dead region servers. + */ + default void preListDeadServers(ObserverContext ctx) + throws IOException {} + + /** + * Called after list dead region servers. + */ + default void postListDeadServers(ObserverContext ctx) + throws IOException {} + + /** + * Called before clear dead region servers. + */ + default void preClearDeadServers(ObserverContext ctx) + throws IOException {} + + /** + * Called after clear dead region servers. + */ + default void postClearDeadServers(ObserverContext ctx) + throws IOException {} } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java index fc86254..093db67 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java @@ -202,4 +202,19 @@ public class DeadServer { return o1.getSecond().compareTo(o2.getSecond()); } }; + + /** + * remove the specified dead server + * @param deadServerName the dead server name + */ + public synchronized void removeDeadServer(final ServerName deadServerName) { + deadServers.remove(deadServerName); + } + + /** + * Clear all dead servers. + */ + public synchronized void clearAllDeadServers() { + deadServers.clear(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index 004f91d..0db65a7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -1878,6 +1878,46 @@ public class MasterCoprocessorHost }); } + public void preListDeadServers() throws IOException { + execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() { + @Override + public void call(MasterObserver oserver, ObserverContext ctx) + throws IOException { + oserver.preListDeadServers(ctx); + } + }); + } + + public void postListDeadServers() throws IOException { + execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() { + @Override + public void call(MasterObserver oserver, ObserverContext ctx) + throws IOException { + oserver.postListDeadServers(ctx); + } + }); + } + + public void preClearDeadServers() throws IOException { + execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() { + @Override + public void call(MasterObserver oserver, ObserverContext ctx) + throws IOException { + oserver.preClearDeadServers(ctx); + } + }); + } + + public void postClearDeadServers() throws IOException { + execOperation(coprocessors.isEmpty() ? null : new CoprocessorOperation() { + @Override + public void call(MasterObserver oserver, ObserverContext ctx) + throws IOException { + oserver.postClearDeadServers(ctx); + } + }); + } + private static ImmutableHTableDescriptor toImmutableHTableDescriptor(TableDescriptor desc) { return new ImmutableHTableDescriptor(desc); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 3ec2c45..8af5ac5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -2013,4 +2013,61 @@ public class MasterRpcServices extends RSRpcServices throw new ServiceException(e); } } + + @Override + public ListDeadServersResponse listDeadServers(RpcController controller, + ListDeadServersRequest request) throws ServiceException { + + LOG.debug(master.getClientIdAuditPrefix() + " list dead region servers."); + ListDeadServersResponse.Builder response = ListDeadServersResponse.newBuilder(); + try { + master.checkInitialized(); + if (master.cpHost != null) { + master.cpHost.preListDeadServers(); + } + + Set servers = master.getServerManager().getDeadServers().copyServerNames(); + for (ServerName server : servers) { + response.addServerName(ProtobufUtil.toServerName(server)); + } + + if (master.cpHost != null) { + master.cpHost.postListDeadServers(); + } + } catch (IOException io) { + throw new ServiceException(io); + } + + return response.build(); + } + + @Override + public ClearDeadServersResponse clearDeadServers(RpcController controller, + ClearDeadServersRequest request) throws ServiceException { + LOG.debug(master.getClientIdAuditPrefix() + " clear dead region servers."); + ClearDeadServersResponse.Builder response = ClearDeadServersResponse.newBuilder(); + try { + master.checkInitialized(); + if (master.cpHost != null) { + master.cpHost.preClearDeadServers(); + } + + if (master.getServerManager().areDeadServersInProgress()) { + LOG.debug("Some dead server is still under processing, won't clear the dead server list"); + return response.setCleared(false).build(); + } + for (HBaseProtos.ServerName pbServer : request.getServerNameList()) { + master.getServerManager().getDeadServers() + .removeDeadServer(ProtobufUtil.toServerName(pbServer)); + } + + if (master.cpHost != null) { + master.cpHost.postClearDeadServers(); + } + } catch (IOException io) { + throw new ServiceException(io); + } + + return response.setCleared(true).build(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java index 3b7988e..3c03a91 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java @@ -1454,6 +1454,11 @@ public class AccessController implements MasterObserver, RegionObserver, RegionS requirePermission(getActiveUser(ctx), "split", tableName, null, null, Action.ADMIN); } + @Override + public void preClearDeadServers(ObserverContext ctx) throws IOException { + requirePermission(getActiveUser(ctx), "clearDeadServers", Action.ADMIN); + } + /* ---- RegionObserver implementation ---- */ @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java index fd18b6c..a529708 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDeadServer.java @@ -151,5 +151,24 @@ public class TestDeadServer { Assert.assertTrue(d.isEmpty()); } + @Test + public void testClearDeadServer(){ + DeadServer d = new DeadServer(); + d.add(hostname123); + d.add(hostname1234); + Assert.assertEquals(2, d.size()); + + d.clearAllDeadServers(); + Assert.assertTrue(d.isEmpty()); + d.add(hostname123); + Assert.assertEquals(1, d.size()); + + d.removeDeadServer(hostname123); + Assert.assertTrue(d.isEmpty()); + + d.removeDeadServer(hostname123_2); + Assert.assertTrue(d.isEmpty()); + } + } diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 2aacd7f..c4ec71b 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -1249,5 +1249,27 @@ module Hbase end @admin.clearCompactionQueues(ServerName.valueOf(server_name), queues) end + + #---------------------------------------------------------------------------------------------- + # clear dead region servers + def list_deadservers + @admin.listDeadServers.to_a + end + + #---------------------------------------------------------------------------------------------- + # clear dead region servers + def clear_deadservers(dead_servers) + # Flatten params array + dead_servers = dead_servers.flatten.compact + if dead_servers.empty? + servers = list_deadservers + else + servers = java.util.ArrayList.new + dead_servers.each do |s| + servers.add(ServerName.valueOf(s)) + end + end + @admin.clearDeadServers(servers) + end end end diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 469505f..759898b 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -358,6 +358,8 @@ Shell.load_command_group( splitormerge_switch splitormerge_enabled clear_compaction_queues + list_deadservers + clear_deadservers ], # TODO: remove older hlog_roll command aliases: { diff --git a/hbase-shell/src/main/ruby/shell/commands/clear_deadservers.rb b/hbase-shell/src/main/ruby/shell/commands/clear_deadservers.rb new file mode 100644 index 0000000..903159b --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/clear_deadservers.rb @@ -0,0 +1,42 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ClearDeadservers < Command + def help + <<-EOF + Clear the dead region servers that are never used. + Examples: + Clear all dead region servers: + hbase> clear_deadservers + Clear the specified dead region servers + hbase> clear_deadservers 'host187.example.com,60020,1289493121758' + or + hbase> clear_deadservers 'host187.example.com,60020,1289493121758', + 'host188.example.com,60020,1289493121758' + EOF + end + + def command(*dead_servers) + formatter.row([admin.clear_deadservers(dead_servers) ? 'true' : 'false']) + end + end + end +end diff --git a/hbase-shell/src/main/ruby/shell/commands/list_deadservers.rb b/hbase-shell/src/main/ruby/shell/commands/list_deadservers.rb new file mode 100644 index 0000000..5b67532 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/list_deadservers.rb @@ -0,0 +1,43 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListDeadservers < Command + def help + <<-EOF + List all dead region servers in hbase + Examples: + hbase> list_deadservers +EOF + end + + def command() + formatter.header(['SERVERNAME']) + + servers = admin.list_deadservers + servers.each do |server| + formatter.row([server.toString]) + end + + formatter.footer(servers.size) + end + end + end +end -- 1.9.5.msysgit.0