From 4a84b486b8ebfc7814765f41b143bbb009fe227f Mon Sep 17 00:00:00 2001 From: jingyuntian Date: Wed, 28 Mar 2018 15:59:14 +0800 Subject: [PATCH] replicationUI --- .../hbase/tmpl/master/MasterStatusTmpl.jamon | 47 ++++++++++++ .../hbase/tmpl/master/RegionServerListTmpl.jamon | 88 +++++++++++++++++++--- .../org/apache/hadoop/hbase/master/HMaster.java | 26 +++++++ 3 files changed, 152 insertions(+), 9 deletions(-) diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon index 3091e18..9b292ea 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon @@ -31,8 +31,11 @@ AssignmentManager assignmentManager = null; <%import> java.util.*; java.io.IOException; +org.apache.hadoop.hbase.client.replication.ReplicationPeerConfigUtil; org.apache.hadoop.hbase.client.RegionInfo; org.apache.hadoop.hbase.client.TableDescriptor; +org.apache.hadoop.hbase.replication.ReplicationPeerConfig; +org.apache.hadoop.hbase.replication.ReplicationPeerDescription; org.apache.hadoop.hbase.HBaseConfiguration; org.apache.hadoop.hbase.HConstants; org.apache.hadoop.hbase.HTableDescriptor; @@ -259,6 +262,10 @@ AssignmentManager assignmentManager = master.getAssignmentManager(); +
+

Peers

+ <& peerConfigs &> +
<%if master.getAssignmentManager() != null %> <& AssignmentManagerStatusTmpl; assignmentManager=master.getAssignmentManager()&> @@ -564,3 +571,43 @@ AssignmentManager assignmentManager = master.getAssignmentManager(); + +<%def peerConfigs> +<%java> + List peers = null; + if (master.getReplicationPeerManager() != null) { + peers = master.getReplicationPeerManager().listPeers(null); + } + + + + + + + + + + +<%if (peers != null && peers.size() > 0)%> + <%for ReplicationPeerDescription peer : peers %> + <%java> + String peerId = peer.getPeerId(); + ReplicationPeerConfig peerConfig = peer.getPeerConfig(); + + + + + + + + + + + + +
Peer IdCluster KeyStateBandwidthNamespacesTable Cfs
<% peerId %><% peerConfig.getClusterKey() %><% peer.isEnabled() ? "ENABLED" : "DISABLED" %><% StringUtils.humanReadableInt(peerConfig.getBandwidth()) %> + <% peerConfig.getNamespaces() == null ? "" : ReplicationPeerConfigUtil.convertToString(peerConfig.getNamespaces()).replaceAll(";", "; ") %> + + <% peerConfig.getTableCFsMap() == null ? "" : ReplicationPeerConfigUtil.convertToString(peerConfig.getTableCFsMap()).replaceAll(";", "; ") %> +
Total: <% (peers != null) ? peers.size() : 0 %>
+ diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon index f353d32..c6c7fc3 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/RegionServerListTmpl.jamon @@ -26,11 +26,14 @@ HMaster master; <%import> java.util.*; org.apache.hadoop.hbase.master.HMaster; + org.apache.hadoop.hbase.procedure2.util.StringUtils; + org.apache.hadoop.hbase.replication.ReplicationLoadSource; org.apache.hadoop.hbase.RegionMetrics; org.apache.hadoop.hbase.ServerMetrics; org.apache.hadoop.hbase.ServerName; org.apache.hadoop.hbase.Size; org.apache.hadoop.hbase.util.VersionInfo; + org.apache.hadoop.hbase.util.Pair; org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; @@ -48,7 +51,8 @@ Arrays.sort(serverNames);
  • Memory
  • Requests
  • Storefiles
  • -
  • Compactions
  • +
  • Compactions
  • +
  • Replications
  • @@ -63,9 +67,12 @@ Arrays.sort(serverNames);
    <& storeStats; serverNames = serverNames; &>
    -
    +
    <& compactionStats; serverNames = serverNames; &>
    +
    + <& replicationStats; serverNames = serverNames; &> +
    @@ -111,7 +118,7 @@ Arrays.sort(serverNames); long startcode = serverName.getStartcode(); - <& serverNameLink; serverName=serverName; serverLoad = sl; &> + <& serverNameLink; serverName=serverName; &> <% new Date(startcode) %> <% TraditionalBinaryPrefix.long2String(lastContact, "s", 1) %> <% version %> @@ -158,7 +165,7 @@ for (ServerName serverName: serverNames) { } - <& serverNameLink; serverName=serverName; serverLoad = sl; &> + <& serverNameLink; serverName=serverName; &> <% TraditionalBinaryPrefix.long2String((long) sl.getUsedHeapSize().get(Size.Unit.MEGABYTE) * TraditionalBinaryPrefix.MEGA.value, "B", 1) %> <% TraditionalBinaryPrefix.long2String((long) sl.getMaxHeapSize().get(Size.Unit.MEGABYTE) @@ -206,7 +213,7 @@ if (sl != null) { } -<& serverNameLink; serverName=serverName; serverLoad = sl; &> +<& serverNameLink; serverName=serverName; &> <% sl.getRequestCountPerSecond() %> <% readRequestCount %> <% filteredReadRequestCount %> @@ -259,7 +266,7 @@ if (sl != null) { } -<& serverNameLink; serverName=serverName; serverLoad = sl; &> +<& serverNameLink; serverName=serverName; &> <% storeCount %> <% storeFileCount %> <% TraditionalBinaryPrefix.long2String( @@ -312,7 +319,7 @@ if (totalCompactingCells > 0) { } -<& serverNameLink; serverName=serverName; serverLoad = sl; &> +<& serverNameLink; serverName=serverName; &> <% totalCompactingCells %> <% totalCompactedCells %> <% totalCompactingCells - totalCompactedCells %> @@ -329,11 +336,74 @@ if (totalCompactingCells > 0) { +<%def replicationStats> +<%args> + ServerName [] serverNames; + +<%java> + HashMap>> replicationLoadSourceMap + = master.getReplicationLoad(serverNames); + List peers = null; + if (replicationLoadSourceMap != null && replicationLoadSourceMap.size() > 0){ + peers = new ArrayList<>(replicationLoadSourceMap.keySet()); + Collections.sort(peers); + } + + +<%if (replicationLoadSourceMap != null && replicationLoadSourceMap.size() > 0) %> + +
    + +
    + <%java> + active = "active"; + for (String peer : peers){ + +
    + + + + + + + + + <%for Pair pair: replicationLoadSourceMap.get(peer) %> + + + + + + + +
    ServerAgeOfLastShippedOpSizeOfLogQueueReplicationLag
    <& serverNameLink; serverName=pair.getFirst(); &><% StringUtils.humanTimeDiff(pair.getSecond().getAgeOfLastShippedOp()) %><% pair.getSecond().getSizeOfLogQueue() %><% StringUtils.humanTimeDiff(pair.getSecond().getReplicationLag()) %>
    +
    + <%java> + active = ""; + } + +
    +
    +<%else> +

    No Peers Metrics

    + + + + <%def serverNameLink> <%args> ServerName serverName; - ServerMetrics serverLoad; <%java> int infoPort = master.getRegionServerInfoPort(serverName); @@ -352,7 +422,7 @@ if (totalCompactingCells > 0) { ServerName serverName; - <& serverNameLink; serverName=serverName; serverLoad = null; &> + <& serverNameLink; serverName=serverName; &> diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index f5bd0de..6a6508d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -167,6 +167,7 @@ import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy; import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy; import org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy; import org.apache.hadoop.hbase.replication.ReplicationException; +import org.apache.hadoop.hbase.replication.ReplicationLoadSource; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; import org.apache.hadoop.hbase.replication.ReplicationUtils; @@ -3595,6 +3596,31 @@ public class HMaster extends HRegionServer implements MasterServices { return replicationPeerManager; } + public HashMap>> + getReplicationLoad(ServerName[] serverNames) { + List peerList = this.getReplicationPeerManager().listPeers(null); + if (peerList == null) { + return null; + } + HashMap>> replicationLoadSourceMap = + new HashMap<>(peerList.size()); + peerList.stream() + .forEach(peer -> replicationLoadSourceMap.put(peer.getPeerId(), new ArrayList())); + for (ServerName serverName : serverNames) { + List replicationLoadSources = getServerManager().getLoad(serverName).getReplicationLoadSourceList(); + for (ReplicationLoadSource replicationLoadSource : replicationLoadSources) { + replicationLoadSourceMap.get(replicationLoadSource.getPeerID()) + .add(new Pair<>(serverName, replicationLoadSource)); + } + } + for (List> loads : replicationLoadSourceMap.values()) { + if (loads.size() > 0) { + loads.sort(Comparator.comparingLong(load -> (-1) * load.getSecond().getReplicationLag())); + } + } + return replicationLoadSourceMap; + } + /** * This method modifies the master's configuration in order to inject replication-related features */ -- 2.7.4