From 51505c6ee54bdb5aea9dbce4ec5f055086e25d47 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Sun, 4 Nov 2018 00:49:57 +0000 Subject: [PATCH] HBASE-21406 inital version for branch-1 --- .../hadoop/hbase/protobuf/ProtobufUtil.java | 3 ++- .../hbase/replication/ReplicationLoadSink.java | 11 ++++++++++- .../MetricsReplicationSinkSource.java | 1 + .../MetricsReplicationSinkSourceImpl.java | 2 ++ .../src/main/protobuf/ClusterStatus.proto | 2 ++ .../replication/regionserver/MetricsSink.java | 15 +++++++++++++++ .../replication/regionserver/ReplicationLoad.java | 2 ++ hbase-shell/src/main/ruby/hbase/admin.rb | 12 +++++++++--- 8 files changed, 43 insertions(+), 5 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index e630b98a79..1fd6bc0e15 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -3489,7 +3489,8 @@ public final class ProtobufUtil { public static ReplicationLoadSink toReplicationLoadSink( ClusterStatusProtos.ReplicationLoadSink cls) { - return new ReplicationLoadSink(cls.getAgeOfLastAppliedOp(), cls.getTimeStampsOfLastAppliedOp()); + return new ReplicationLoadSink(cls.getAgeOfLastAppliedOp(), cls.getTimeStampsOfLastAppliedOp(), + cls.getTimeStampStarted(), cls.getTotalOpsProcessed()); } public static ReplicationLoadSource toReplicationLoadSource( diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java index 63fe334958..60c00b245e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationLoadSink.java @@ -19,10 +19,15 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; public class ReplicationLoadSink { private long ageOfLastAppliedOp; private long timeStampsOfLastAppliedOp; + private long timeStampStarted; + private long totalOpsProcessed; - public ReplicationLoadSink(long age, long timeStamp) { + public ReplicationLoadSink(long age, long timeStamp, long timeStampStarted, + long totalOpsProcessed) { this.ageOfLastAppliedOp = age; this.timeStampsOfLastAppliedOp = timeStamp; + this.timeStampStarted = timeStampStarted; + this.totalOpsProcessed = totalOpsProcessed; } public long getAgeOfLastAppliedOp() { @@ -33,4 +38,8 @@ public class ReplicationLoadSink { return this.timeStampsOfLastAppliedOp; } + public long getTimeStampStarted() { return timeStampStarted; } + + public long getTotalOpsProcessed() { return totalOpsProcessed; } + } \ No newline at end of file diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java index 9fb8415d0c..1191290493 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSource.java @@ -29,4 +29,5 @@ public interface MetricsReplicationSinkSource { void incrAppliedOps(long batchsize); long getLastAppliedOpAge(); void incrAppliedHFiles(long hfileSize); + long getSinkAppliedOps(); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java index 74592d9752..4f60dab3be 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsReplicationSinkSourceImpl.java @@ -57,4 +57,6 @@ public class MetricsReplicationSinkSourceImpl implements MetricsReplicationSinkS public void incrAppliedHFiles(long hfiles) { hfilesCounter.incr(hfiles); } + + @Override public long getSinkAppliedOps() { return opsCounter.value(); } } diff --git a/hbase-protocol/src/main/protobuf/ClusterStatus.proto b/hbase-protocol/src/main/protobuf/ClusterStatus.proto index 228be7ebb5..0a33e5b4df 100644 --- a/hbase-protocol/src/main/protobuf/ClusterStatus.proto +++ b/hbase-protocol/src/main/protobuf/ClusterStatus.proto @@ -143,6 +143,8 @@ message RegionLoad { message ReplicationLoadSink { required uint64 ageOfLastAppliedOp = 1; required uint64 timeStampsOfLastAppliedOp = 2; + required uint64 timeStampStarted = 3; + required uint64 totalOpsProcessed = 4; } message ReplicationLoadSource { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSink.java index f308daf8e8..339923069c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/MetricsSink.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.CompatibilitySingletonFactory; public class MetricsSink { private long lastTimestampForAge = System.currentTimeMillis(); + private long startTimestamp = System.currentTimeMillis(); private final MetricsReplicationSinkSource mss; public MetricsSink() { @@ -99,4 +100,18 @@ public class MetricsSink { return this.lastTimestampForAge; } + /** + * Gets the time stamp from when the Sink was initialized. + * @return startTimestamp + */ + public long getStartTimestamp() { return this.startTimestamp; } + + /** + * + * @return + */ + public long getAppliedOps() { + return this.mss.getSinkAppliedOps(); + } + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationLoad.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationLoad.java index 2ead3df99e..20bc38f1b4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationLoad.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationLoad.java @@ -65,6 +65,8 @@ public class ReplicationLoad { ClusterStatusProtos.ReplicationLoadSink.newBuilder(); rLoadSinkBuild.setAgeOfLastAppliedOp(sinkMetrics.getAgeOfLastAppliedOp()); rLoadSinkBuild.setTimeStampsOfLastAppliedOp(sinkMetrics.getTimeStampOfLastAppliedOp()); + rLoadSinkBuild.setTimeStampStarted(sinkMetrics.getStartTimestamp()); + rLoadSinkBuild.setTotalOpsProcessed(sinkMetrics.getAppliedOps()); this.replicationLoadSink = rLoadSinkBuild.build(); // build the SourceLoad List diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 468832db8c..67fa27da90 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -727,9 +727,15 @@ module Hbase rSinkString = " SINK :" rSourceString = " SOURCE:" rLoadSink = sl.getReplicationLoadSink() - rSinkString << " AgeOfLastAppliedOp=" + rLoadSink.getAgeOfLastAppliedOp().to_s - rSinkString << ", TimeStampsOfLastAppliedOp=" + - (java.util.Date.new(rLoadSink.getTimeStampsOfLastAppliedOp())).toString() + if (rLoadSink.getTimeStampsOfLastAppliedOp() == rLoadSink.getTimeStampStarted()) + rSinkString << " TimeStampStarted=" + rLoadSink.getTimeStampStarted().to_s + rSinkString << ", Waiting for OPs... " + else + rSinkString << " TimeStampStarted=" + rLoadSink.getTimeStampStarted().to_s + rSinkString << ", AgeOfLastAppliedOp=" + rLoadSink.getAgeOfLastAppliedOp().to_s + rSinkString << ", TimeStampsOfLastAppliedOp=" + + (java.util.Date.new(rLoadSink.getTimeStampsOfLastAppliedOp())).toString() + end rLoadSourceList = sl.getReplicationLoadSourceList() index = 0 while index < rLoadSourceList.size() -- 2.17.1 (Apple Git-112)