From 690de36c4eff0ee210c53410e3b415c32850dbb9 Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Tue, 5 Feb 2019 19:42:39 +0000 Subject: [PATCH] HBASE-21784 - Dump replication queue should show list of wal files ordered chronologically Change-Id: Iff660dadd6e51d9d58f93b4c59630cae6da52695 --- .../regionserver/DumpReplicationQueues.java | 2 + .../TestDumpReplicationQueues.java | 70 +++++++++++++++++++ .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 2 + 3 files changed, 74 insertions(+) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java index a960c3146a..432dbcddae 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java @@ -21,6 +21,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -325,6 +326,7 @@ public class DumpReplicationQueues extends Configured implements Tool { for (String queueId : queueIds) { ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId); List wals = queueStorage.getWALsInQueue(regionserver, queueId); + Collections.sort(wals); if (!peerIds.contains(queueInfo.getPeerId())) { deletedQueues.add(regionserver + "/" + queueId); sb.append(formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, true, hdfs)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java new file mode 100644 index 0000000000..fe63fb3bf9 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java @@ -0,0 +1,70 @@ +package org.apache.hadoop.hbase.replication.regionserver; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; + +import org.apache.hadoop.hbase.testclassification.ReplicationTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper; +import org.apache.hadoop.hbase.zookeeper.ZKWatcher; +import org.apache.hadoop.hbase.zookeeper.ZNodePaths; +import org.junit.Test; +import org.junit.experimental.categories.Category; + + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Category({ ReplicationTests.class, SmallTests.class}) +public class TestDumpReplicationQueues { + + @Test + public void testDumpReplicationReturnsWalSorted() throws Exception { + Configuration config = HBaseConfiguration.create(); + ZKWatcher zkWatcherMock = mock(ZKWatcher.class); + ZNodePaths zNodePath = new ZNodePaths(config); + RecoverableZooKeeper recoverableZooKeeperMock = mock(RecoverableZooKeeper.class); + when(zkWatcherMock.getRecoverableZooKeeper()).thenReturn(recoverableZooKeeperMock); + when(zkWatcherMock.getZNodePaths()).thenReturn(zNodePath); + List nodes = new ArrayList<>(); + String server = "rs1,60030,"+System.currentTimeMillis(); + nodes.add(server); + when(recoverableZooKeeperMock.getChildren("/hbase/rs", null)).thenReturn(nodes); + when(recoverableZooKeeperMock.getChildren("/hbase/replication/rs", null)). + thenReturn(nodes); + List queuesIds = new ArrayList<>(); + queuesIds.add("1"); + when(recoverableZooKeeperMock.getChildren("/hbase/replication/rs/"+server, null)). + thenReturn(queuesIds); + List wals = new ArrayList<>(); + wals.add("rs1%2C60964%2C1549394085556.1549394101427"); + wals.add("rs1%2C60964%2C1549394085556.1549394101426"); + wals.add("rs1%2C60964%2C1549394085556.1549394101428"); + when(recoverableZooKeeperMock.getChildren("/hbase/replication/rs/"+server+"/1", + null)).thenReturn(wals); + DumpReplicationQueues dumpQueues = new DumpReplicationQueues(); + Set peerIds = new HashSet<>(); + peerIds.add("1"); + dumpQueues.setConf(config); + String dump = dumpQueues.dumpQueues(zkWatcherMock, peerIds, false); + String[] parsedDump = dump.split("Replication position for"); + assertEquals("Parsed dump should have 4 parts.", 4, parsedDump.length); + assertTrue("First wal should be rs1%2C60964%2C1549394085556.1549394101426, but got: " + + parsedDump[1], + parsedDump[1].indexOf("rs1%2C60964%2C1549394085556.1549394101426")>=0); + assertTrue("Second wal should be rs1%2C60964%2C1549394085556.1549394101427, but got: " + + parsedDump[2], + parsedDump[2].indexOf("rs1%2C60964%2C1549394085556.1549394101427")>=0); + assertTrue("Third wal should be rs1%2C60964%2C1549394085556.1549394101428, but got: " + + parsedDump[3], + parsedDump[3].indexOf("rs1%2C60964%2C1549394085556.1549394101428")>=0); + } + +} diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java index 9153191975..facc5ccf77 100644 --- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java +++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java @@ -29,6 +29,7 @@ import java.net.Socket; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Deque; import java.util.HashMap; import java.util.LinkedList; @@ -1809,6 +1810,7 @@ public final class ZKUtil { // do a ls -r on this znode sb.append("\n").append(replicationZnode).append(": "); List children = ZKUtil.listChildrenNoWatch(zkw, replicationZnode); + Collections.sort(children); for (String child : children) { String znode = ZNodePaths.joinZNode(replicationZnode, child); if (znode.equals(zkw.getZNodePaths().peersZNode)) { -- 2.17.2 (Apple Git-113)