From c3014b578b03c8893e206439e58b3ca1a9bfb3be Mon Sep 17 00:00:00 2001 From: Wellington Chevreuil Date: Wed, 6 Feb 2019 15:28:32 +0000 Subject: [PATCH] HBASE-21784 - Dump replication queue should show list of wal files ordered chronologically Change-Id: I7fba7daf31da4980f24b778a2905e23ade8d1e2e --- .../regionserver/DumpReplicationQueues.java | 2 + .../TestDumpReplicationQueues.java | 74 +++++++++++++++++++ .../apache/hadoop/hbase/zookeeper/ZKUtil.java | 2 + 3 files changed, 78 insertions(+) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java index a960c3146a..432dbcddae 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java @@ -21,6 +21,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -325,6 +326,7 @@ public class DumpReplicationQueues extends Configured implements Tool { for (String queueId : queueIds) { ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId); List wals = queueStorage.getWALsInQueue(regionserver, queueId); + Collections.sort(wals); if (!peerIds.contains(queueInfo.getPeerId())) { deletedQueues.add(regionserver + "/" + queueId); sb.append(formatQueue(regionserver, queueStorage, queueInfo, queueId, wals, true, hdfs)); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java new file mode 100644 index 0000000000..533d1425e3 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestDumpReplicationQueues.java @@ -0,0 +1,74 @@ +package org.apache.hadoop.hbase.replication.regionserver; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.testclassification.ReplicationTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper; +import org.apache.hadoop.hbase.zookeeper.ZKWatcher; +import org.apache.hadoop.hbase.zookeeper.ZNodePaths; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ ReplicationTests.class, SmallTests.class}) +public class TestDumpReplicationQueues { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestDumpReplicationQueues.class); + + @Test + public void testDumpReplicationReturnsWalSorted() throws Exception { + Configuration config = HBaseConfiguration.create(); + ZKWatcher zkWatcherMock = mock(ZKWatcher.class); + ZNodePaths zNodePath = new ZNodePaths(config); + RecoverableZooKeeper recoverableZooKeeperMock = mock(RecoverableZooKeeper.class); + when(zkWatcherMock.getRecoverableZooKeeper()).thenReturn(recoverableZooKeeperMock); + when(zkWatcherMock.getZNodePaths()).thenReturn(zNodePath); + List nodes = new ArrayList<>(); + String server = "rs1,60030,"+System.currentTimeMillis(); + nodes.add(server); + when(recoverableZooKeeperMock.getChildren("/hbase/rs", null)).thenReturn(nodes); + when(recoverableZooKeeperMock.getChildren("/hbase/replication/rs", null)). + thenReturn(nodes); + List queuesIds = new ArrayList<>(); + queuesIds.add("1"); + when(recoverableZooKeeperMock.getChildren("/hbase/replication/rs/"+server, null)). + thenReturn(queuesIds); + List wals = new ArrayList<>(); + wals.add("rs1%2C60964%2C1549394085556.1549394101427"); + wals.add("rs1%2C60964%2C1549394085556.1549394101426"); + wals.add("rs1%2C60964%2C1549394085556.1549394101428"); + when(recoverableZooKeeperMock.getChildren("/hbase/replication/rs/"+server+"/1", + null)).thenReturn(wals); + DumpReplicationQueues dumpQueues = new DumpReplicationQueues(); + Set peerIds = new HashSet<>(); + peerIds.add("1"); + dumpQueues.setConf(config); + String dump = dumpQueues.dumpQueues(zkWatcherMock, peerIds, false); + String[] parsedDump = dump.split("Replication position for"); + assertEquals("Parsed dump should have 4 parts.", 4, parsedDump.length); + assertTrue("First wal should be rs1%2C60964%2C1549394085556.1549394101426, but got: " + + parsedDump[1], + parsedDump[1].indexOf("rs1%2C60964%2C1549394085556.1549394101426")>=0); + assertTrue("Second wal should be rs1%2C60964%2C1549394085556.1549394101427, but got: " + + parsedDump[2], + parsedDump[2].indexOf("rs1%2C60964%2C1549394085556.1549394101427")>=0); + assertTrue("Third wal should be rs1%2C60964%2C1549394085556.1549394101428, but got: " + + parsedDump[3], + parsedDump[3].indexOf("rs1%2C60964%2C1549394085556.1549394101428")>=0); + } + +} diff --git a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java index 57c847c0fe..7703bce867 100644 --- a/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java +++ b/hbase-zookeeper/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java @@ -29,6 +29,7 @@ import java.net.Socket; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Deque; import java.util.HashMap; import java.util.LinkedList; @@ -1808,6 +1809,7 @@ public final class ZKUtil { // do a ls -r on this znode sb.append("\n").append(replicationZnode).append(": "); List children = ZKUtil.listChildrenNoWatch(zkw, replicationZnode); + Collections.sort(children); for (String child : children) { String znode = ZNodePaths.joinZNode(replicationZnode, child); if (znode.equals(zkw.getZNodePaths().peersZNode)) { -- 2.17.2 (Apple Git-113)