From 7d939f787a9af789151b199c71795864b770912a Mon Sep 17 00:00:00 2001 From: Josh Elser Date: Mon, 8 Jan 2018 16:47:18 -0500 Subject: [PATCH] HBASE-19734 Fix IntegrationTestReplication and related impl changes Adds (client-side) validation to ReplicationPeerConfigBuilder and javadoc to builder methods in addition to the test fix. --- .../hbase/replication/ReplicationPeerConfig.java | 2 + .../replication/ReplicationPeerConfigBuilder.java | 71 ++++++++++++++++++++++ .../replication/TestReplicationPeerConfig.java | 1 - .../hbase/test/IntegrationTestReplication.java | 34 ++++++----- 4 files changed, 92 insertions(+), 16 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java index ab75dff171..b80ee16301 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfig.java @@ -315,6 +315,8 @@ public class ReplicationPeerConfig { @Override public ReplicationPeerConfig build() { + // It would be nice to validate the configuration, but we have to work with "old" data + // from ZK which makes it much more difficult. return new ReplicationPeerConfig(this); } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigBuilder.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigBuilder.java index bbf8dd82de..0b2f2e2989 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigBuilder.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeerConfigBuilder.java @@ -43,32 +43,103 @@ public interface ReplicationPeerConfigBuilder { */ ReplicationPeerConfigBuilder setReplicationEndpointImpl(String replicationEndpointImpl); + /** + * Sets a "raw" configuration property for this replication peer. For experts only. + * @param key Configuration property key + * @param value Configuration property value + * @return {@code this} + */ + @InterfaceAudience.Private ReplicationPeerConfigBuilder putConfiguration(String key, String value); + /** + * Adds all of the provided "raw" configuration entries to {@code this}. + * @param configuration A collection of raw configuration entries + * @return {@code this} + */ + @InterfaceAudience.Private default ReplicationPeerConfigBuilder putAllConfiguration(Map configuration) { configuration.forEach(this::putConfiguration); return this; } + /** + * Sets the serialized peer configuration data + * @return {@code this} + */ + @InterfaceAudience.Private ReplicationPeerConfigBuilder putPeerData(byte[] key, byte[] value); + /** + * Sets all of the provided serialized peer configuration data. + * @return {@code this} + */ + @InterfaceAudience.Private default ReplicationPeerConfigBuilder putAllPeerData(Map peerData) { peerData.forEach(this::putPeerData); return this; } + /** + * Sets an explicit map of tables and column families in those tables that should be replicated + * to the given peer. Use {@link #setReplicateAllUserTables(boolean)} to replicate all tables + * to a peer. + * + * @param tableCFsMap A map from tableName to column family names. An empty collection can be + * passed to indicate replicating all column families. + * @return {@code this} + * @see #setReplicateAllUserTables(boolean) + */ ReplicationPeerConfigBuilder setTableCFsMap(Map> tableCFsMap); + /** + * Sets a unique collection of HBase namespaces that should be replicated to this peer. + * @param namespaces A set of namespaces to be replicated to this peer. + * @return {@code this} + */ ReplicationPeerConfigBuilder setNamespaces(Set namespaces); + /** + * Sets the speed, in bytes per second, for any one RegionServer to replicate data to the peer. + * @param bandwidth Bytes per second + * @return {@code this}. + */ ReplicationPeerConfigBuilder setBandwidth(long bandwidth); + /** + * Configures HBase to replicate all user tables (not system tables) to the peer. Default is + * {@code true}. + * @param replicateAllUserTables True if all user tables should be replicated, else false. + * @return {@code this} + */ ReplicationPeerConfigBuilder setReplicateAllUserTables(boolean replicateAllUserTables); + /** + * Sets the mapping of table name to column families which should not be replicated. This + * method sets state which is mutually exclusive to {@link #setTableCFsMap(Map)}. Invoking this + * method is only relevant when all user tables are being replicated. + * + * @param tableCFsMap A mapping of table names to column families which should not be + * replicated. An empty list of column families implies all families for the table. + * @return {@code this}. + */ ReplicationPeerConfigBuilder setExcludeTableCFsMap(Map> tableCFsMap); + /** + * Sets the collection of namespaces which should not be replicated when all user tables are + * configured to be replicated. This method sets state which is mutually exclusive to + * {@link #setNamespaces(Set)}. Invoking this method is only relevant when all user tables are + * being replicated. + * + * @param namespaces A set of namespaces whose tables should not be replicated. + * @return {@code this} + */ ReplicationPeerConfigBuilder setExcludeNamespaces(Set namespaces); + /** + * Builds the configuration object from the current state of {@code this}. + * @return A {@link ReplicationPeerConfig} instance. + */ ReplicationPeerConfig build(); } diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeerConfig.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeerConfig.java index a0b8a32405..edda760fd6 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeerConfig.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationPeerConfig.java @@ -43,5 +43,4 @@ public class TestReplicationPeerConfig { BuilderStyleTest.assertClassesAreBuilderStyle(ReplicationPeerConfig.class); } - } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestReplication.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestReplication.java index cb9aa56dd1..42dfafab56 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestReplication.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestReplication.java @@ -34,13 +34,16 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.replication.ReplicationAdmin; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; +import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Set; import java.util.TreeSet; import java.util.UUID; @@ -222,24 +225,25 @@ public class IntegrationTestReplication extends IntegrationTestBigLinkedList { // setup the replication on the source if (!source.equals(sink)) { - ReplicationAdmin replicationAdmin = new ReplicationAdmin(source.getConfiguration()); - // remove any old replication peers - for (String oldPeer : replicationAdmin.listPeerConfigs().keySet()) { - replicationAdmin.removePeer(oldPeer); - } - - // set the sink to be the target - ReplicationPeerConfig peerConfig = new ReplicationPeerConfig(); - peerConfig.setClusterKey(sink.toString()); + try (final Admin admin = source.getConnection().getAdmin()) { + // remove any old replication peers + for (ReplicationPeerDescription peer : admin.listReplicationPeers()) { + admin.removeReplicationPeer(peer.getPeerId()); + } - // set the test table to be the table to replicate - HashMap> toReplicate = new HashMap<>(); - toReplicate.put(tableName, new ArrayList<>(0)); + // set the test table to be the table to replicate + HashMap> toReplicate = new HashMap<>(); + toReplicate.put(tableName, Collections.emptyList()); - replicationAdmin.addPeer("TestPeer", peerConfig, toReplicate); + // set the sink to be the target + final ReplicationPeerConfig peerConfig = ReplicationPeerConfig.newBuilder() + .setClusterKey(sink.toString()) + .setReplicateAllUserTables(false) + .setTableCFsMap(toReplicate).build(); - replicationAdmin.enableTableRep(tableName); - replicationAdmin.close(); + admin.addReplicationPeer("TestPeer", peerConfig); + admin.enableTableReplication(tableName); + } } for (ClusterID cluster : clusters) { -- 2.15.1