diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java index 1b14dab..6270538 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationPeer.java @@ -32,6 +32,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; @@ -130,7 +131,7 @@ public class ReplicationPeer implements Abortable, Closeable { Map> tableCFsMap = null; // parse out (table, cf-list) pairs from tableCFsConfig - // format: "table1:cf1,cf2;table2:cfA,cfB" + // format: "ns:table1 cf1,cf2,cf3; ns:table2 cf1,cf2,cf3: ns:table3; table4" String[] tables = tableCFsConfig.split(";"); for (String tab : tables) { // 1 ignore empty table config @@ -139,9 +140,9 @@ public class ReplicationPeer implements Abortable, Closeable { continue; } // 2 split to "table" and "cf1,cf2" - // for each table: "table:cf1,cf2" or "table" - String[] pair = tab.split(":"); - String tabName = pair[0].trim(); + // for each table: "ns:table:cf1,cf2" or "ns:table" or "table" + String[] pair = tab.split(" ", 2); + String tabName = TableName.valueOf(pair[0].trim()).getNameAsString(); if (pair.length > 2 || tabName.length() == 0) { LOG.error("ignore invalid tableCFs setting: " + tab); continue; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestPerTableCFReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestPerTableCFReplication.java index ee102fc..7200945 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestPerTableCFReplication.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestPerTableCFReplication.java @@ -32,7 +32,14 @@ import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HBaseAdmin; @@ -64,20 +71,21 @@ public class TestPerTableCFReplication { private static final long SLEEP_TIME = 500; private static final int NB_RETRIES = 100; - private static final byte[] tableName = Bytes.toBytes("test"); - private static final byte[] tabAName = Bytes.toBytes("TA"); - private static final byte[] tabBName = Bytes.toBytes("TB"); + private static final String ns1Namespace = "ns1"; + private static final String ns2Namespace = "ns2"; + private static final byte[] tabAName = Bytes.toBytes("ns1:TA"); + private static final byte[] tabBName = Bytes.toBytes("ns2:TB"); private static final byte[] tabCName = Bytes.toBytes("TC"); - private static final byte[] famName = Bytes.toBytes("f"); private static final byte[] f1Name = Bytes.toBytes("f1"); private static final byte[] f2Name = Bytes.toBytes("f2"); private static final byte[] f3Name = Bytes.toBytes("f3"); private static final byte[] row1 = Bytes.toBytes("row1"); private static final byte[] row2 = Bytes.toBytes("row2"); - private static final byte[] noRepfamName = Bytes.toBytes("norep"); private static final byte[] val = Bytes.toBytes("myval"); - private static HTableDescriptor table; + private static NamespaceDescriptor ns1; + private static NamespaceDescriptor ns2; + private static HTableDescriptor tabA; private static HTableDescriptor tabB; private static HTableDescriptor tabC; @@ -118,15 +126,12 @@ public class TestPerTableCFReplication { utility3.setZkCluster(miniZK); new ZooKeeperWatcher(conf3, "cluster3", null, true); - table = new HTableDescriptor(TableName.valueOf(tableName)); - HColumnDescriptor fam = new HColumnDescriptor(famName); - fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); - table.addFamily(fam); - fam = new HColumnDescriptor(noRepfamName); - table.addFamily(fam); + + ns1 = NamespaceDescriptor.create(ns1Namespace).build(); + ns2 = NamespaceDescriptor.create(ns2Namespace).build(); tabA = new HTableDescriptor(tabAName); - fam = new HColumnDescriptor(f1Name); + HColumnDescriptor fam = new HColumnDescriptor(f1Name); fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); tabA.addFamily(fam); fam = new HColumnDescriptor(f2Name); @@ -184,82 +189,100 @@ public class TestPerTableCFReplication { tabCFsMap = ReplicationPeer.parseTableCFsFromConfig(" "); assertEquals(null, tabCFsMap); - // 2. single table: "tab1" / "tab2:cf1" / "tab3:cf1,cf3" - tabCFsMap = ReplicationPeer.parseTableCFsFromConfig("tab1"); + // 2. single table: "ns1:tab1" / "ns2:tab2 cf1" / "tab3 cf1,cf3" + String tab21 = "ns1" + TableName.NAMESPACE_DELIM + "tab1"; + tabCFsMap = ReplicationPeer.parseTableCFsFromConfig(tab21); assertEquals(1, tabCFsMap.size()); // only one table - assertTrue(tabCFsMap.containsKey("tab1")); // its table name is "tab1" - assertFalse(tabCFsMap.containsKey("tab2")); // not other table - assertEquals(null, tabCFsMap.get("tab1")); // null cf-list, + assertTrue(tabCFsMap.containsKey(tab21)); // its table name is "ns1:tab1" + assertFalse(tabCFsMap.containsKey("ns1" + TableName.NAMESPACE_DELIM + "tab2")); // not other + // table + assertEquals(null, tabCFsMap.get(tab21)); // null cf-list, - tabCFsMap = ReplicationPeer.parseTableCFsFromConfig("tab2:cf1"); + String table22 = "ns2" + TableName.NAMESPACE_DELIM + "tab2"; + tabCFsMap = ReplicationPeer.parseTableCFsFromConfig(table22 + " cf1"); assertEquals(1, tabCFsMap.size()); // only one table - assertTrue(tabCFsMap.containsKey("tab2")); // its table name is "tab2" - assertFalse(tabCFsMap.containsKey("tab1")); // not other table - assertEquals(1, tabCFsMap.get("tab2").size()); // cf-list contains only 1 cf - assertEquals("cf1", tabCFsMap.get("tab2").get(0));// the only cf is "cf1" - - tabCFsMap = ReplicationPeer.parseTableCFsFromConfig("tab3 : cf1 , cf3"); + assertTrue(tabCFsMap.containsKey(table22)); // its table name is "tab2" + assertFalse(tabCFsMap.containsKey("ns1" + TableName.NAMESPACE_DELIM + "tab2")); // not other + // table + assertEquals(1, tabCFsMap.get(table22).size()); // cf-list contains only 1 cf + assertEquals("cf1", tabCFsMap.get(table22).get(0));// the only cf is "cf1" + + String table23 = TableName.valueOf("tab3").getNameAsString(); + tabCFsMap = ReplicationPeer.parseTableCFsFromConfig("tab3 cf1 , cf3"); assertEquals(1, tabCFsMap.size()); // only one table - assertTrue(tabCFsMap.containsKey("tab3")); // its table name is "tab2" - assertFalse(tabCFsMap.containsKey("tab1")); // not other table - assertEquals(2, tabCFsMap.get("tab3").size()); // cf-list contains 2 cf - assertTrue(tabCFsMap.get("tab3").contains("cf1"));// contains "cf1" - assertTrue(tabCFsMap.get("tab3").contains("cf3"));// contains "cf3" - - // 3. multiple tables: "tab1 ; tab2:cf1 ; tab3:cf1,cf3" - tabCFsMap = ReplicationPeer.parseTableCFsFromConfig("tab1 ; tab2:cf1 ; tab3:cf1,cf3"); - // 3.1 contains 3 tables : "tab1", "tab2" and "tab3" - assertEquals(3, tabCFsMap.size()); - assertTrue(tabCFsMap.containsKey("tab1")); - assertTrue(tabCFsMap.containsKey("tab2")); - assertTrue(tabCFsMap.containsKey("tab3")); + assertTrue(tabCFsMap.containsKey(table23)); // its table name is "tab3" + assertFalse(tabCFsMap.containsKey("tab1")); // not other table + assertEquals(2, tabCFsMap.get(table23).size()); // cf-list contains 2 cf + assertTrue(tabCFsMap.get(table23).contains("cf1"));// contains "cf1" + assertTrue(tabCFsMap.get(table23).contains("cf3"));// contains "cf3" + + // 3. multiple tables: "ns1:tab1 ; ns2:tab2 cf1 ; ns3:tab3 cf1,cf3 ; tab4 ; tab5 cf2" + tabCFsMap = + ReplicationPeer + .parseTableCFsFromConfig("ns1:tab1 ; ns2:tab2 cf1 ; ns3:tab3 cf1,cf3 ; tab4 ; tab5 cf2,cf3"); + String tab31 = "ns1" + TableName.NAMESPACE_DELIM + "tab1"; + String tab32 = "ns2" + TableName.NAMESPACE_DELIM + "tab2"; + String tab33 = "ns3" + TableName.NAMESPACE_DELIM + "tab3"; + String tab34 = TableName.valueOf("tab4").getNameAsString(); + String tab35 = TableName.valueOf("tab5").getNameAsString(); + // 3.1 contains 5 tables : "ns1:tab1", "ns2:tab2", "ns3:tab3", "default:tab4", "default:tab5" + assertEquals(5, tabCFsMap.size()); + assertTrue(tabCFsMap.containsKey(tab31)); + assertTrue(tabCFsMap.containsKey(tab32)); + assertTrue(tabCFsMap.containsKey(tab33)); + assertTrue(tabCFsMap.containsKey(tab34)); + assertTrue(tabCFsMap.containsKey(tab35)); // 3.2 table "tab1" : null cf-list - assertEquals(null, tabCFsMap.get("tab1")); + assertEquals(null, tabCFsMap.get(tab31)); // 3.3 table "tab2" : cf-list contains a single cf "cf1" - assertEquals(1, tabCFsMap.get("tab2").size()); - assertEquals("cf1", tabCFsMap.get("tab2").get(0)); + assertEquals(1, tabCFsMap.get(tab32).size()); + assertEquals("cf1", tabCFsMap.get(tab32).get(0)); // 3.4 table "tab3" : cf-list contains "cf1" and "cf3" - assertEquals(2, tabCFsMap.get("tab3").size()); - assertTrue(tabCFsMap.get("tab3").contains("cf1")); - assertTrue(tabCFsMap.get("tab3").contains("cf3")); + assertEquals(2, tabCFsMap.get(tab33).size()); + assertTrue(tabCFsMap.get(tab33).contains("cf1")); + assertTrue(tabCFsMap.get(tab33).contains("cf3")); + // 3.5 table "tab4" :null cf-list + assertEquals(null, tabCFsMap.get(tab34)); + // 3.6 table "tab5" :cf-list contains "cf2" and "cf3" + assertEquals(2, tabCFsMap.get(tab35).size()); + assertTrue(tabCFsMap.get(tab35).contains("cf2")); + assertTrue(tabCFsMap.get(tab35).contains("cf3")); // 4. contiguous or additional ";"(table delimiter) or ","(cf delimiter) can be tolerated - // still use the example of multiple tables: "tab1 ; tab2:cf1 ; tab3:cf1,cf3" - tabCFsMap = ReplicationPeer.parseTableCFsFromConfig("tab1 ; ; tab2:cf1 ; tab3:cf1,,cf3 ;"); + // still use the example of multiple tables: "ns1:tab1 ; ; ns2:tab2 cf1 ; ns3:tab3 cf1,,cf3 ;" + tabCFsMap = + ReplicationPeer.parseTableCFsFromConfig("ns1:tab1 ; ; ns2:tab2 cf1 ; ns3:tab3 cf1,,cf3 ;"); + String tab41 = "ns1" + TableName.NAMESPACE_DELIM + "tab1"; + String tab42 = "ns2" + TableName.NAMESPACE_DELIM + "tab2"; + String tab43 = "ns3" + TableName.NAMESPACE_DELIM + "tab3"; // 4.1 contains 3 tables : "tab1", "tab2" and "tab3" assertEquals(3, tabCFsMap.size()); - assertTrue(tabCFsMap.containsKey("tab1")); - assertTrue(tabCFsMap.containsKey("tab2")); - assertTrue(tabCFsMap.containsKey("tab3")); + assertTrue(tabCFsMap.containsKey(tab41)); + assertTrue(tabCFsMap.containsKey(tab42)); + assertTrue(tabCFsMap.containsKey(tab43)); // 4.2 table "tab1" : null cf-list - assertEquals(null, tabCFsMap.get("tab1")); + assertEquals(null, tabCFsMap.get(tab41)); // 4.3 table "tab2" : cf-list contains a single cf "cf1" - assertEquals(1, tabCFsMap.get("tab2").size()); - assertEquals("cf1", tabCFsMap.get("tab2").get(0)); + assertEquals(1, tabCFsMap.get(tab42).size()); + assertEquals("cf1", tabCFsMap.get(tab42).get(0)); // 4.4 table "tab3" : cf-list contains "cf1" and "cf3" - assertEquals(2, tabCFsMap.get("tab3").size()); - assertTrue(tabCFsMap.get("tab3").contains("cf1")); - assertTrue(tabCFsMap.get("tab3").contains("cf3")); - - // 5. invalid format "tab1:tt:cf1 ; tab2::cf1 ; tab3:cf1,cf3" - // "tab1:tt:cf1" and "tab2::cf1" are invalid and will be ignored totally - tabCFsMap = ReplicationPeer.parseTableCFsFromConfig("tab1:tt:cf1 ; tab2::cf1 ; tab3:cf1,cf3"); - // 5.1 no "tab1" and "tab2", only "tab3" - assertEquals(1, tabCFsMap.size()); // only one table - assertFalse(tabCFsMap.containsKey("tab1")); - assertFalse(tabCFsMap.containsKey("tab2")); - assertTrue(tabCFsMap.containsKey("tab3")); - // 5.2 table "tab3" : cf-list contains "cf1" and "cf3" - assertEquals(2, tabCFsMap.get("tab3").size()); - assertTrue(tabCFsMap.get("tab3").contains("cf1")); - assertTrue(tabCFsMap.get("tab3").contains("cf3")); - } + assertEquals(2, tabCFsMap.get(tab43).size()); + assertTrue(tabCFsMap.get(tab43).contains("cf1")); + assertTrue(tabCFsMap.get(tab43).contains("cf3")); + } @Test(timeout=300000) public void testPerTableCFReplication() throws Exception { LOG.info("testPerTableCFReplication"); ReplicationAdmin admin1 = new ReplicationAdmin(conf1); + new HBaseAdmin(conf1).createNamespace(ns1); + new HBaseAdmin(conf1).createNamespace(ns2); + new HBaseAdmin(conf2).createNamespace(ns1); + new HBaseAdmin(conf2).createNamespace(ns2); + new HBaseAdmin(conf3).createNamespace(ns1); + new HBaseAdmin(conf3).createNamespace(ns2); + new HBaseAdmin(conf1).createTable(tabA); new HBaseAdmin(conf1).createTable(tabB); new HBaseAdmin(conf1).createTable(tabC); @@ -283,8 +306,8 @@ public class TestPerTableCFReplication { HTable htab3C = new HTable(conf3, tabCName); // A. add cluster2/cluster3 as peers to cluster1 - admin1.addPeer("2", utility2.getClusterKey(), "TC;TB:f1,f3"); - admin1.addPeer("3", utility3.getClusterKey(), "TA;TB:f1,f2"); + admin1.addPeer("2", utility2.getClusterKey(), "TC;ns2:TB f1,f3"); + admin1.addPeer("3", utility3.getClusterKey(), "ns1:TA;ns2:TB f1,f2"); // A1. tableA can only replicated to cluster3 putAndWaitWithFamily(row1, f1Name, htab1A, htab3A); @@ -327,8 +350,8 @@ public class TestPerTableCFReplication { deleteAndWaitWithFamily(row1, f3Name, htab1C, htab2C); // B. change peers' replicable table-cf config - admin1.setPeerTableCFs("2", "TA:f1,f2; TC:f2,f3"); - admin1.setPeerTableCFs("3", "TB; TC:f3"); + admin1.setPeerTableCFs("2", "ns1:TA f1,f2; TC f2,f3"); + admin1.setPeerTableCFs("3", "ns2:TB; TC f3"); // B1. cf 'f1' of tableA can only replicated to cluster2 putAndWaitWithFamily(row2, f1Name, htab1A, htab2A); diff --git a/hbase-shell/src/main/ruby/shell/commands/add_peer.rb b/hbase-shell/src/main/ruby/shell/commands/add_peer.rb index ecd8e75..65bcf2f 100644 --- a/hbase-shell/src/main/ruby/shell/commands/add_peer.rb +++ b/hbase-shell/src/main/ruby/shell/commands/add_peer.rb @@ -30,7 +30,7 @@ Examples: hbase> add_peer '1', "server1.cie.com:2181:/hbase" hbase> add_peer '2', "zk1,zk2,zk3:2182:/hbase-prod" - hbase> add_peer '3', "zk4,zk5,zk6:11000:/hbase-test", "tab1; tab2:cf1; tab3:cf2,cf3" + hbase> add_peer '3', "zk4,zk5,zk6:11000:/hbase-test", "ns1:tab1; tab2; ns2:tab2 cf1; tab3 cf2,cf3; ns4:tab4 cf1,cf3,cf5" EOF end diff --git a/hbase-shell/src/main/ruby/shell/commands/set_peer_tableCFs.rb b/hbase-shell/src/main/ruby/shell/commands/set_peer_tableCFs.rb index 3a88dbb..b07a070 100644 --- a/hbase-shell/src/main/ruby/shell/commands/set_peer_tableCFs.rb +++ b/hbase-shell/src/main/ruby/shell/commands/set_peer_tableCFs.rb @@ -32,7 +32,7 @@ module Shell # set table / table-cf to be replicable for a peer, for a table without # an explicit column-family list, all replicable column-families (with # replication_scope == 1) will be replicated - hbase> set_peer_tableCFs '2', "table1; table2:cf1,cf2; table3:cfA,cfB" + hbase> set_peer_tableCFs '2', "ns1:tab1; tab2; ns2:tab2 cf1; tab3 cf2,cf3; ns4:tab4 cf1,cf3,cf5" EOF end