diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java index ce200fa..e236d99 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.List; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -174,6 +175,31 @@ public class MetaEditor extends MetaReader { } /** + * Deletes some replica columns corresponding to replicas for the passed rows + * @param metaRows + * @param replicaIndexToDeleteFrom the replica ID we would start deleting from + * @param numReplicasToRemove + * @param ct + * @throws IOException + */ + public static void removeRegionReplicasFromMeta(Set metaRows, int replicaIndexToDeleteFrom, + int numReplicasToRemove, CatalogTracker ct) throws IOException { + int absoluteIndex = replicaIndexToDeleteFrom + numReplicasToRemove; + for (byte[] row : metaRows) { + Delete deleteReplicaLocations = new Delete(row); + for (int i = replicaIndexToDeleteFrom; i < absoluteIndex; i++) { + deleteReplicaLocations.deleteColumns(HConstants.CATALOG_FAMILY, + MetaReader.getServerColumn(i)); + deleteReplicaLocations.deleteColumns(HConstants.CATALOG_FAMILY, + MetaReader.getSeqNumColumn(i)); + deleteReplicaLocations.deleteColumns(HConstants.CATALOG_FAMILY, + MetaReader.getStartCodeColumn(i)); + } + deleteFromMetaTable(ct, deleteReplicaLocations); + } + } + + /** * Execute the passed mutations against hbase:meta table. * @param ct CatalogTracker on whose back we will ride the edit. * @param mutations Puts and Deletes to execute on hbase:meta diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java index 2fd0607..56eca74 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java @@ -453,9 +453,11 @@ public class RegionStates { } HRegionInfo defaultReplica = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri); Set replicas = defaultReplicaToOtherReplicas.get(defaultReplica); - replicas.remove(hri); - if (replicas.isEmpty()) { - defaultReplicaToOtherReplicas.remove(defaultReplica); + if (replicas != null) { + replicas.remove(hri); + if (replicas.isEmpty()) { + defaultReplicaToOtherReplicas.remove(defaultReplica); + } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java index 0e21a25..a6bcbf9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.master.handler; import java.io.IOException; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; @@ -42,6 +43,7 @@ import org.apache.hadoop.hbase.master.BulkAssigner; import org.apache.hadoop.hbase.master.GeneralBulkAssigner; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; +import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.RegionStates; import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.master.TableLockManager; @@ -62,6 +64,7 @@ public class EnableTableHandler extends EventHandler { private final CatalogTracker catalogTracker; private boolean skipTableStateCheck = false; private TableLock tableLock; + private MasterServices services; public EnableTableHandler(Server server, TableName tableName, CatalogTracker catalogTracker, AssignmentManager assignmentManager, @@ -74,6 +77,14 @@ public class EnableTableHandler extends EventHandler { this.skipTableStateCheck = skipTableStateCheck; } + public EnableTableHandler(MasterServices services, TableName tableName, + CatalogTracker catalogTracker, AssignmentManager assignmentManager, + TableLockManager tableLockManager, boolean skipTableStateCheck) { + this((Server)services, tableName, catalogTracker, assignmentManager, tableLockManager, + skipTableStateCheck); + this.services = services; + } + public EnableTableHandler prepare() throws TableNotFoundException, TableNotDisabledException, IOException { //acquire the table write lock, blocking @@ -183,6 +194,16 @@ public class EnableTableHandler extends EventHandler { int countOfRegionsInTable = tableRegionsAndLocations.size(); Map regionsToAssign = regionsToAssignWithServerName(tableRegionsAndLocations); + if (services != null) { + // need to potentially create some regions for the replicas + List unrecordedReplicas = AssignmentManager.replicaRegionsNotRecordedInMeta( + new HashSet(regionsToAssign.keySet()), services); + for (HRegionInfo h : unrecordedReplicas) { + regionsToAssign.put(h, + this.assignmentManager.getBalancer().randomAssignment(h, + serverManager.getOnlineServersList())); + } + } int regionsCount = regionsToAssign.size(); if (regionsCount == 0) { done = true; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java index 622a6ca..c0bdbb6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ModifyTableHandler.java @@ -19,16 +19,24 @@ package org.apache.hadoop.hbase.master.handler; import java.io.IOException; +import java.util.HashSet; import java.util.List; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.catalog.MetaEditor; +import org.apache.hadoop.hbase.catalog.MetaReader; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.executor.EventType; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; @@ -53,8 +61,12 @@ public class ModifyTableHandler extends TableEventHandler { @Override protected void prepareWithTableLock() throws IOException { super.prepareWithTableLock(); - // Check table exists. - getTableDescriptor(); + // Check operation is possible on the table in its current state + // Also checks whether the table exists + if (masterServices.getAssignmentManager().getZKTable().isEnabledTable(this.htd.getTableName()) + && this.htd.getRegionReplication() != getTableDescriptor().getRegionReplication()) { + throw new IOException("REGION_REPLICATION change is not supported for enabled tables"); + } } @Override @@ -68,11 +80,35 @@ public class ModifyTableHandler extends TableEventHandler { HTableDescriptor oldHtd = getTableDescriptor(); this.masterServices.getTableDescriptors().add(this.htd); deleteFamilyFromFS(hris, oldHtd.getFamiliesKeys()); + removeReplicaColumnsIfNeeded(this.htd.getRegionReplication(), oldHtd.getRegionReplication(), + htd.getTableName()); if (cpHost != null) { cpHost.postModifyTableHandler(this.tableName, this.htd); } } + private void removeReplicaColumnsIfNeeded(int newReplicaCount, int oldReplicaCount, + TableName table) throws IOException { + if (newReplicaCount >= oldReplicaCount) return; + Set tableRows = new HashSet(); + Scan scan = MetaReader.getScanForTableName(table); + scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); + HTable htable = null; + try { + htable = new HTable(masterServices.getConfiguration(), TableName.META_TABLE_NAME); + ResultScanner resScanner = htable.getScanner(scan); + for (Result result : resScanner) { + tableRows.add(result.getRow()); + } + MetaEditor.removeRegionReplicasFromMeta(tableRows, newReplicaCount, + oldReplicaCount - newReplicaCount, masterServices.getCatalogTracker()); + } finally { + if (htable != null) { + htable.close(); + } + } + } + /** * Removes from hdfs the families that are not longer present in the new table descriptor. */ diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java index 8f51c51..c12b3bd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java @@ -181,44 +181,43 @@ public class TestMasterOperationsForRegionReplicas { // TEST_UTIL.getMiniHBaseCluster().startRegionServer(); // } - //TODO: HBASE-10361 patch should uncomment the test below -// //check on alter table -// admin.disableTable(table); -// assert(admin.isTableDisabled(table)); -// //increase the replica -// desc.setRegionReplication(numReplica + 1); -// admin.modifyTable(table, desc); -// admin.enableTable(table); -// assert(admin.isTableEnabled(table)); -// List regions = TEST_UTIL.getMiniHBaseCluster().getMaster() -// .getAssignmentManager().getRegionStates().getRegionsOfTable(table); -// assert(regions.size() == numRegions * (numReplica + 1)); -// -// //decrease the replica(earlier, table was modified to have a replica count of numReplica + 1) -// admin.disableTable(table); -// desc.setRegionReplication(numReplica); -// admin.modifyTable(table, desc); -// admin.enableTable(table); -// assert(admin.isTableEnabled(table)); -// regions = TEST_UTIL.getMiniHBaseCluster().getMaster() -// .getAssignmentManager().getRegionStates().getRegionsOfTable(table); -// assert(regions.size() == numRegions * numReplica); -// //also make sure the meta table has the replica locations removed -// hris = MetaReader.getTableRegions(ct, table); -// assert(hris.size() == numRegions * numReplica); -// //just check that the number of default replica regions in the meta table are the same -// //as the number of regions the table was created with, and the count of the -// //replicas is numReplica for each region -// Map defaultReplicas = new HashMap(); -// for (HRegionInfo hri : hris) { -// Integer i; -// HRegionInfo regionReplica0 = hri.getRegionInfoForReplica(0); -// defaultReplicas.put(regionReplica0, -// (i = defaultReplicas.get(regionReplica0)) == null ? 1 : i + 1); -// } -// assert(defaultReplicas.size() == numRegions); -// Collection counts = new HashSet(defaultReplicas.values()); -// assert(counts.size() == 1 && counts.contains(new Integer(numReplica))); + //check on alter table + admin.disableTable(table); + assert(admin.isTableDisabled(table)); + //increase the replica + desc.setRegionReplication(numReplica + 1); + admin.modifyTable(table, desc); + admin.enableTable(table); + assert(admin.isTableEnabled(table)); + List regions = TEST_UTIL.getMiniHBaseCluster().getMaster() + .getAssignmentManager().getRegionStates().getRegionsOfTable(table); + assert(regions.size() == numRegions * (numReplica + 1)); + + //decrease the replica(earlier, table was modified to have a replica count of numReplica + 1) + admin.disableTable(table); + desc.setRegionReplication(numReplica); + admin.modifyTable(table, desc); + admin.enableTable(table); + assert(admin.isTableEnabled(table)); + regions = TEST_UTIL.getMiniHBaseCluster().getMaster() + .getAssignmentManager().getRegionStates().getRegionsOfTable(table); + assert(regions.size() == numRegions * numReplica); + //also make sure the meta table has the replica locations removed + hris = MetaReader.getTableRegions(ct, table); + assert(hris.size() == numRegions * numReplica); + //just check that the number of default replica regions in the meta table are the same + //as the number of regions the table was created with, and the count of the + //replicas is numReplica for each region + Map defaultReplicas = new HashMap(); + for (HRegionInfo hri : hris) { + Integer i; + HRegionInfo regionReplica0 = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri); + defaultReplicas.put(regionReplica0, + (i = defaultReplicas.get(regionReplica0)) == null ? 1 : i + 1); + } + assert(defaultReplicas.size() == numRegions); + Collection counts = new HashSet(defaultReplicas.values()); + assert(counts.size() == 1 && counts.contains(new Integer(numReplica))); } finally { admin.disableTable(table); admin.deleteTable(table); diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 14ec293..67084d5 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -478,6 +478,7 @@ module Hbase htd.setMemStoreFlushSize(JLong.valueOf(arg.delete(MEMSTORE_FLUSHSIZE))) if arg[MEMSTORE_FLUSHSIZE] htd.setAsyncLogFlush(JBoolean.valueOf(arg.delete(DEFERRED_LOG_FLUSH))) if arg[DEFERRED_LOG_FLUSH] htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(DURABILITY))) if arg[DURABILITY] + htd.setRegionReplication(JInteger.valueOf(arg.delete(REGION_REPLICATION))) if arg[REGION_REPLICATION] set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA] set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] diff --git a/hbase-shell/src/main/ruby/shell/commands/alter.rb b/hbase-shell/src/main/ruby/shell/commands/alter.rb index f22348c..caf35a6 100644 --- a/hbase-shell/src/main/ruby/shell/commands/alter.rb +++ b/hbase-shell/src/main/ruby/shell/commands/alter.rb @@ -74,6 +74,10 @@ You can also remove a table-scope attribute: hbase> alter 't1', METHOD => 'table_att_unset', NAME => 'coprocessor$1' +You can also set REGION_REPLICATION: + + hbase> alter 't1', {REGION_REPLICATION => 2} + There could be more than one alteration in one command: hbase> alter 't1', { NAME => 'f1', VERSIONS => 3 },