From 2e0b8e1b5c1731ef4bcb8cbd61c4f88f31022ca9 Mon Sep 17 00:00:00 2001 From: Ben Lau Date: Mon, 22 Jun 2015 13:20:00 -0700 Subject: [PATCH] Humongous Table Support Credit for most of this code goes to Huaiyu Zhu --- .../java/org/apache/hadoop/hbase/HRegionInfo.java | 117 +++++++++++-- .../org/apache/hadoop/hbase/HTableDescriptor.java | 29 ++++ .../hadoop/hbase/client/RegionReplicaUtil.java | 4 +- .../hbase/protobuf/generated/HBaseProtos.java | 169 ++++++++++++++----- hbase-protocol/src/main/protobuf/HBase.proto | 1 + .../apache/hadoop/hbase/backup/HFileArchiver.java | 7 +- .../java/org/apache/hadoop/hbase/io/HFileLink.java | 68 ++++---- .../org/apache/hadoop/hbase/master/HMaster.java | 5 + .../hadoop/hbase/master/MasterFileSystem.java | 10 +- .../master/procedure/DeleteTableProcedure.java | 3 +- .../hbase/master/snapshot/SnapshotManager.java | 6 + .../apache/hadoop/hbase/regionserver/HRegion.java | 59 ++++--- .../hbase/regionserver/HRegionFileSystem.java | 33 +++- .../apache/hadoop/hbase/regionserver/HStore.java | 16 +- .../regionserver/RegionMergeTransactionImpl.java | 2 +- .../hbase/regionserver/SplitTransactionImpl.java | 10 +- .../hadoop/hbase/regionserver/StoreFileInfo.java | 16 +- .../hbase/snapshot/RestoreSnapshotHelper.java | 15 +- .../java/org/apache/hadoop/hbase/util/FSUtils.java | 31 +++- .../org/apache/hadoop/hbase/util/HBaseFsck.java | 38 ++++- .../apache/hadoop/hbase/util/HFileArchiveUtil.java | 37 +--- .../hadoop/hbase/util/ModifyRegionUtils.java | 7 +- .../hadoop/hbase/util/ServerRegionReplicaUtil.java | 1 + .../hbase/util/hbck/HFileCorruptionChecker.java | 7 +- .../org/apache/hadoop/hbase/wal/WALSplitter.java | 21 ++- .../apache/hadoop/hbase/HBaseTestingUtility.java | 20 ++- .../apache/hadoop/hbase/TestHumongousTable.java | 177 ++++++++++++++++++++ .../hadoop/hbase/backup/TestHFileArchiving.java | 34 +++- .../hbase/backup/TestHumongousHFileArchiving.java | 35 ++++ .../org/apache/hadoop/hbase/io/TestHFileLink.java | 4 +- .../hbase/master/TestDistributedLogSplitting.java | 21 ++- .../TestHumongousTableDistributedLogSplitting.java | 32 ++++ .../hbase/master/cleaner/TestHFileLinkCleaner.java | 7 +- .../regionserver/TestCompactHumongousRegion.java | 37 ++++ .../hadoop/hbase/regionserver/TestCompaction.java | 6 +- .../hbase/regionserver/TestFlushRegionEntry.java | 2 +- .../regionserver/TestHumongousRegionMerge.java | 32 ++++ .../hbase/regionserver/TestHumongousStoreFile.java | 44 +++++ .../regionserver/TestRegionMergeTransaction.java | 18 +- ...estSplitHumongousTableTransactionOnCluster.java | 32 ++++ .../hbase/regionserver/TestSplitTransaction.java | 19 ++- .../TestSplitTransactionForHumongousTable.java | 32 ++++ .../TestSplitTransactionOnCluster.java | 12 +- .../hadoop/hbase/regionserver/TestStoreFile.java | 38 +++-- .../regionserver/TestStripeStoreFileManager.java | 5 +- .../hadoop/hbase/util/HFileArchiveTestingUtil.java | 7 +- .../apache/hadoop/hbase/util/TestHBaseFsck.java | 37 ++-- .../hbase/util/TestHBaseFsckHumongousTable.java | 53 ++++++ .../org/apache/hadoop/hbase/wal/TestWALSplit.java | 9 +- hbase-shell/src/main/ruby/hbase.rb | 1 + hbase-shell/src/main/ruby/hbase/admin.rb | 1 + 51 files changed, 1133 insertions(+), 294 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/TestHumongousTable.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHumongousHFileArchiving.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestHumongousTableDistributedLogSplitting.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactHumongousRegion.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHumongousRegionMerge.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHumongousStoreFile.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitHumongousTableTransactionOnCluster.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionForHumongousTable.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckHumongousTable.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java index c134063..d6cd111 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java @@ -199,6 +199,10 @@ public class HRegionInfo implements Comparable { // Current TableName private TableName tableName = null; + + // Part of a humongous table + private boolean humongous = false; + final static String DISPLAY_KEYS_KEY = "hbase.display.keys"; public final static byte[] HIDDEN_END_KEY = Bytes.toBytes("hidden-end-key"); public final static byte[] HIDDEN_START_KEY = Bytes.toBytes("hidden-start-key"); @@ -218,20 +222,24 @@ public class HRegionInfo implements Comparable { this.hashCode = result; } + public boolean isHumongous() { + return humongous; + } /** * Private constructor used constructing HRegionInfo for the * first meta regions */ private HRegionInfo(long regionId, TableName tableName) { - this(regionId, tableName, DEFAULT_REPLICA_ID); + this(regionId, tableName, DEFAULT_REPLICA_ID, false); } - - public HRegionInfo(long regionId, TableName tableName, int replicaId) { + + public HRegionInfo(long regionId, TableName tableName, int replicaId, boolean humongous) { super(); this.regionId = regionId; this.tableName = tableName; this.replicaId = replicaId; + this.humongous = humongous; // Note: First Meta region replicas names are in old format this.regionName = createRegionName(tableName, null, regionId, replicaId, false); setHashCode(); @@ -240,7 +248,7 @@ public class HRegionInfo implements Comparable { public HRegionInfo(final TableName tableName) { this(tableName, null, null); } - + /** * Construct HRegionInfo with explicit parameters * @@ -284,9 +292,41 @@ public class HRegionInfo implements Comparable { public HRegionInfo(final TableName tableName, final byte[] startKey, final byte[] endKey, final boolean split, final long regionid) throws IllegalArgumentException { - this(tableName, startKey, endKey, split, regionid, DEFAULT_REPLICA_ID); + this(tableName, false, startKey, endKey, split, regionid, DEFAULT_REPLICA_ID); } - + + /** + * Construct HRegionInfo with explicit parameters + * + * @param tableName the table descriptor + * @param humongous whether region is humongous + * @param startKey first key in region + * @param endKey end of key range + * @param split true if this region has split and we have daughter regions + * regions that may or may not hold references to this region. + * @param regionid Region id to use. + * @throws IllegalArgumentException + */ + public HRegionInfo(final TableName tableName, final boolean humongous, final byte[] startKey, + final byte[] endKey, final boolean split, final long regionid) + throws IllegalArgumentException { + this(tableName, humongous, startKey, endKey, split, regionid, DEFAULT_REPLICA_ID); + } + + /** + * Construct HRegionInfo with explicit parameters + * + * @param tableName the table name + * @param humongous whether region is humongous + * @param startKey first key in region + * @param endKey end of key range + * @throws IllegalArgumentException + */ + public HRegionInfo(final TableName tableName, final boolean humongous, final byte[] startKey, final byte[] endKey) + throws IllegalArgumentException { + this(tableName, humongous, startKey, endKey, false, System.currentTimeMillis(), DEFAULT_REPLICA_ID); + } + /** * Construct HRegionInfo with explicit parameters * @@ -303,11 +343,32 @@ public class HRegionInfo implements Comparable { final byte[] endKey, final boolean split, final long regionid, final int replicaId) throws IllegalArgumentException { + this(tableName, false, startKey, endKey, split, regionid, replicaId); + } + + /** + * Construct HRegionInfo with explicit parameters + * + * @param tableName the table descriptor + * @param humongous whether the region is for a humongous table + * @param startKey first key in region + * @param endKey end of key range + * @param split true if this region has split and we have daughter regions + * regions that may or may not hold references to this region. + * @param regionid Region id to use. + * @param replicaId the replicaId to use + * @throws IllegalArgumentException + */ + public HRegionInfo(final TableName tableName, final boolean humongous, final byte[] startKey, + final byte[] endKey, final boolean split, final long regionid, + final int replicaId) + throws IllegalArgumentException { super(); if (tableName == null) { throw new IllegalArgumentException("TableName cannot be null"); } this.tableName = tableName; + this.humongous = humongous; this.offLine = false; this.regionId = regionid; this.replicaId = replicaId; @@ -322,6 +383,22 @@ public class HRegionInfo implements Comparable { this.startKey = startKey == null? HConstants.EMPTY_START_ROW: startKey.clone(); this.tableName = tableName; + this.humongous = humongous; + setHashCode(); + } + + /** + * Construct a HRegionInfo directly from tableName and provided encodedName instead of + * generating the encodedName automatically. + * + * @param tableName the table name + * @param encodedName region encodedName + * @param humongous region humongous attribute + */ + public HRegionInfo(TableName tableName, String encodedName, boolean humongous) { + this.tableName = tableName; + this.encodedName = encodedName; + this.humongous = humongous; setHashCode(); } @@ -341,6 +418,7 @@ public class HRegionInfo implements Comparable { this.hashCode = other.hashCode(); this.encodedName = other.getEncodedName(); this.tableName = other.tableName; + this.humongous = other.humongous; this.replicaId = other.replicaId; } @@ -756,7 +834,8 @@ public class HRegionInfo implements Comparable { Bytes.toStringBinary(this.endKey) + "'" + (isOffline()? ", OFFLINE => true": "") + (isSplit()? ", SPLIT => true": "") + - ((replicaId > 0)? ", REPLICA_ID => " + replicaId : "") + "}"; + ((replicaId > 0)? ", REPLICA_ID => " + replicaId : "") + + (isHumongous()? ", HUMONGOUS => true": "") + "}"; } /** @@ -832,11 +911,19 @@ public class HRegionInfo implements Comparable { int replicaDiff = this.getReplicaId() - o.getReplicaId(); if (replicaDiff != 0) return replicaDiff; - if (this.offLine == o.offLine) - return 0; - if (this.offLine == true) return -1; - - return 1; + if (this.offLine && !o.offLine) { + return -1; + } else if (!this.offLine && o.offLine) { + return 1; + } + + // define humongous regions to be bigger than normal regions + if (this.humongous && !o.humongous) { + return 1; + } else if (!this.humongous && o.humongous) { + return -1; + } + return 0; } /** @@ -868,6 +955,7 @@ public class HRegionInfo implements Comparable { if (info == null) return null; RegionInfo.Builder builder = RegionInfo.newBuilder(); builder.setTableName(ProtobufUtil.toProtoTableName(info.getTable())); + builder.setHumongous(info.isHumongous()); builder.setRegionId(info.getRegionId()); if (info.getStartKey() != null) { builder.setStartKey(ByteStringer.wrap(info.getStartKey())); @@ -895,6 +983,10 @@ public class HRegionInfo implements Comparable { return RegionReplicaUtil.getRegionInfoForReplica(FIRST_META_REGIONINFO, proto.getReplicaId()); } + boolean isHumongous = false; + if(proto.hasHumongous()) { + isHumongous = proto.getHumongous(); + } long regionId = proto.getRegionId(); int replicaId = proto.hasReplicaId() ? proto.getReplicaId() : DEFAULT_REPLICA_ID; byte[] startKey = null; @@ -911,6 +1003,7 @@ public class HRegionInfo implements Comparable { } HRegionInfo hri = new HRegionInfo( tableName, + isHumongous, startKey, endKey, split, regionId, replicaId); if (proto.hasOffline()) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index 58067ea..55ecb47 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -188,6 +188,14 @@ public class HTableDescriptor implements Comparable { /** Default durability for HTD is USE_DEFAULT, which defaults to HBase-global default value */ private static final Durability DEFAULT_DURABLITY = Durability.USE_DEFAULT; + /** + * INTERNAL Used by HRegion to tell if the table is humongous. + */ + public static final String HUMONGOUS = "HUMONGOUS"; + private static final Bytes HUMONGOUS_KEY = + new Bytes(Bytes.toBytes("HUMONGOUS")); + private static final boolean DEFAULT_HUMONGOUS = false; + /* * The below are ugly but better than creating them each time till we * replace booleans being saved as Strings with plain booleans. Need a @@ -235,6 +243,7 @@ public class HTableDescriptor implements Comparable { DEFAULT_VALUES.put(DEFERRED_LOG_FLUSH, String.valueOf(DEFAULT_DEFERRED_LOG_FLUSH)); DEFAULT_VALUES.put(DURABILITY, DEFAULT_DURABLITY.name()); //use the enum name + DEFAULT_VALUES.put(HUMONGOUS, String.valueOf(DEFAULT_HUMONGOUS)); DEFAULT_VALUES.put(REGION_REPLICATION, String.valueOf(DEFAULT_REGION_REPLICATION)); for (String s : DEFAULT_VALUES.keySet()) { RESERVED_KEYWORDS.add(new Bytes(Bytes.toBytes(s))); @@ -594,6 +603,25 @@ public class HTableDescriptor implements Comparable { } /** + * Check if the humongous flag of the table is set. If the humongous flag is + * set then regions are organized by sub directories on hdfs. + * + * @return true if table is humongous + */ + public boolean isHumongous() { + return isSomething(HUMONGOUS_KEY, DEFAULT_HUMONGOUS); + } + + /** + * Setting the table as humongous. + * + * @param humongous True if table is humongous. + */ + public HTableDescriptor setHumongous(final boolean humongous) { + return setValue(HUMONGOUS_KEY, humongous? TRUE: FALSE); + } + + /** * Check if the compaction enable flag of the table is true. If flag is * false then no minor/major compactions will be done in real. * @@ -946,6 +974,7 @@ public class HTableDescriptor implements Comparable { } s.append("}"); } + s.append("}"); } // step 3: printing all configuration: diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionReplicaUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionReplicaUtil.java index c2dcbc0..6257299 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionReplicaUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RegionReplicaUtil.java @@ -63,9 +63,9 @@ public class RegionReplicaUtil { } HRegionInfo replicaInfo; if (regionInfo.isMetaRegion()) { - replicaInfo = new HRegionInfo(regionInfo.getRegionId(), regionInfo.getTable(), replicaId); + replicaInfo = new HRegionInfo(regionInfo.getRegionId(), regionInfo.getTable(), replicaId, false); } else { - replicaInfo = new HRegionInfo(regionInfo.getTable(), regionInfo.getStartKey(), + replicaInfo = new HRegionInfo(regionInfo.getTable(), regionInfo.isHumongous(), regionInfo.getStartKey(), regionInfo.getEndKey(), regionInfo.isSplit(), regionInfo.getRegionId(), replicaId); } replicaInfo.setOffline(regionInfo.isOffline()); diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java index 3007d25..6ba7d3a 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/HBaseProtos.java @@ -4988,6 +4988,16 @@ public final class HBaseProtos { * optional int32 replica_id = 7 [default = 0]; */ int getReplicaId(); + + // optional bool humongous = 8; + /** + * optional bool humongous = 8; + */ + boolean hasHumongous(); + /** + * optional bool humongous = 8; + */ + boolean getHumongous(); } /** * Protobuf type {@code RegionInfo} @@ -5088,6 +5098,11 @@ public final class HBaseProtos { replicaId_ = input.readInt32(); break; } + case 64: { + bitField0_ |= 0x00000080; + humongous_ = input.readBool(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -5246,6 +5261,22 @@ public final class HBaseProtos { return replicaId_; } + // optional bool humongous = 8; + public static final int HUMONGOUS_FIELD_NUMBER = 8; + private boolean humongous_; + /** + * optional bool humongous = 8; + */ + public boolean hasHumongous() { + return ((bitField0_ & 0x00000080) == 0x00000080); + } + /** + * optional bool humongous = 8; + */ + public boolean getHumongous() { + return humongous_; + } + private void initFields() { regionId_ = 0L; tableName_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TableName.getDefaultInstance(); @@ -5254,6 +5285,7 @@ public final class HBaseProtos { offline_ = false; split_ = false; replicaId_ = 0; + humongous_ = false; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -5300,6 +5332,9 @@ public final class HBaseProtos { if (((bitField0_ & 0x00000040) == 0x00000040)) { output.writeInt32(7, replicaId_); } + if (((bitField0_ & 0x00000080) == 0x00000080)) { + output.writeBool(8, humongous_); + } getUnknownFields().writeTo(output); } @@ -5337,6 +5372,10 @@ public final class HBaseProtos { size += com.google.protobuf.CodedOutputStream .computeInt32Size(7, replicaId_); } + if (((bitField0_ & 0x00000080) == 0x00000080)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(8, humongous_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -5395,6 +5434,11 @@ public final class HBaseProtos { result = result && (getReplicaId() == other.getReplicaId()); } + result = result && (hasHumongous() == other.hasHumongous()); + if (hasHumongous()) { + result = result && (getHumongous() + == other.getHumongous()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -5436,6 +5480,10 @@ public final class HBaseProtos { hash = (37 * hash) + REPLICA_ID_FIELD_NUMBER; hash = (53 * hash) + getReplicaId(); } + if (hasHumongous()) { + hash = (37 * hash) + HUMONGOUS_FIELD_NUMBER; + hash = (53 * hash) + hashBoolean(getHumongous()); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -5569,6 +5617,8 @@ public final class HBaseProtos { bitField0_ = (bitField0_ & ~0x00000020); replicaId_ = 0; bitField0_ = (bitField0_ & ~0x00000040); + humongous_ = false; + bitField0_ = (bitField0_ & ~0x00000080); return this; } @@ -5629,6 +5679,10 @@ public final class HBaseProtos { to_bitField0_ |= 0x00000040; } result.replicaId_ = replicaId_; + if (((from_bitField0_ & 0x00000080) == 0x00000080)) { + to_bitField0_ |= 0x00000080; + } + result.humongous_ = humongous_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -5666,6 +5720,9 @@ public final class HBaseProtos { if (other.hasReplicaId()) { setReplicaId(other.getReplicaId()); } + if (other.hasHumongous()) { + setHumongous(other.getHumongous()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -6026,6 +6083,39 @@ public final class HBaseProtos { return this; } + // optional bool humongous = 8; + private boolean humongous_ ; + /** + * optional bool humongous = 8; + */ + public boolean hasHumongous() { + return ((bitField0_ & 0x00000080) == 0x00000080); + } + /** + * optional bool humongous = 8; + */ + public boolean getHumongous() { + return humongous_; + } + /** + * optional bool humongous = 8; + */ + public Builder setHumongous(boolean value) { + bitField0_ |= 0x00000080; + humongous_ = value; + onChanged(); + return this; + } + /** + * optional bool humongous = 8; + */ + public Builder clearHumongous() { + bitField0_ = (bitField0_ & ~0x00000080); + humongous_ = false; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:RegionInfo) } @@ -17794,46 +17884,47 @@ public final class HBaseProtos { "bleSchema\"o\n\022ColumnFamilySchema\022\014\n\004name\030" + "\001 \002(\014\022#\n\nattributes\030\002 \003(\0132\017.BytesBytesPa" + "ir\022&\n\rconfiguration\030\003 \003(\0132\017.NameStringPa" + - "ir\"\232\001\n\nRegionInfo\022\021\n\tregion_id\030\001 \002(\004\022\036\n\n" + + "ir\"\255\001\n\nRegionInfo\022\021\n\tregion_id\030\001 \002(\004\022\036\n\n" + "table_name\030\002 \002(\0132\n.TableName\022\021\n\tstart_ke" + "y\030\003 \001(\014\022\017\n\007end_key\030\004 \001(\014\022\017\n\007offline\030\005 \001(" + - "\010\022\r\n\005split\030\006 \001(\010\022\025\n\nreplica_id\030\007 \001(\005:\0010\"" + - "1\n\014FavoredNodes\022!\n\014favored_node\030\001 \003(\0132\013." + - "ServerName\"\225\001\n\017RegionSpecifier\0222\n\004type\030\001" + - " \002(\0162$.RegionSpecifier.RegionSpecifierTy", - "pe\022\r\n\005value\030\002 \002(\014\"?\n\023RegionSpecifierType" + - "\022\017\n\013REGION_NAME\020\001\022\027\n\023ENCODED_REGION_NAME" + - "\020\002\"%\n\tTimeRange\022\014\n\004from\030\001 \001(\004\022\n\n\002to\030\002 \001(" + - "\004\"A\n\nServerName\022\021\n\thost_name\030\001 \002(\t\022\014\n\004po" + - "rt\030\002 \001(\r\022\022\n\nstart_code\030\003 \001(\004\"\033\n\013Coproces" + - "sor\022\014\n\004name\030\001 \002(\t\"-\n\016NameStringPair\022\014\n\004n" + - "ame\030\001 \002(\t\022\r\n\005value\030\002 \002(\t\",\n\rNameBytesPai" + - "r\022\014\n\004name\030\001 \002(\t\022\r\n\005value\030\002 \001(\014\"/\n\016BytesB" + - "ytesPair\022\r\n\005first\030\001 \002(\014\022\016\n\006second\030\002 \002(\014\"" + - ",\n\rNameInt64Pair\022\014\n\004name\030\001 \001(\t\022\r\n\005value\030", - "\002 \001(\003\"\314\001\n\023SnapshotDescription\022\014\n\004name\030\001 " + - "\002(\t\022\r\n\005table\030\002 \001(\t\022\030\n\rcreation_time\030\003 \001(" + - "\003:\0010\022.\n\004type\030\004 \001(\0162\031.SnapshotDescription" + - ".Type:\005FLUSH\022\017\n\007version\030\005 \001(\005\022\r\n\005owner\030\006" + - " \001(\t\".\n\004Type\022\014\n\010DISABLED\020\000\022\t\n\005FLUSH\020\001\022\r\n" + - "\tSKIPFLUSH\020\002\"}\n\024ProcedureDescription\022\021\n\t" + - "signature\030\001 \002(\t\022\020\n\010instance\030\002 \001(\t\022\030\n\rcre" + - "ation_time\030\003 \001(\003:\0010\022&\n\rconfiguration\030\004 \003" + - "(\0132\017.NameStringPair\"\n\n\010EmptyMsg\"\033\n\007LongM" + - "sg\022\020\n\010long_msg\030\001 \002(\003\"\037\n\tDoubleMsg\022\022\n\ndou", - "ble_msg\030\001 \002(\001\"\'\n\rBigDecimalMsg\022\026\n\016bigdec" + - "imal_msg\030\001 \002(\014\"5\n\004UUID\022\026\n\016least_sig_bits" + - "\030\001 \002(\004\022\025\n\rmost_sig_bits\030\002 \002(\004\"K\n\023Namespa" + - "ceDescriptor\022\014\n\004name\030\001 \002(\014\022&\n\rconfigurat" + - "ion\030\002 \003(\0132\017.NameStringPair\"$\n\020RegionServ" + - "erInfo\022\020\n\010infoPort\030\001 \001(\005*r\n\013CompareType\022" + - "\010\n\004LESS\020\000\022\021\n\rLESS_OR_EQUAL\020\001\022\t\n\005EQUAL\020\002\022" + - "\r\n\tNOT_EQUAL\020\003\022\024\n\020GREATER_OR_EQUAL\020\004\022\013\n\007" + - "GREATER\020\005\022\t\n\005NO_OP\020\006*n\n\010TimeUnit\022\017\n\013NANO" + - "SECONDS\020\001\022\020\n\014MICROSECONDS\020\002\022\020\n\014MILLISECO", - "NDS\020\003\022\013\n\007SECONDS\020\004\022\013\n\007MINUTES\020\005\022\t\n\005HOURS" + - "\020\006\022\010\n\004DAYS\020\007B>\n*org.apache.hadoop.hbase." + - "protobuf.generatedB\013HBaseProtosH\001\240\001\001" + "\010\022\r\n\005split\030\006 \001(\010\022\025\n\nreplica_id\030\007 \001(\005:\0010\022" + + "\021\n\thumongous\030\010 \001(\010\"1\n\014FavoredNodes\022!\n\014fa" + + "vored_node\030\001 \003(\0132\013.ServerName\"\225\001\n\017Region" + + "Specifier\0222\n\004type\030\001 \002(\0162$.RegionSpecifie", + "r.RegionSpecifierType\022\r\n\005value\030\002 \002(\014\"?\n\023" + + "RegionSpecifierType\022\017\n\013REGION_NAME\020\001\022\027\n\023" + + "ENCODED_REGION_NAME\020\002\"%\n\tTimeRange\022\014\n\004fr" + + "om\030\001 \001(\004\022\n\n\002to\030\002 \001(\004\"A\n\nServerName\022\021\n\tho" + + "st_name\030\001 \002(\t\022\014\n\004port\030\002 \001(\r\022\022\n\nstart_cod" + + "e\030\003 \001(\004\"\033\n\013Coprocessor\022\014\n\004name\030\001 \002(\t\"-\n\016" + + "NameStringPair\022\014\n\004name\030\001 \002(\t\022\r\n\005value\030\002 " + + "\002(\t\",\n\rNameBytesPair\022\014\n\004name\030\001 \002(\t\022\r\n\005va" + + "lue\030\002 \001(\014\"/\n\016BytesBytesPair\022\r\n\005first\030\001 \002" + + "(\014\022\016\n\006second\030\002 \002(\014\",\n\rNameInt64Pair\022\014\n\004n", + "ame\030\001 \001(\t\022\r\n\005value\030\002 \001(\003\"\314\001\n\023SnapshotDes" + + "cription\022\014\n\004name\030\001 \002(\t\022\r\n\005table\030\002 \001(\t\022\030\n" + + "\rcreation_time\030\003 \001(\003:\0010\022.\n\004type\030\004 \001(\0162\031." + + "SnapshotDescription.Type:\005FLUSH\022\017\n\007versi" + + "on\030\005 \001(\005\022\r\n\005owner\030\006 \001(\t\".\n\004Type\022\014\n\010DISAB" + + "LED\020\000\022\t\n\005FLUSH\020\001\022\r\n\tSKIPFLUSH\020\002\"}\n\024Proce" + + "dureDescription\022\021\n\tsignature\030\001 \002(\t\022\020\n\010in" + + "stance\030\002 \001(\t\022\030\n\rcreation_time\030\003 \001(\003:\0010\022&" + + "\n\rconfiguration\030\004 \003(\0132\017.NameStringPair\"\n" + + "\n\010EmptyMsg\"\033\n\007LongMsg\022\020\n\010long_msg\030\001 \002(\003\"", + "\037\n\tDoubleMsg\022\022\n\ndouble_msg\030\001 \002(\001\"\'\n\rBigD" + + "ecimalMsg\022\026\n\016bigdecimal_msg\030\001 \002(\014\"5\n\004UUI" + + "D\022\026\n\016least_sig_bits\030\001 \002(\004\022\025\n\rmost_sig_bi" + + "ts\030\002 \002(\004\"K\n\023NamespaceDescriptor\022\014\n\004name\030" + + "\001 \002(\014\022&\n\rconfiguration\030\002 \003(\0132\017.NameStrin" + + "gPair\"$\n\020RegionServerInfo\022\020\n\010infoPort\030\001 " + + "\001(\005*r\n\013CompareType\022\010\n\004LESS\020\000\022\021\n\rLESS_OR_" + + "EQUAL\020\001\022\t\n\005EQUAL\020\002\022\r\n\tNOT_EQUAL\020\003\022\024\n\020GRE" + + "ATER_OR_EQUAL\020\004\022\013\n\007GREATER\020\005\022\t\n\005NO_OP\020\006*" + + "n\n\010TimeUnit\022\017\n\013NANOSECONDS\020\001\022\020\n\014MICROSEC", + "ONDS\020\002\022\020\n\014MILLISECONDS\020\003\022\013\n\007SECONDS\020\004\022\013\n" + + "\007MINUTES\020\005\022\t\n\005HOURS\020\006\022\010\n\004DAYS\020\007B>\n*org.a" + + "pache.hadoop.hbase.protobuf.generatedB\013H" + + "BaseProtosH\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -17875,7 +17966,7 @@ public final class HBaseProtos { internal_static_RegionInfo_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_RegionInfo_descriptor, - new java.lang.String[] { "RegionId", "TableName", "StartKey", "EndKey", "Offline", "Split", "ReplicaId", }); + new java.lang.String[] { "RegionId", "TableName", "StartKey", "EndKey", "Offline", "Split", "ReplicaId", "Humongous", }); internal_static_FavoredNodes_descriptor = getDescriptor().getMessageTypes().get(6); internal_static_FavoredNodes_fieldAccessorTable = new diff --git a/hbase-protocol/src/main/protobuf/HBase.proto b/hbase-protocol/src/main/protobuf/HBase.proto index 00e2850..17d693a 100644 --- a/hbase-protocol/src/main/protobuf/HBase.proto +++ b/hbase-protocol/src/main/protobuf/HBase.proto @@ -83,6 +83,7 @@ message RegionInfo { optional bool offline = 5; optional bool split = 6; optional int32 replica_id = 7 [default = 0]; + optional bool humongous = 8; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java index d682ccc..b715262 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/HFileArchiver.java @@ -106,9 +106,7 @@ public class HFileArchiver { // make sure the regiondir lives under the tabledir Preconditions.checkArgument(regionDir.toString().startsWith(tableDir.toString())); - Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(rootdir, - FSUtils.getTableName(tableDir), - regionDir.getName()); + Path regionArchiveDir = HFileArchiveUtil.getRegionArchiveDir(rootdir, FSUtils.getTableName(tableDir), regionDir); FileStatusConverter getAsFile = new FileStatusConverter(fs); // otherwise, we attempt to archive the store files @@ -161,7 +159,8 @@ public class HFileArchiver { */ public static void archiveFamily(FileSystem fs, Configuration conf, HRegionInfo parent, Path tableDir, byte[] family) throws IOException { - Path familyDir = new Path(tableDir, new Path(parent.getEncodedName(), Bytes.toString(family))); + Path regionDir = HRegion.getRegionDir(FSUtils.getRootDir(conf), parent); + Path familyDir = new Path(regionDir, Bytes.toString(family)); FileStatus[] storeFiles = FSUtils.listStatus(fs, familyDir); if (storeFiles == null) { LOG.debug("No store files to dispose for region=" + parent.getRegionNameAsString() + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java index c17720c..4b9ea78 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java @@ -32,7 +32,9 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.Pair; @@ -216,8 +218,24 @@ public class HFileLink extends FileLink { String hfileName = m.group(4); String familyName = path.getParent().getName(); Path tableDir = FSUtils.getTableDir(new Path("./"), tableName); - return new Path(tableDir, new Path(regionName, new Path(familyName, - hfileName))); + // TODO: HRegion.isHumongousRegionDir doesn't always work here since path can be + // relative (e.g. family/hfilelink). If we want to enable snapshot on humongous + // table, we need new format of hfilelink name (e.g. humongous:table=region-hfile) + // For now, snapshots on humongous tables are a bit unrealistic and we will block + // them from happening in SnapshotManager by throwing an exception. + // Timeline-consistent read-only region replicas also rely on HFileLinks + // and will also be incompatible with humongous tables. + if (HRegion.isHumongousRegionDir(path.getParent().getParent())) { + // if table is humongous, convert the HFileLink name into a real + // table/bucket/region/cf/hfile path. + String bucket = regionName.substring(HRegionInfo.MD5_HEX_LENGTH + - HRegionFileSystem.HUMONGOUS_DIR_NAME_SIZE); + return new Path(tableDir, new Path(new Path(bucket, regionName), + new Path(familyName, hfileName))); + } else { + return new Path(tableDir, new Path(regionName, new Path(familyName, + hfileName))); + } } /** @@ -310,41 +328,23 @@ public class HFileLink extends FileLink { final String hfileName) throws IOException { TableName linkedTable = hfileRegionInfo.getTable(); String linkedRegion = hfileRegionInfo.getEncodedName(); - return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName); - } - - /** - * Create a new HFileLink - * - *

It also adds a back-reference to the hfile back-reference directory - * to simplify the reference-count and the cleaning process. - * - * @param conf {@link Configuration} to read for the archive directory name - * @param fs {@link FileSystem} on which to write the HFileLink - * @param dstFamilyPath - Destination path (table/region/cf/) - * @param linkedTable - Linked Table Name - * @param linkedRegion - Linked Region Name - * @param hfileName - Linked HFile name - * @return true if the file is created, otherwise the file exists. - * @throws IOException on file or parent directory creation failure - */ - public static boolean create(final Configuration conf, final FileSystem fs, - final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion, - final String hfileName) throws IOException { String familyName = dstFamilyPath.getName(); String regionName = dstFamilyPath.getParent().getName(); - String tableName = FSUtils.getTableName(dstFamilyPath.getParent().getParent()) - .getNameAsString(); - + Path regionDir = dstFamilyPath.getParent(); + Path tableDir = regionDir.getParent(); + if(HRegion.isHumongousRegionDir(regionDir)) { + tableDir = tableDir.getParent(); + } + String tableName = FSUtils.getTableName(tableDir).getNameAsString(); String name = createHFileLinkName(linkedTable, linkedRegion, hfileName); String refName = createBackReferenceName(tableName, regionName); // Make sure the destination directory exists fs.mkdirs(dstFamilyPath); - // Make sure the FileLink reference directory exists Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, - linkedTable, linkedRegion, familyName); + hfileRegionInfo, FSUtils.getTableDir(FSUtils.getRootDir(conf), + linkedTable), Bytes.toBytes(familyName)); Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName); fs.mkdirs(backRefssDir); @@ -376,13 +376,15 @@ public class HFileLink extends FileLink { * @throws IOException on file or parent directory creation failure */ public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs, - final Path dstFamilyPath, final String hfileLinkName) throws IOException { + final Path dstFamilyPath, final String hfileLinkName, final HRegionInfo origin) throws IOException { Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName); if (!m.matches()) { throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!"); } - return create(conf, fs, dstFamilyPath, TableName.valueOf(m.group(1), m.group(2)), - m.group(3), m.group(4)); + // construct a new HRegionInfo where the hfileLink are pointing at + HRegionInfo hfileRegionInfo = new HRegionInfo(TableName.valueOf(m.group(1), + m.group(2)), m.group(3), origin.isHumongous()); + return create(conf, fs, dstFamilyPath, hfileRegionInfo, m.group(4)); } /** @@ -415,7 +417,9 @@ public class HFileLink extends FileLink { String linkName = createHFileLinkName(FSUtils.getTableName(tablePath), regionPath.getName(), hfileName); Path linkTableDir = FSUtils.getTableDir(rootDir, linkTableName); - Path regionDir = HRegion.getRegionDir(linkTableDir, linkRegionName); + // TODO: Another place that needs to be updated if we want snapshots/HFileLinks to work + // with humongous tables + Path regionDir = new Path(linkTableDir, linkRegionName); return new Path(new Path(regionDir, familyPath.getName()), linkName); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 86bcdae..680674f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -1390,6 +1390,11 @@ public class HMaster extends HRegionServer implements MasterServices, Server { if (tableVal != null && !Boolean.valueOf(tableVal)) { logWarn = true; } + + if (htd.isHumongous() && htd.getRegionReplication() > 1) { + warnOrThrowExceptionForFailure(logWarn, CONF_KEY, + "Humongous tables do not support region replicas currently", null); + } // check max file size long maxFileSizeLowerLimit = 2 * 1024 * 1024L; // 2M is the default lower limit diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java index 3718a5a..a06904c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java @@ -604,9 +604,9 @@ public class MasterFileSystem { Path tableDir = FSUtils.getTableDir(rootdir, region.getTable()); HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName); - // delete the family folder - Path familyDir = new Path(tableDir, - new Path(region.getEncodedName(), Bytes.toString(familyName))); + Path regionDir = getRegionDir(tableDir, region); + Path familyDir = new Path(regionDir, Bytes.toString(familyName)); + if (fs.delete(familyDir, true) == false) { if (fs.exists(familyDir)) { throw new IOException("Could not delete family " @@ -617,6 +617,10 @@ public class MasterFileSystem { } } + public Path getRegionDir(Path tableDir, HRegionInfo regionInfo) { + return HRegion.getRegionDir(tableDir, regionInfo.getEncodedName(), regionInfo.isHumongous()); + } + public void stop() { if (splitLogManager != null) { this.splitLogManager.stop(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java index 60212e8..935bde5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.exceptions.HBaseException; -import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.MasterCoprocessorHost; import org.apache.hadoop.hbase.master.MasterFileSystem; @@ -337,7 +336,7 @@ public class DeleteTableProcedure for (HRegionInfo hri : regions) { LOG.debug("Archiving region " + hri.getRegionNameAsString() + " from FS"); HFileArchiver.archiveRegion(fs, mfs.getRootDir(), - tempTableDir, HRegion.getRegionDir(tempTableDir, hri.getEncodedName())); + tempTableDir, mfs.getRegionDir(tempTableDir, hri)); } LOG.debug("Table '" + tableName + "' archived!"); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java index 20cbcc7..350a917 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java @@ -562,6 +562,12 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable throw new SnapshotCreationException("Table '" + snapshot.getTable() + "' doesn't exist, can't take snapshot.", snapshot); } + + if (desc.isHumongous()) { + throw new SnapshotCreationException("Table '" + snapshot.getTable() + + "' is a humongous table and we do not support creating snapshots for humongous tables."); + } + SnapshotDescription.Builder builder = snapshot.toBuilder(); // if not specified, set the snapshot format if (!snapshot.hasVersion()) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index a3e862d..b661194 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -6231,33 +6231,54 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi Bytes.toBytes(HConstants.META_VERSION))); meta.put(row, HConstants.CATALOG_FAMILY, cells); } - - /** - * Computes the Path of the HRegion - * - * @param tabledir qualified path for table - * @param name ENCODED region name - * @return Path of HRegion directory - * @deprecated For tests only; to be removed. - */ - @Deprecated - public static Path getRegionDir(final Path tabledir, final String name) { - return new Path(tabledir, name); - } - + /** * Computes the Path of the HRegion * * @param rootdir qualified path of HBase root directory * @param info HRegionInfo for the region * @return qualified path of region directory - * @deprecated For tests only; to be removed. */ - @Deprecated - @VisibleForTesting public static Path getRegionDir(final Path rootdir, final HRegionInfo info) { - return new Path( - FSUtils.getTableDir(rootdir, info.getTable()), info.getEncodedName()); + return getRegionDir(FSUtils.getTableDir(rootdir, info.getTable()), info.getEncodedName(), + info.isHumongous()); + } + + public static Path getRegionDir(final Path tableDir, final String name, final boolean humongous) { + if (humongous) { + return getHumongousRegionDir(tableDir, name); + } else { + return getNormalRegionDir(tableDir, name); + } + } + + public static Path getNormalRegionDir(final Path tabledir, final String name) { + return new Path(tabledir, name); + } + + public static Path getHumongousRegionDir(final Path tabledir, final String name) { + if (name.length() != HRegionInfo.MD5_HEX_LENGTH) { + throw new IllegalArgumentException("The region with encoded name " + name + + " is not a humongous region, cannot get humongous region dir from it."); + } + return new Path(new Path(tabledir, + name.substring(HRegionInfo.MD5_HEX_LENGTH + - HRegionFileSystem.HUMONGOUS_DIR_NAME_SIZE)), name); + } + + public static boolean isHumongousRegionDir(final Path regionDir) { + int idx = HRegionInfo.MD5_HEX_LENGTH - HRegionFileSystem.HUMONGOUS_DIR_NAME_SIZE; + return regionDir.getName().length() == HRegionInfo.MD5_HEX_LENGTH + && regionDir.getName().substring(idx).equals(regionDir.getParent().getName()) + && regionDir.getParent().getParent().getParent().getParent().getName() + .equals(HConstants.BASE_NAMESPACE_DIR); + } + + public static boolean isNormalRegionDir(final Path regionDir) { + return (regionDir.getName().length() == HRegionInfo.MD5_HEX_LENGTH + || regionDir.getParent().getName().equals(TableName.OLD_ROOT_TABLE_NAME.getQualifierAsString()) + || regionDir.getParent().getName().equals(TableName.META_TABLE_NAME.getQualifierAsString())) + && regionDir.getParent().getParent().getParent().getName().equals(HConstants.BASE_NAMESPACE_DIR); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java index f4eaaf9..3c40026 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java @@ -27,6 +27,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -80,6 +81,9 @@ public class HRegionFileSystem { private final Configuration conf; private final Path tableDir; private final FileSystem fs; + + /** Number of characters for DIR name, 2 characters for 16 * 16 = 256 buckets. */ + public static final int HUMONGOUS_DIR_NAME_SIZE = 4; /** * In order to handle NN connectivity hiccups, one need to retry non-idempotent operation at the @@ -131,7 +135,11 @@ public class HRegionFileSystem { /** @return {@link Path} to the region directory. */ public Path getRegionDir() { - return new Path(this.tableDir, this.regionInfoForFs.getEncodedName()); + return getRegionDir(this.tableDir, this.regionInfoForFs); + } + + private Path getRegionDir(Path tableDir, HRegionInfo regionInfoForFs) { + return HRegion.getRegionDir(tableDir, regionInfoForFs.getEncodedName(), regionInfoForFs.isHumongous()); } // =========================================================================== @@ -512,7 +520,7 @@ public class HRegionFileSystem { * @throws IOException */ void cleanupDaughterRegion(final HRegionInfo regionInfo) throws IOException { - Path regionDir = new Path(this.tableDir, regionInfo.getEncodedName()); + Path regionDir = getRegionDir(this.tableDir, regionInfo); if (this.fs.exists(regionDir) && !deleteDir(regionDir)) { throw new IOException("Failed delete of " + regionDir); } @@ -527,7 +535,7 @@ public class HRegionFileSystem { */ Path commitDaughterRegion(final HRegionInfo regionInfo) throws IOException { - Path regionDir = new Path(this.tableDir, regionInfo.getEncodedName()); + Path regionDir = getRegionDir(this.tableDir, regionInfo); Path daughterTmpDir = this.getSplitsDir(regionInfo); if (fs.exists(daughterTmpDir)) { @@ -536,7 +544,10 @@ public class HRegionFileSystem { Path regionInfoFile = new Path(daughterTmpDir, REGION_INFO_FILE); byte[] regionInfoContent = getRegionInfoFileContent(regionInfo); writeRegionInfoFileContent(conf, fs, regionInfoFile, regionInfoContent); - + + if (regionInfo.isHumongous()) { + fs.mkdirs(regionDir.getParent()); + } // Move the daughter temp dir to the table dir if (!rename(daughterTmpDir, regionDir)) { throw new IOException("Unable to rename " + daughterTmpDir + " to " + regionDir); @@ -652,7 +663,7 @@ public class HRegionFileSystem { * @throws IOException */ void cleanupMergedRegion(final HRegionInfo mergedRegion) throws IOException { - Path regionDir = new Path(this.tableDir, mergedRegion.getEncodedName()); + Path regionDir = getRegionDir(this.tableDir, mergedRegion); if (this.fs.exists(regionDir) && !this.fs.delete(regionDir, true)) { throw new IOException("Failed delete of " + regionDir); } @@ -713,8 +724,11 @@ public class HRegionFileSystem { * @throws IOException */ void commitMergedRegion(final HRegionInfo mergedRegionInfo) throws IOException { - Path regionDir = new Path(this.tableDir, mergedRegionInfo.getEncodedName()); + Path regionDir = getRegionDir(this.tableDir, mergedRegionInfo); Path mergedRegionTmpDir = this.getMergesDir(mergedRegionInfo); + if(mergedRegionInfo.isHumongous()) { + fs.mkdirs(regionDir.getParent()); + } // Move the tmp dir in the expected location if (mergedRegionTmpDir != null && fs.exists(mergedRegionTmpDir)) { if (!fs.rename(mergedRegionTmpDir, regionDir)) { @@ -865,10 +879,13 @@ public class HRegionFileSystem { * @param fs {@link FileSystem} from which to add the region * @param tableDir {@link Path} to where the table is being stored * @param regionInfo {@link HRegionInfo} for region to be added + * @param humongousTable * @throws IOException if the region creation fails due to a FileSystem exception. */ - public static HRegionFileSystem createRegionOnFileSystem(final Configuration conf, - final FileSystem fs, final Path tableDir, final HRegionInfo regionInfo) throws IOException { + public static HRegionFileSystem createRegionOnFileSystem( + final Configuration conf, final FileSystem fs, final Path tableDir, + final HRegionInfo regionInfo) + throws IOException { HRegionFileSystem regionFs = new HRegionFileSystem(conf, fs, tableDir, regionInfo); Path regionDir = regionFs.getRegionDir(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 33d4e1e..66a1c93 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -89,6 +89,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ChecksumType; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.hadoop.ipc.RemoteException; @@ -463,19 +464,8 @@ public class HStore implements Store { @Deprecated public static Path getStoreHomedir(final Path tabledir, final HRegionInfo hri, final byte[] family) { - return getStoreHomedir(tabledir, hri.getEncodedName(), family); - } - - /** - * @param tabledir {@link Path} to where the table is being stored - * @param encodedName Encoded region name. - * @param family {@link HColumnDescriptor} describing the column family - * @return Path to family/Store home directory. - */ - @Deprecated - public static Path getStoreHomedir(final Path tabledir, - final String encodedName, final byte[] family) { - return new Path(tabledir, new Path(encodedName, Bytes.toString(family))); + return new Path(HRegion.getRegionDir(FSUtils.getRootDir(tabledir), hri), + Bytes.toString(family)); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java index cd05425..1f5856a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransactionImpl.java @@ -475,7 +475,7 @@ public class RegionMergeTransactionImpl implements RegionMergeTransaction { } // Merged region is sorted between two merging regions in META - HRegionInfo mergedRegionInfo = new HRegionInfo(a.getTable(), startKey, + HRegionInfo mergedRegionInfo = new HRegionInfo(a.getTable(), a.isHumongous(), startKey, endKey, false, rid); return mergedRegionInfo; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java index 8695c77..f64859a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java @@ -173,8 +173,8 @@ public class SplitTransactionImpl implements SplitTransaction { return false; } long rid = getDaughterRegionIdTimestamp(hri); - this.hri_a = new HRegionInfo(hri.getTable(), startKey, this.splitrow, false, rid); - this.hri_b = new HRegionInfo(hri.getTable(), this.splitrow, endKey, false, rid); + this.hri_a = new HRegionInfo(hri.getTable(), hri.isHumongous(), startKey, this.splitrow, false, rid); + this.hri_b = new HRegionInfo(hri.getTable(), hri.isHumongous(), this.splitrow, endKey, false, rid); transition(SplitTransactionPhase.PREPARED); @@ -363,7 +363,8 @@ public class SplitTransactionImpl implements SplitTransaction { this.parent.getRegionFileSystem().getSplitsDir(this.hri_a)); HRegion a = this.parent.createDaughterRegionFromSplits(this.hri_a); assertReferenceFileCount(expectedReferences.getFirst(), - new Path(this.parent.getRegionFileSystem().getTableDir(), this.hri_a.getEncodedName())); + HRegion.getRegionDir(this.parent.getRegionFileSystem().getTableDir(), + this.hri_a.getEncodedName(), this.hri_a.isHumongous())); // Ditto @@ -373,7 +374,8 @@ public class SplitTransactionImpl implements SplitTransaction { this.parent.getRegionFileSystem().getSplitsDir(this.hri_b)); HRegion b = this.parent.createDaughterRegionFromSplits(this.hri_b); assertReferenceFileCount(expectedReferences.getSecond(), - new Path(this.parent.getRegionFileSystem().getTableDir(), this.hri_b.getEncodedName())); + HRegion.getRegionDir(this.parent.getRegionFileSystem().getTableDir(), + this.hri_b.getEncodedName(), this.hri_b.isHumongous())); return new PairOfSameType(a, b); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java index 6516a3e..e09256f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java @@ -391,6 +391,9 @@ public class StoreFileInfo { /* * Return path to the file referred to by a Reference. Presumes a directory * hierarchy of ${hbase.rootdir}/data/${namespace}/tablename/regionname/familyname. + * Unless the table is a humongous table in which case the hierarchy is + * ${hbase.rootdir}/data/${namespace}/tablename/bucket/regionname/familyname. + * * @param p Path to a Reference file. * @return Calculated path to parent region file. * @throws IllegalArgumentException when path regex fails to match. @@ -406,7 +409,8 @@ public class StoreFileInfo { // Other region name is suffix on the passed Reference file name String otherRegion = m.group(2); // Tabledir is up two directories from where Reference was written. - Path tableDir = p.getParent().getParent().getParent(); + Path regionDir = p.getParent().getParent(); + Path tableDir = regionDir.getParent(); String nameStrippedOfSuffix = m.group(1); if (LOG.isDebugEnabled()) { LOG.debug("reference '" + p + "' to region=" + otherRegion @@ -415,8 +419,14 @@ public class StoreFileInfo { // Build up new path with the referenced region in place of our current // region in the reference path. Also strip regionname suffix from name. - return new Path(new Path(new Path(tableDir, otherRegion), - p.getParent().getName()), nameStrippedOfSuffix); + if (HRegion.isHumongousRegionDir(regionDir)) { + tableDir = tableDir.getParent(); + regionDir = HRegion.getHumongousRegionDir(tableDir, otherRegion); + return new Path(new Path(regionDir, p.getParent().getName()), nameStrippedOfSuffix); + } else { + return new Path(new Path(HRegion.getNormalRegionDir(tableDir, otherRegion), p.getParent() + .getName()), nameStrippedOfSuffix); + } } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java index 441dbbf..1484408 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java @@ -399,7 +399,7 @@ public class RestoreSnapshotHelper { Map> snapshotFiles = getRegionHFileReferences(regionManifest); - Path regionDir = new Path(tableDir, regionInfo.getEncodedName()); + Path regionDir = HRegion.getRegionDir(rootDir, regionInfo); String tableName = tableDesc.getTableName().getNameAsString(); // Restore families present in the table @@ -531,7 +531,7 @@ public class RestoreSnapshotHelper { */ private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException { - final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName()); + final Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo()); final String tableName = tableDesc.getTableName().getNameAsString(); for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) { Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8()); @@ -558,7 +558,7 @@ public class RestoreSnapshotHelper { final SnapshotRegionManifest.StoreFile storeFile) throws IOException { String hfileName = storeFile.getName(); if (HFileLink.isHFileLink(hfileName)) { - HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName); + HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, regionInfo); } else if (StoreFileInfo.isReference(hfileName)) { restoreReferenceFile(familyDir, regionInfo, storeFile); } else { @@ -619,7 +619,9 @@ public class RestoreSnapshotHelper { if (linkPath != null) { in = HFileLink.buildFromHFileLinkPattern(conf, linkPath).open(fs); } else { - linkPath = new Path(new Path(HRegion.getRegionDir(snapshotManifest.getSnapshotDir(), + // See caveat in HFileLink.java about snapshots on 'humongous' tables + // Will need to update below snippet to get snapshots on humongous tables to work + linkPath = new Path(new Path(new Path(snapshotManifest.getSnapshotDir(), regionInfo.getEncodedName()), familyDir.getName()), hfileName); in = fs.open(linkPath); } @@ -651,8 +653,9 @@ public class RestoreSnapshotHelper { */ public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) { HRegionInfo regionInfo = new HRegionInfo(tableDesc.getTableName(), - snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(), - snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId()); + snapshotRegionInfo.isHumongous(), snapshotRegionInfo.getStartKey(), + snapshotRegionInfo.getEndKey(), snapshotRegionInfo.isSplit(), + snapshotRegionInfo.getRegionId()); regionInfo.setOffline(snapshotRegionInfo.isOffline()); return regionInfo; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java index 6d10351..54bbdf5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java @@ -1190,6 +1190,16 @@ public abstract class FSUtils { } /** + * Returns the {@link org.apache.hadoop.fs.Path} object representing the HBase root directory + * + * @param tabledir qualified path of HBase table directory + * @return {@link org.apache.hadoop.fs.Path} for HBase root directory + */ + public static Path getRootDir(Path tabledir) { + return tabledir.getParent().getParent().getParent(); + } + + /** * Returns the {@link org.apache.hadoop.hbase.TableName} object representing * the table directory under * path rootdir @@ -1443,10 +1453,18 @@ public abstract class FSUtils { public static List getRegionDirs(final FileSystem fs, final Path tableDir) throws IOException { // assumes we are in a table dir. FileStatus[] rds = fs.listStatus(tableDir, new RegionDirFilter(fs)); - List regionDirs = new ArrayList(rds.length); - for (FileStatus rdfs: rds) { - Path rdPath = rdfs.getPath(); - regionDirs.add(rdPath); + List regionDirs = new ArrayList(); + for (FileStatus rdfs : rds) { + if (HRegion.isNormalRegionDir(rdfs.getPath())) { + regionDirs.add(rdfs.getPath()); + } else { + // get all region dirs from bucket dir + FileStatus[] bucket_rds = fs.listStatus(rdfs.getPath(), + new RegionDirFilter(fs)); + for (FileStatus bucket_rdfs : bucket_rds) { + regionDirs.add(bucket_rdfs.getPath()); + } + } } return regionDirs; } @@ -1616,12 +1634,11 @@ public abstract class FSUtils { // Inside a table, there are compaction.dir directories to skip. Otherwise, all else // should be regions. PathFilter familyFilter = new FamilyDirFilter(fs); - FileStatus[] regionDirs = fs.listStatus(tableDir, new RegionDirFilter(fs)); - for (FileStatus regionDir : regionDirs) { + List regionDirs = getRegionDirs(fs, tableDir); + for (Path dd : regionDirs) { if (null != errors) { errors.progress(); } - Path dd = regionDir.getPath(); // else its a region name, now look in region for families FileStatus[] familyDirs = fs.listStatus(dd, familyFilter); for (FileStatus familyDir : familyDirs) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index cc87f64..13cfb82 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -759,7 +759,7 @@ public class HBaseFsck extends Configured implements Closeable { currentRegionBoundariesInformation.regionName = regionInfo.getRegionName(); // For each region, get the start and stop key from the META and compare them to the // same information from the Stores. - Path path = new Path(tableDir, regionInfo.getEncodedName()); + Path path = HRegion.getRegionDir(hbaseRoot, regionInfo); FileSystem fs = path.getFileSystem(getConf()); FileStatus[] files = fs.listStatus(path); // For all the column families in this region... @@ -922,7 +922,7 @@ public class HBaseFsck extends Configured implements Closeable { Bytes.toString(orphanRegionRange.getSecond()) + ")"); // create new region on hdfs. move data into place. - HRegionInfo hri = new HRegionInfo(template.getTableName(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond()); + HRegionInfo hri = new HRegionInfo(template.getTableName(), template.isHumongous(), orphanRegionRange.getFirst(), orphanRegionRange.getSecond()); LOG.info("Creating new region : " + hri); HRegion region = HBaseFsckRepair.createHDFSRegionDir(getConf(), hri, template); Path target = region.getRegionFileSystem().getRegionDir(); @@ -2788,7 +2788,7 @@ public class HBaseFsck extends Configured implements Closeable { HTableDescriptor htd = getTableInfo().getHTD(); // from special EMPTY_START_ROW to next region's startKey HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), - HConstants.EMPTY_START_ROW, next.getStartKey()); + htd.isHumongous(), HConstants.EMPTY_START_ROW, next.getStartKey()); // TODO test HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); @@ -2804,8 +2804,8 @@ public class HBaseFsck extends Configured implements Closeable { + "region and regioninfo in HDFS to plug the hole.", getTableInfo()); HTableDescriptor htd = getTableInfo().getHTD(); // from curEndKey to EMPTY_START_ROW - HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), curEndKey, - HConstants.EMPTY_START_ROW); + HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), + htd.isHumongous(), curEndKey, HConstants.EMPTY_START_ROW); HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); LOG.info("Table region end key was not empty. Created new empty region: " + newRegion @@ -2827,7 +2827,7 @@ public class HBaseFsck extends Configured implements Closeable { + ". Creating a new regioninfo and region " + "dir in hdfs to plug the hole."); HTableDescriptor htd = getTableInfo().getHTD(); - HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), holeStartKey, holeStopKey); + HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), htd.isHumongous(), holeStartKey, holeStopKey); HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region); fixes++; @@ -2915,8 +2915,8 @@ public class HBaseFsck extends Configured implements Closeable { // create new empty container region. HTableDescriptor htd = getTableInfo().getHTD(); // from start key to end Key - HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), range.getFirst(), - range.getSecond()); + HRegionInfo newRegion = new HRegionInfo(htd.getTableName(), + htd.isHumongous(), range.getFirst(), range.getSecond()); HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); LOG.info("[" + thread + "] Created new empty container region: " + newRegion + " to contain regions: " + Joiner.on(",").join(overlap)); @@ -3602,6 +3602,9 @@ public class HBaseFsck extends Configured implements Closeable { // we are only guaranteed to have a path and not an HRI for hdfsEntry, // so we get the name from the Path Path tableDir = this.hdfsEntry.hdfsRegionDir.getParent(); + if (HRegion.isHumongousRegionDir(this.hdfsEntry.hdfsRegionDir)) { + tableDir = tableDir.getParent(); + } return FSUtils.getTableName(tableDir); } else { // return the info from the first online/deployed hri @@ -4016,6 +4019,23 @@ public class HBaseFsck extends Configured implements Closeable { } } + private static List getRegionDirs(final FileSystem fs, final Path tableDir) throws IOException { + FileStatus[] rds = fs.listStatus(tableDir); + List regionStatus = new ArrayList(); + for (FileStatus rdfs : rds) { + if (HRegion.isNormalRegionDir(rdfs.getPath())) { + regionStatus.add(rdfs); + } else { + // get all region dirs from bucket dir + FileStatus[] bucket_rds = fs.listStatus(rdfs.getPath()); + for (FileStatus bucket_rdfs : bucket_rds) { + regionStatus.add(bucket_rdfs); + } + } + } + return regionStatus; + } + /** * Contact hdfs and get all information about specified table directory into * regioninfo list. @@ -4038,7 +4058,7 @@ public class HBaseFsck extends Configured implements Closeable { public synchronized Void call() throws IOException { try { // level 2: //* - FileStatus[] regionDirs = fs.listStatus(tableDir.getPath()); + List regionDirs = getRegionDirs(fs, tableDir.getPath()); for (FileStatus regionDir : regionDirs) { errors.progress(); String encodedName = regionDir.getPath().getName(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java index 937e9b2..89db54d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HFileArchiveUtil.java @@ -37,22 +37,6 @@ public class HFileArchiveUtil { /** * Get the directory to archive a store directory - * @param conf {@link Configuration} to read for the archive directory name - * @param tableName table name under which the store currently lives - * @param regionName region encoded name under which the store currently lives - * @param familyName name of the family in the store - * @return {@link Path} to the directory to archive the given store or - * null if it should not be archived - */ - public static Path getStoreArchivePath(final Configuration conf, - final TableName tableName, - final String regionName, final String familyName) throws IOException { - Path tableArchiveDir = getTableArchivePath(conf, tableName); - return HStore.getStoreHomedir(tableArchiveDir, regionName, Bytes.toBytes(familyName)); - } - - /** - * Get the directory to archive a store directory * @param conf {@link Configuration} to read for the archive directory name. * @param region parent region information under which the store currently lives * @param tabledir directory for the table under which the store currently lives @@ -86,22 +70,13 @@ public class HFileArchiveUtil { // then add on the region path under the archive String encodedRegionName = regiondir.getName(); - return HRegion.getRegionDir(archiveDir, encodedRegionName); - } + String parentName = regiondir.getParent().getName(); - /** - * Get the archive directory for a given region under the specified table - * @param rootDir {@link Path} to the root directory where hbase files are stored (for building - * the archive path) - * @param tableName name of the table to archive. Cannot be null. - * @return {@link Path} to the directory to archive the given region, or null if it - * should not be archived - */ - public static Path getRegionArchiveDir(Path rootDir, - TableName tableName, String encodedRegionName) { - // get the archive directory for a table - Path archiveDir = getTableArchivePath(rootDir, tableName); - return HRegion.getRegionDir(archiveDir, encodedRegionName); + if(HRegion.isHumongousRegionDir(regiondir)) { + return new Path(archiveDir, new Path(parentName, encodedRegionName)); + } else { + return new Path(archiveDir, encodedRegionName); + } } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java index 347cad5..d227091 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ModifyRegionUtils.java @@ -73,7 +73,8 @@ public abstract class ModifyRegionUtils { HRegionInfo[] hRegionInfos = null; if (splitKeys == null || splitKeys.length == 0) { hRegionInfos = new HRegionInfo[]{ - new HRegionInfo(hTableDescriptor.getTableName(), null, null, false, regionId) + new HRegionInfo(hTableDescriptor.getTableName(), hTableDescriptor.isHumongous(), + null, null, false, regionId) }; } else { int numRegions = splitKeys.length + 1; @@ -83,8 +84,8 @@ public abstract class ModifyRegionUtils { for (int i = 0; i < numRegions; i++) { endKey = (i == splitKeys.length) ? null : splitKeys[i]; hRegionInfos[i] = - new HRegionInfo(hTableDescriptor.getTableName(), startKey, endKey, - false, regionId); + new HRegionInfo(hTableDescriptor.getTableName(), hTableDescriptor.isHumongous(), + startKey, endKey, false, regionId); startKey = endKey; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java index 5c61afb..b3a7abb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ServerRegionReplicaUtil.java @@ -132,6 +132,7 @@ public class ServerRegionReplicaUtil extends RegionReplicaUtil { return new StoreFileInfo(conf, fs, link.getFileStatus(fs), reference); } + // TODO: This can throw an exception for 'humongous' tables return new StoreFileInfo(conf, fs, link.getFileStatus(fs), link); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java index 29ab24e..c6a6504 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/hbck/HFileCorruptionChecker.java @@ -217,8 +217,8 @@ public class HFileCorruptionChecker { * @throws IOException */ void checkTableDir(Path tableDir) throws IOException { - FileStatus[] rds = fs.listStatus(tableDir, new RegionDirFilter(fs)); - if (rds.length == 0 && !fs.exists(tableDir)) { + List rds = FSUtils.getRegionDirs(fs, tableDir); + if (rds.size() == 0 && !fs.exists(tableDir)) { // interestingly listStatus does not throw an exception if the path does not exist. LOG.warn("Table Directory " + tableDir + " does not exist. Likely due to concurrent delete. Skipping."); @@ -230,8 +230,7 @@ public class HFileCorruptionChecker { List rdcs = new ArrayList(); List> rdFutures; - for (FileStatus rdFs : rds) { - Path rdDir = rdFs.getPath(); + for (Path rdDir : rds) { RegionDirChecker work = new RegionDirChecker(rdDir); rdcs.add(work); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java index 9a26a24..053dd8d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java @@ -512,21 +512,26 @@ public class WALSplitter { * @return Path to file into which to dump split log edits. * @throws IOException */ - @SuppressWarnings("deprecation") static Path getRegionSplitEditsPath(final FileSystem fs, final Entry logEntry, final Path rootDir, boolean isCreate) throws IOException { Path tableDir = FSUtils.getTableDir(rootDir, logEntry.getKey().getTablename()); String encodedRegionName = Bytes.toString(logEntry.getKey().getEncodedRegionName()); - Path regiondir = HRegion.getRegionDir(tableDir, encodedRegionName); - Path dir = getRegionDirRecoveredEditsDir(regiondir); - + Path regiondir = HRegion.getNormalRegionDir(tableDir, encodedRegionName); if (!fs.exists(regiondir)) { - LOG.info("This region's directory doesn't exist: " - + regiondir.toString() + ". It is very likely that it was" + - " already split so it's safe to discard those edits."); - return null; + // If the region dir doesn't exist, check its humongous dir + LOG.info("This region's directory doesn't exist: " + regiondir.toString() + + ". It might be a humongous table region." + + " Try to get its humongous region path."); + regiondir = HRegion.getHumongousRegionDir(tableDir, encodedRegionName); + if (!fs.exists(regiondir)) { + LOG.info("This region's humongous directory doesn't exist either." + + " It is very likely that it was" + + " already split so it's safe to discard those edits."); + return null; + } } + Path dir = getRegionDirRecoveredEditsDir(regiondir); if (fs.exists(dir) && fs.isFile(dir)) { Path tmp = new Path("/tmp"); if (!fs.exists(tmp)) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index a6d4d77..5cce3e5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -1313,7 +1313,25 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { return createTable(tableName, new byte[][] { family }, splitKeys); } + + /** + * Create a table with multiple regions. + * @param desc + * @param family + * @param numRegions + * @return An HTable instance for the created table. + * @throws IOException + */ + public HTable createMultiRegionTable(HTableDescriptor desc, byte[] family, int numRegions) + throws IOException { + if (numRegions < 3) throw new IOException("Must create at least 3 regions"); + byte[] startKey = Bytes.toBytes("aaaaa"); + byte[] endKey = Bytes.toBytes("zzzzz"); + byte[][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3); + return createTable(desc, new byte[][] { family }, splitKeys, + new Configuration(getConfiguration())); + } /** * Create a table. @@ -2403,7 +2421,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { // add custom ones for (int i = 0; i < startKeys.length; i++) { int j = (i + 1) % startKeys.length; - HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKeys[i], + HRegionInfo hri = new HRegionInfo(htd.getTableName(), htd.isHumongous(), startKeys[i], startKeys[j]); MetaTableAccessor.addRegionToMeta(meta, hri); newRegions.add(hri); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHumongousTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHumongousTable.java new file mode 100644 index 0000000..3b863ea --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHumongousTable.java @@ -0,0 +1,177 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSTableDescriptors; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(MediumTests.class) +public class TestHumongousTable { + protected static final Log LOG = LogFactory.getLog(TestHumongousTable.class); + protected final static int NUM_SLAVES_BASE = 4; + private static HBaseTestingUtility TEST_UTIL; + private static Configuration conf; + protected static HBaseAdmin admin; + protected static FileSystem fs; + + @BeforeClass + public static void setUp() throws Exception { + TEST_UTIL = new HBaseTestingUtility(); + conf = TEST_UTIL.getConfiguration(); + TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE); + admin = TEST_UTIL.getHBaseAdmin(); + LOG.info("Done initializing cluster"); + } + + @AfterClass + public static void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void beforeMethod() throws IOException { + for (HTableDescriptor desc : admin.listTables(".*")) { + admin.disableTable(desc.getTableName()); + admin.deleteTable(desc.getTableName()); + } + } + + @Test(timeout = 60000) + public void testCreateHumongousTable() throws IOException, InterruptedException { + // create a humongous table with splits + String tableNameStr = "testCreateHumongousTable"; + TableName tableName = TableName.valueOf(tableNameStr); + String familyName = "col"; + TableName testTable = TableName.valueOf(tableNameStr); + HTableDescriptor desc = new HTableDescriptor(testTable); + HColumnDescriptor family = new HColumnDescriptor(familyName); + desc.addFamily(family); + desc.setHumongous(true); + admin.createTable(desc, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); + + // check tableDir and table descriptor + Path tableDir = FSUtils.getTableDir(TEST_UTIL.getDefaultRootDirPath(), testTable); + fs = TEST_UTIL.getTestFileSystem(); + assertTrue(fs.exists(tableDir)); + TableDescriptor table_desc = FSTableDescriptors.getTableDescriptorFromFs( + FileSystem.get(conf), tableDir); + assertTrue(table_desc.getHTableDescriptor().isHumongous()); + + // load table with rows and flush stores + Connection connection = TEST_UTIL.getConnection(); + Table table = connection.getTable(testTable); + int rowCount = TEST_UTIL.loadTable(table, Bytes.toBytes(familyName)); + admin.flush(tableName); + assertEquals(TEST_UTIL.countRows(table), rowCount); + + verifyColumnFamilies(testTable, familyName); + + // test alteration of humongous table too + String familyName2 = "col2"; + HColumnDescriptor family2 = new HColumnDescriptor(familyName2); + admin.addColumnFamily(tableName, family2); + + // Wait for async add column to finish + Thread.sleep(5000); + + TEST_UTIL.loadTable(table, Bytes.toBytes(familyName2)); + admin.flush(tableName); + + verifyColumnFamilies(testTable, familyName, familyName2); + + // drop the test table + admin.disableTable(testTable); + assertTrue(admin.isTableDisabled(testTable)); + admin.deleteTable(testTable); + assertEquals(admin.getTableRegions(testTable), null); + assertFalse(fs.exists(tableDir)); + } + + private void verifyColumnFamilies(TableName testTable, String... colFamNames) throws IOException { + Path tableDir = FSUtils.getTableDir(TEST_UTIL.getDefaultRootDirPath(), testTable); + + List tableRegions = admin.getTableRegions(testTable); + + // check region dirs and files on fs + for (HRegionInfo hri : tableRegions) { + assertTrue(hri.isHumongous()); + // check region dir structure + Path humongousRegionDir = HRegion.getRegionDir( + TEST_UTIL.getDefaultRootDirPath(), hri); + Path normalRegionDir = HRegion.getNormalRegionDir(tableDir, + hri.getEncodedName()); + assertTrue(fs.exists(humongousRegionDir)); + assertFalse(fs.exists(normalRegionDir)); + String bucket = hri.getEncodedName().substring( + HRegionInfo.MD5_HEX_LENGTH + - HRegionFileSystem.HUMONGOUS_DIR_NAME_SIZE); + assertEquals(humongousRegionDir.getParent().getName(), bucket); + + FileStatus[] statList = fs.listStatus(humongousRegionDir); + Set contents = new HashSet(); + + LOG.debug("Contents of humongous region dir: " + contents); + + for (FileStatus stat : statList) { + contents.add(stat.getPath().getName()); + } + + assertTrue(contents.contains(HRegionFileSystem.REGION_INFO_FILE)); + assertTrue(contents.contains(HConstants.HBASE_TEMP_DIRECTORY)); + assertTrue(contents.contains(HConstants.RECOVERED_EDITS_DIR)); + + for (String colFam : colFamNames) { + assertTrue(contents.contains(colFam)); + + // familyDir has one store file + Path famPath = new Path(humongousRegionDir, colFam); + assertEquals(1, fs.listStatus(famPath).length); + } + + assertEquals("Contents: " + contents + " and fam names: " + Arrays.toString(colFamNames), + 3 + colFamNames.length, contents.size()); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java index e30d719..7445892 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java @@ -35,7 +35,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; @@ -66,8 +68,9 @@ import org.junit.experimental.categories.Category; public class TestHFileArchiving { private static final Log LOG = LogFactory.getLog(TestHFileArchiving.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - private static final byte[] TEST_FAM = Bytes.toBytes("fam"); + static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + static final byte[] TEST_FAM = Bytes.toBytes("fam"); + static final byte[] TEST_FAM_2 = Bytes.toBytes("fam2"); /** * Setup the config for the cluster @@ -117,7 +120,7 @@ public class TestHFileArchiving { public void testRemovesRegionDirOnArchive() throws Exception { TableName TABLE_NAME = TableName.valueOf("testRemovesRegionDirOnArchive"); - UTIL.createTable(TABLE_NAME, TEST_FAM); + createTable(TABLE_NAME); final Admin admin = UTIL.getHBaseAdmin(); @@ -136,7 +139,7 @@ public class TestHFileArchiving { FileSystem fs = UTIL.getTestFileSystem(); // now attempt to depose the region - Path rootDir = region.getRegionFileSystem().getTableDir().getParent(); + Path rootDir = FSUtils.getRootDir(region.getRegionFileSystem().getTableDir()); Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo()); HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo()); @@ -177,7 +180,7 @@ public class TestHFileArchiving { public void testDeleteRegionWithNoStoreFiles() throws Exception { TableName TABLE_NAME = TableName.valueOf("testDeleteRegionWithNoStoreFiles"); - UTIL.createTable(TABLE_NAME, TEST_FAM); + createTable(TABLE_NAME); // get the current store files for the region List servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); @@ -226,7 +229,7 @@ public class TestHFileArchiving { public void testArchiveOnTableDelete() throws Exception { TableName TABLE_NAME = TableName.valueOf("testArchiveOnTableDelete"); - UTIL.createTable(TABLE_NAME, TEST_FAM); + createTable(TABLE_NAME); List servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); // make sure we only have 1 region serving this table @@ -306,7 +309,7 @@ public class TestHFileArchiving { public void testArchiveOnTableFamilyDelete() throws Exception { TableName TABLE_NAME = TableName.valueOf("testArchiveOnTableFamilyDelete"); - UTIL.createTable(TABLE_NAME, new byte[][] {TEST_FAM, Bytes.toBytes("fam2")}); + createTwoFamilyTable(TABLE_NAME); List servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); // make sure we only have 1 region serving this table @@ -451,4 +454,21 @@ public class TestHFileArchiving { } return fileNames; } + + void createTable(TableName tn) throws Exception { + HTableDescriptor desc = makeDescriptor(tn); + desc.addFamily(new HColumnDescriptor(TEST_FAM)); + UTIL.createTable(desc, null); + } + + void createTwoFamilyTable(TableName tn) throws Exception { + HTableDescriptor desc = makeDescriptor(tn); + desc.addFamily(new HColumnDescriptor(TEST_FAM)); + desc.addFamily(new HColumnDescriptor(TEST_FAM_2)); + UTIL.createTable(desc, null); + } + + HTableDescriptor makeDescriptor(TableName tn) { + return new HTableDescriptor(tn); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHumongousHFileArchiving.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHumongousHFileArchiving.java new file mode 100644 index 0000000..4d156ab --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHumongousHFileArchiving.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.backup; + +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.junit.experimental.categories.Category; + +@Category({MediumTests.class, MiscTests.class}) +public class TestHumongousHFileArchiving extends TestHFileArchiving { + + @Override + HTableDescriptor makeDescriptor(TableName tn) { + HTableDescriptor desc = super.makeDescriptor(tn); + desc.setHumongous(true); + return desc; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHFileLink.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHFileLink.java index f2b26c1..5492260 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHFileLink.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHFileLink.java @@ -96,7 +96,7 @@ public class TestHFileLink { for(TableName refTable : refTables) { Path refTableDir = FSUtils.getTableDir(archiveDir, refTable); - Path refRegionDir = HRegion.getRegionDir(refTableDir, encodedRegion); + Path refRegionDir = HRegion.getNormalRegionDir(refTableDir, encodedRegion); Path refDir = new Path(refRegionDir, cf); Path refLinkDir = new Path(refDir, linkDir); String refStoreFileName = refTable.getNameAsString().replace( @@ -107,7 +107,7 @@ public class TestHFileLink { for( TableName tableName : tableNames) { Path tableDir = FSUtils.getTableDir(rootDir, tableName); - Path regionDir = HRegion.getRegionDir(tableDir, encodedRegion); + Path regionDir = HRegion.getNormalRegionDir(tableDir, encodedRegion); Path cfDir = new Path(regionDir, cf); //Verify back reference creation diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java index bc437fc..2c45c21 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -120,7 +120,7 @@ import org.junit.experimental.categories.Category; @Category({MasterTests.class, LargeTests.class}) @SuppressWarnings("deprecation") public class TestDistributedLogSplitting { - private static final Log LOG = LogFactory.getLog(TestSplitLogManager.class); + static final Log LOG = LogFactory.getLog(TestSplitLogManager.class); static { // Uncomment the following line if more verbosity is needed for // debugging (see HBASE-12285 for details). @@ -224,7 +224,6 @@ public class TestDistributedLogSplitting { installTable(new ZooKeeperWatcher(conf, "table-creation", null), "table", "family", 40); - TableName table = TableName.valueOf("table"); List regions = null; HRegionServer hrs = null; for (int i = 0; i < NUM_RS; i++) { @@ -259,9 +258,8 @@ public class TestDistributedLogSplitting { int count = 0; for (HRegionInfo hri : regions) { - Path tdir = FSUtils.getTableDir(rootdir, table); Path editsdir = - WALSplitter.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, hri.getEncodedName())); + WALSplitter.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(rootdir, hri)); LOG.debug("checking edits dir " + editsdir); FileStatus[] files = fs.listStatus(editsdir, new PathFilter() { @Override @@ -847,10 +845,9 @@ public class TestDistributedLogSplitting { int count = 0; FileSystem fs = master.getMasterFileSystem().getFileSystem(); Path rootdir = FSUtils.getRootDir(conf); - Path tdir = FSUtils.getTableDir(rootdir, TableName.valueOf("disableTable")); for (HRegionInfo hri : regions) { Path editsdir = - WALSplitter.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, hri.getEncodedName())); + WALSplitter.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(rootdir, hri)); LOG.debug("checking edits dir " + editsdir); if(!fs.exists(editsdir)) continue; FileStatus[] files = fs.listStatus(editsdir, new PathFilter() { @@ -879,7 +876,7 @@ public class TestDistributedLogSplitting { // clean up for (HRegionInfo hri : regions) { Path editsdir = - WALSplitter.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, hri.getEncodedName())); + WALSplitter.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(rootdir, hri)); fs.delete(editsdir, true); } disablingHT.close(); @@ -1435,7 +1432,9 @@ public class TestDistributedLogSplitting { TableName table = TableName.valueOf(tname); byte [] family = Bytes.toBytes(fname); LOG.info("Creating table with " + nrs + " regions"); - HTable ht = TEST_UTIL.createMultiRegionTable(table, family, nrs); + HTableDescriptor desc = new HTableDescriptor(table); + desc.setHumongous(shouldMakeHumongousTable()); + HTable ht = TEST_UTIL.createMultiRegionTable(desc, family, nrs); int numRegions = -1; try (RegionLocator r = ht.getRegionLocator()) { numRegions = r.getStartKeys().length; @@ -1465,6 +1464,10 @@ public class TestDistributedLogSplitting { assertEquals(numRegions + 2 + existingRegions, regions.size()); return ht; } + + boolean shouldMakeHumongousTable() { + return false; + } void populateDataInTable(int nrows, String fname) throws Exception { byte [] family = Bytes.toBytes(fname); @@ -1612,7 +1615,7 @@ public class TestDistributedLogSplitting { return count; } - private void blockUntilNoRIT(ZooKeeperWatcher zkw, HMaster master) throws Exception { + void blockUntilNoRIT(ZooKeeperWatcher zkw, HMaster master) throws Exception { TEST_UTIL.waitUntilNoRegionsInTransition(60000); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestHumongousTableDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestHumongousTableDistributedLogSplitting.java new file mode 100644 index 0000000..eb703f3 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestHumongousTableDistributedLogSplitting.java @@ -0,0 +1,32 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.junit.experimental.categories.Category; + +@Category({MasterTests.class, LargeTests.class}) +public class TestHumongousTableDistributedLogSplitting extends TestDistributedLogSplitting { + + @Override + boolean shouldMakeHumongousTable() { + return true; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java index 66874e6..2e4ade1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestHFileLinkCleaner.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; @@ -71,10 +72,8 @@ public class TestHFileLinkCleaner { HRegionInfo hriLink = new HRegionInfo(tableLinkName); Path archiveDir = HFileArchiveUtil.getArchivePath(conf); - Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, - tableName, hri.getEncodedName(), familyName); - Path archiveLinkStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, - tableLinkName, hriLink.getEncodedName(), familyName); + Path tableDir = FSUtils.getTableDir(rootDir, tableName); + Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, hri, tableDir, Bytes.toBytes(familyName)); // Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf); Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactHumongousRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactHumongousRegion.java new file mode 100644 index 0000000..6641439 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactHumongousRegion.java @@ -0,0 +1,37 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.junit.experimental.categories.Category; + +@Category({RegionServerTests.class, MediumTests.class}) +public class TestCompactHumongousRegion extends TestCompaction { + + @Override + public void setUp() throws Exception { + this.htd = UTIL.createTableDescriptor(name.getMethodName()); + this.htd.setHumongous(true); + HRegionInfo hri = new HRegionInfo(this.htd.getTableName(), + this.htd.isHumongous(), null, null); + this.r = UTIL.createLocalHRegion(hri, this.htd); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java index fcc9fc3..6464db4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java @@ -86,11 +86,11 @@ import org.mockito.stubbing.Answer; public class TestCompaction { @Rule public TestName name = new TestName(); private static final Log LOG = LogFactory.getLog(TestCompaction.class.getName()); - private static final HBaseTestingUtility UTIL = HBaseTestingUtility.createLocalHTU(); + static final HBaseTestingUtility UTIL = HBaseTestingUtility.createLocalHTU(); protected Configuration conf = UTIL.getConfiguration(); - private HRegion r = null; - private HTableDescriptor htd = null; + HRegion r = null; + HTableDescriptor htd = null; private static final byte [] COLUMN_FAMILY = fam1; private final byte [] STARTROW = Bytes.toBytes(START_KEY); private static final byte [] COLUMN_FAMILY_TEXT = COLUMN_FAMILY; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFlushRegionEntry.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFlushRegionEntry.java index abd8c59..9c1569d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFlushRegionEntry.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFlushRegionEntry.java @@ -42,7 +42,7 @@ public class TestFlushRegionEntry { @Test public void testFlushRegionEntryEquality() { - HRegionInfo hri = new HRegionInfo(1, TableName.valueOf("TestTable"), 0); + HRegionInfo hri = new HRegionInfo(1, TableName.valueOf("TestTable"), 0, false); HRegion r = mock(HRegion.class); doReturn(hri).when(r).getRegionInfo(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHumongousRegionMerge.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHumongousRegionMerge.java new file mode 100644 index 0000000..b123183 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHumongousRegionMerge.java @@ -0,0 +1,32 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.experimental.categories.Category; + +@Category({RegionServerTests.class, SmallTests.class}) +public class TestHumongousRegionMerge extends TestRegionMergeTransaction { + + @Override + boolean shouldTestHumongousTable() { + return true; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHumongousStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHumongousStoreFile.java new file mode 100644 index 0000000..8324544 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHumongousStoreFile.java @@ -0,0 +1,44 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.experimental.categories.Category; + +@Category({RegionServerTests.class, SmallTests.class}) +public class TestHumongousStoreFile extends TestStoreFile { + + @Override + HRegionInfo createHRegionInfo(final TableName tableName, final byte[] startKey, + final byte[] endKey) { + return new HRegionInfo(tableName, true, startKey, endKey); + } + + @Override + Path getTestRegionPath() { + String regionName = "d9ffc3a5cd016ae58e23d7a6cb937949"; + String bucket = regionName.substring(HRegionInfo.MD5_HEX_LENGTH + - HRegionFileSystem.HUMONGOUS_DIR_NAME_SIZE); + return new Path(bucket, regionName); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java index b2115b3..0874cda 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java @@ -64,7 +64,7 @@ import com.google.common.collect.ImmutableList; */ @Category({RegionServerTests.class, SmallTests.class}) public class TestRegionMergeTransaction { - private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private final Path testdir = TEST_UTIL.getDataTestDir(this.getClass() .getName()); private HRegion region_a; @@ -77,7 +77,7 @@ public class TestRegionMergeTransaction { private static final byte[] STARTROW_B = new byte[] { 'g', 'g', 'g' }; private static final byte[] STARTROW_C = new byte[] { 'w', 'w', 'w' }; private static final byte[] ENDROW = new byte[] { '{', '{', '{' }; - private static final byte[] CF = HConstants.CATALOG_FAMILY; + static final byte[] CF = HConstants.CATALOG_FAMILY; @Before public void setup() throws IOException { @@ -348,10 +348,9 @@ public class TestRegionMergeTransaction { // Make sure that merged region is still in the filesystem, that // they have not been removed; this is supposed to be the case if we go // past point of no return. - Path tableDir = this.region_a.getRegionFileSystem().getRegionDir() - .getParent(); - Path mergedRegionDir = new Path(tableDir, mt.getMergedRegionInfo() - .getEncodedName()); + Path tableDir = this.region_a.getRegionFileSystem().getTableDir(); + Path mergedRegionDir = HRegion.getRegionDir(FSUtils.getRootDir(tableDir), + mt.getMergedRegionInfo()); assertTrue(TEST_UTIL.getTestFileSystem().exists(mergedRegionDir)); } @@ -404,13 +403,14 @@ public class TestRegionMergeTransaction { private class MockedFailedMergedRegionOpen extends IOException { } - private HRegion createRegion(final Path testdir, final WALFactory wals, + HRegion createRegion(final Path testdir, final WALFactory wals, final byte[] startrow, final byte[] endrow) throws IOException { // Make a region with start and end keys. HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table")); HColumnDescriptor hcd = new HColumnDescriptor(CF); htd.addFamily(hcd); + htd.setHumongous(shouldTestHumongousTable()); HRegionInfo hri = new HRegionInfo(htd.getTableName(), startrow, endrow); HRegion a = HBaseTestingUtility.createRegionAndWAL(hri, testdir, TEST_UTIL.getConfiguration(), htd); @@ -418,6 +418,10 @@ public class TestRegionMergeTransaction { return HRegion.openHRegion(testdir, hri, htd, wals.getWAL(hri.getEncodedNameAsBytes()), TEST_UTIL.getConfiguration()); } + + boolean shouldTestHumongousTable() { + return false; + } private int countRows(final HRegion r) throws IOException { int rowcount = 0; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitHumongousTableTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitHumongousTableTransactionOnCluster.java new file mode 100644 index 0000000..e8b6b77 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitHumongousTableTransactionOnCluster.java @@ -0,0 +1,32 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.junit.experimental.categories.Category; + +@Category({RegionServerTests.class, LargeTests.class}) +public class TestSplitHumongousTableTransactionOnCluster extends TestSplitTransactionOnCluster { + + @Override + boolean shouldCreateHumongousTable() { + return true; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java index 4f371bd..906a978 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransaction.java @@ -71,17 +71,17 @@ import com.google.common.collect.ImmutableList; */ @Category({RegionServerTests.class, SmallTests.class}) public class TestSplitTransaction { - private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private final Path testdir = TEST_UTIL.getDataTestDir(this.getClass().getName()); private HRegion parent; private WALFactory wals; private FileSystem fs; - private static final byte [] STARTROW = new byte [] {'a', 'a', 'a'}; + static final byte [] STARTROW = new byte [] {'a', 'a', 'a'}; // '{' is next ascii after 'z'. - private static final byte [] ENDROW = new byte [] {'{', '{', '{'}; + static final byte [] ENDROW = new byte [] {'{', '{', '{'}; private static final byte [] GOOD_SPLIT_ROW = new byte [] {'d', 'd', 'd'}; - private static final byte [] CF = HConstants.CATALOG_FAMILY; + static final byte [] CF = HConstants.CATALOG_FAMILY; private static boolean preRollBackCalled = false; private static boolean postRollBackCalled = false; @@ -146,8 +146,10 @@ public class TestSplitTransaction { // they have not been removed; this is supposed to be the case if we go // past point of no return. Path tableDir = this.parent.getRegionFileSystem().getTableDir(); - Path daughterADir = new Path(tableDir, spiedUponSt.getFirstDaughter().getEncodedName()); - Path daughterBDir = new Path(tableDir, spiedUponSt.getSecondDaughter().getEncodedName()); + Path daughterADir = HRegion.getRegionDir(FSUtils.getRootDir(tableDir), + spiedUponSt.getFirstDaughter()); + Path daughterBDir = HRegion.getRegionDir(FSUtils.getRootDir(tableDir), + spiedUponSt.getSecondDaughter()); assertTrue(TEST_UTIL.getTestFileSystem().exists(daughterADir)); assertTrue(TEST_UTIL.getTestFileSystem().exists(daughterBDir)); } @@ -374,6 +376,7 @@ public class TestSplitTransaction { // Make a region with start and end keys. Use 'aaa', to 'AAA'. The load // region utility will add rows between 'aaa' and 'zzz'. HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table")); + htd.setHumongous(shouldTestWithHumongousTable()); HColumnDescriptor hcd = new HColumnDescriptor(CF); htd.addFamily(hcd); HRegionInfo hri = new HRegionInfo(htd.getTableName(), STARTROW, ENDROW); @@ -384,6 +387,10 @@ public class TestSplitTransaction { TEST_UTIL.getConfiguration()); } + boolean shouldTestWithHumongousTable() { + return false; + } + public static class CustomObserver extends BaseRegionObserver{ @Override public void preRollBackSplit( diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionForHumongousTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionForHumongousTable.java new file mode 100644 index 0000000..22e606a --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionForHumongousTable.java @@ -0,0 +1,32 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.experimental.categories.Category; + +@Category({RegionServerTests.class, SmallTests.class}) +public class TestSplitTransactionForHumongousTable extends TestSplitTransaction { + + @Override + boolean shouldTestWithHumongousTable() { + return true; + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java index 86d196e..6202b69 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java @@ -1256,7 +1256,7 @@ public class TestSplitTransactionOnCluster { } } - private List awaitTableRegions(final TableName tableName) throws InterruptedException { + List awaitTableRegions(final TableName tableName) throws InterruptedException { List regions = null; for (int i = 0; i < 100; i++) { regions = cluster.getRegions(tableName); @@ -1266,14 +1266,20 @@ public class TestSplitTransactionOnCluster { return regions; } - private HTable createTableAndWait(TableName tableName, byte[] cf) throws IOException, + HTable createTableAndWait(TableName tableName, byte[] cf) throws IOException, InterruptedException { - HTable t = TESTING_UTIL.createTable(tableName, cf); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.setHumongous(shouldCreateHumongousTable()); + HTable t = TESTING_UTIL.createTable(desc, new byte[][]{cf}, TESTING_UTIL.getConfiguration()); awaitTableRegions(tableName); assertTrue("Table not online: " + tableName, cluster.getRegions(tableName).size() != 0); return t; } + + boolean shouldCreateHumongousTable() { + return false; + } private static class SplittingNodeCreationFailedException extends IOException { private static final long serialVersionUID = 1652404976265623004L; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java index 499e57c..fc197a0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java @@ -98,10 +98,10 @@ public class TestStoreFile extends HBaseTestCase { */ @Test public void testBasicHalfMapFile() throws Exception { - final HRegionInfo hri = - new HRegionInfo(TableName.valueOf("testBasicHalfMapFileTb")); + final HRegionInfo hri = createHRegionInfo(TableName.valueOf("testBasicHalfMapFileTb"), null, null); + Path tableDir = FSUtils.getTableDir(this.testDir, TableName.valueOf("testBasicHalfMapFileTb")); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem( - conf, fs, new Path(testDir, hri.getTable().getNameAsString()), hri); + conf, fs, tableDir, hri); HFileContext meta = new HFileContextBuilder().withBlockSize(2*1024).build(); StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, cacheConf, this.fs) @@ -151,9 +151,10 @@ public class TestStoreFile extends HBaseTestCase { */ @Test public void testReference() throws IOException { - final HRegionInfo hri = new HRegionInfo(TableName.valueOf("testReferenceTb")); + final HRegionInfo hri = createHRegionInfo(TableName.valueOf("testReferenceTb"), null, null); + Path tableDir = FSUtils.getTableDir(this.testDir, TableName.valueOf("testReferenceTb")); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem( - conf, fs, new Path(testDir, hri.getTable().getNameAsString()), hri); + conf, fs, tableDir, hri); HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).build(); // Make a store file and write data to it. @@ -175,7 +176,7 @@ public class TestStoreFile extends HBaseTestCase { kv = KeyValueUtil.createKeyValueFromKey(reader.getLastKey()); byte [] finalRow = kv.getRow(); // Make a reference - HRegionInfo splitHri = new HRegionInfo(hri.getTable(), null, midRow); + HRegionInfo splitHri = createHRegionInfo(hri.getTable(), null, midRow); Path refPath = splitStoreFile(regionFs, splitHri, TEST_FAMILY, hsf, midRow, true); StoreFile refHsf = new StoreFile(this.fs, refPath, conf, cacheConf, BloomType.NONE); @@ -195,7 +196,7 @@ public class TestStoreFile extends HBaseTestCase { @Test public void testHFileLink() throws IOException { - final HRegionInfo hri = new HRegionInfo(TableName.valueOf("testHFileLinkTb")); + final HRegionInfo hri = createHRegionInfo(TableName.valueOf("testHFileLinkTb"), null, null); // force temp data in hbase/target/test-data instead of /tmp/hbase-xxxx/ Configuration testConf = new Configuration(this.conf); FSUtils.setRootDir(testConf, testDir); @@ -211,7 +212,7 @@ public class TestStoreFile extends HBaseTestCase { writeStoreFile(writer); Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath()); - Path dstPath = new Path(regionFs.getTableDir(), new Path("test-region", TEST_FAMILY)); + Path dstPath = new Path(regionFs.getTableDir(), new Path(getTestRegionPath(), TEST_FAMILY)); HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName()); Path linkFilePath = new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName())); @@ -243,7 +244,7 @@ public class TestStoreFile extends HBaseTestCase { FSUtils.setRootDir(testConf, testDir); // adding legal table name chars to verify regex handles it. - HRegionInfo hri = new HRegionInfo(TableName.valueOf("_original-evil-name")); + HRegionInfo hri = createHRegionInfo(TableName.valueOf("_original-evil-name"), null, null); HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem( testConf, fs, FSUtils.getTableDir(testDir, hri.getTable()), hri); @@ -257,7 +258,7 @@ public class TestStoreFile extends HBaseTestCase { Path storeFilePath = regionFs.commitStoreFile(TEST_FAMILY, writer.getPath()); // create link to store file. /clone/region//--
- HRegionInfo hriClone = new HRegionInfo(TableName.valueOf("clone")); + HRegionInfo hriClone = createHRegionInfo(TableName.valueOf("clone"), null, null); HRegionFileSystem cloneRegionFs = HRegionFileSystem.createRegionOnFileSystem( testConf, fs, FSUtils.getTableDir(testDir, hri.getTable()), hriClone); @@ -269,8 +270,8 @@ public class TestStoreFile extends HBaseTestCase { // create splits of the link. // /clone/splitA//, // /clone/splitB// - HRegionInfo splitHriA = new HRegionInfo(hri.getTable(), null, SPLITKEY); - HRegionInfo splitHriB = new HRegionInfo(hri.getTable(), SPLITKEY, null); + HRegionInfo splitHriA = createHRegionInfo(hri.getTable(), null, SPLITKEY); + HRegionInfo splitHriB = createHRegionInfo(hri.getTable(), SPLITKEY, null); StoreFile f = new StoreFile(fs, linkFilePath, testConf, cacheConf, BloomType.NONE); Path pathA = splitStoreFile(cloneRegionFs, splitHriA, TEST_FAMILY, f, SPLITKEY, true); // top Path pathB = splitStoreFile(cloneRegionFs, splitHriB, TEST_FAMILY, f, SPLITKEY, false);// bottom @@ -318,11 +319,11 @@ public class TestStoreFile extends HBaseTestCase { KeyValue midKV = (KeyValue)midkey; byte [] midRow = midKV.getRow(); // Create top split. - HRegionInfo topHri = new HRegionInfo(regionFs.getRegionInfo().getTable(), + HRegionInfo topHri = createHRegionInfo(regionFs.getRegionInfo().getTable(), null, midRow); Path topPath = splitStoreFile(regionFs, topHri, TEST_FAMILY, f, midRow, true); // Create bottom split. - HRegionInfo bottomHri = new HRegionInfo(regionFs.getRegionInfo().getTable(), + HRegionInfo bottomHri = createHRegionInfo(regionFs.getRegionInfo().getTable(), midRow, null); Path bottomPath = splitStoreFile(regionFs, bottomHri, TEST_FAMILY, f, midRow, false); // Make readers on top and bottom. @@ -1037,5 +1038,14 @@ public class TestStoreFile extends HBaseTestCase { byte[] value = fileInfo.get(HFileDataBlockEncoder.DATA_BLOCK_ENCODING); assertEquals(dataBlockEncoderAlgo.getNameInBytes(), value); } + + HRegionInfo createHRegionInfo(final TableName tableName, final byte[] startKey, + final byte[] endKey) { + return new HRegionInfo(tableName, startKey, endKey); + } + + Path getTestRegionPath() { + return new Path("test-region"); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreFileManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreFileManager.java index 3bb0384..8667d0e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreFileManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStripeStoreFileManager.java @@ -40,7 +40,9 @@ import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; @@ -55,7 +57,8 @@ public class TestStripeStoreFileManager { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final Path BASEDIR = TEST_UTIL.getDataTestDir(TestStripeStoreFileManager.class.getSimpleName()); - private static final Path CFDIR = HStore.getStoreHomedir(BASEDIR, "region", Bytes.toBytes("cf")); + private static final HRegionInfo hri = new HRegionInfo(TableName.valueOf("TestStripeStoreFileManager")); + private static final Path CFDIR = HStore.getStoreHomedir(BASEDIR, hri, Bytes.toBytes("cf")); private static final byte[] KEY_A = Bytes.toBytes("aaa"); private static final byte[] KEY_B = Bytes.toBytes("bbb"); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java index 7b5aed5..90767f7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java @@ -211,10 +211,9 @@ public class HFileArchiveTestingUtil { * @return {@link Path} to the archive directory for the given region */ public static Path getRegionArchiveDir(Configuration conf, HRegion region) throws IOException { - return HFileArchiveUtil.getRegionArchiveDir( - FSUtils.getRootDir(conf), - region.getTableDesc().getTableName(), - region.getRegionInfo().getEncodedName()); + return HFileArchiveUtil.getRegionArchiveDir(FSUtils.getRootDir(conf), + region.getTableDesc().getTableName(), region.getRegionFileSystem() + .getRegionDir()); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java index 0ff87af..9894442 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java @@ -132,23 +132,23 @@ import com.google.common.collect.Multimap; public class TestHBaseFsck { static final int POOL_SIZE = 7; private static final Log LOG = LogFactory.getLog(TestHBaseFsck.class); - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private final static Configuration conf = TEST_UTIL.getConfiguration(); - private final static String FAM_STR = "fam"; - private final static byte[] FAM = Bytes.toBytes(FAM_STR); + final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + final static Configuration conf = TEST_UTIL.getConfiguration(); + final static String FAM_STR = "fam"; + final static byte[] FAM = Bytes.toBytes(FAM_STR); private final static int REGION_ONLINE_TIMEOUT = 800; private static RegionStates regionStates; - private static ExecutorService tableExecutorService; - private static ScheduledThreadPoolExecutor hbfsckExecutorService; - private static ClusterConnection connection; + static ExecutorService tableExecutorService; + static ScheduledThreadPoolExecutor hbfsckExecutorService; + static ClusterConnection connection; private static Admin admin; // for the instance, reset every test run - private HTable tbl; - private final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("A"), + HTable tbl; + final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("A"), Bytes.toBytes("B"), Bytes.toBytes("C") }; // one row per region. - private final static byte[][] ROWKEYS= new byte[][] { + final static byte[][] ROWKEYS= new byte[][] { Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"), Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") }; @@ -281,11 +281,11 @@ public class TestHBaseFsck { /** * Create a new region in META. */ - private HRegionInfo createRegion(final HTableDescriptor + HRegionInfo createRegion(final HTableDescriptor htd, byte[] startKey, byte[] endKey) throws IOException { Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService); - HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey); + HRegionInfo hri = new HRegionInfo(htd.getTableName(), shouldTestWithHumongousTable(), startKey, endKey); MetaTableAccessor.addRegionToMeta(meta, hri); meta.close(); return hri; @@ -364,8 +364,7 @@ public class TestHBaseFsck { LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString()); Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); - Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()), - hri.getEncodedName()); + Path p = HRegion.getRegionDir(rootDir, hri); Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE); fs.delete(hriPath, true); } @@ -374,8 +373,7 @@ public class TestHBaseFsck { LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString()); Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); - Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()), - hri.getEncodedName()); + Path p = HRegion.getRegionDir(rootDir, hri); HBaseFsck.debugLsr(conf, p); boolean success = fs.delete(p, true); LOG.info("Deleted " + p + " sucessfully? " + success); @@ -422,6 +420,7 @@ public class TestHBaseFsck { void setupTableWithRegionReplica(TableName tablename, int replicaCount) throws Exception { HTableDescriptor desc = new HTableDescriptor(tablename); desc.setRegionReplication(replicaCount); + desc.setHumongous(shouldTestWithHumongousTable()); HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM)); desc.addFamily(hcd); // If a table has no CF's it doesn't get checked createTable(TEST_UTIL, desc, SPLITS); @@ -436,6 +435,10 @@ public class TestHBaseFsck { tbl.put(puts); tbl.flushCommits(); } + + boolean shouldTestWithHumongousTable() { + return false; + } /** * Counts the number of row to verify data loss or non-dataloss. @@ -2321,7 +2324,7 @@ public class TestHBaseFsck { Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table); Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0); Path famDir = new Path(regionDir, FAM_STR); - Path fakeReferenceFile = new Path(famDir, "fbce357483ceea.12144538"); + Path fakeReferenceFile = new Path(famDir, "fbce357483ceea.d9ffc3a5cd016ae58e23d7a6cb937949"); fs.create(fakeReferenceFile); HBaseFsck hbck = doFsck(conf, false); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckHumongousTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckHumongousTable.java new file mode 100644 index 0000000..a023b48 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckHumongousTable.java @@ -0,0 +1,53 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.junit.experimental.categories.Category; + +/** + * This tests HBaseFsck's ability to detect reasons for inconsistent humongous tables. + */ +@Category({MiscTests.class, LargeTests.class}) +public class TestHBaseFsckHumongousTable extends TestHBaseFsck { + + @Override + boolean shouldTestWithHumongousTable() { + return true; + } + + /* Humongous tables are incompatible with the read-only region replicas feature currently */ + + @Override + public void testHbckWithRegionReplica() throws Exception { + } + + @Override + public void testHbckWithFewerReplica() throws Exception { + } + + @Override + public void testHbckWithExcessReplica() throws Exception { + } + + @Override + public void testNotInHdfsWithReplicas() throws Exception { + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java index e263cdb..2e0584b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java @@ -711,7 +711,7 @@ public class TestWALSplit { @Test (timeout=300000) public void testSplitDeletedRegion() throws IOException { REGIONS.clear(); - String region = "region_that_splits"; + String region = "d9ffc3a5cd016-region_that_splits"; REGIONS.add(region); generateWALs(1); @@ -1008,7 +1008,7 @@ public class TestWALSplit { @Test (timeout=300000) public void testSplitLogFileDeletedRegionDir() throws IOException { LOG.info("testSplitLogFileDeletedRegionDir"); - final String REGION = "region__1"; + final String REGION = "testSplitLogFileDeletedRegionDir"; REGIONS.clear(); REGIONS.add(REGION); @@ -1161,9 +1161,8 @@ public class TestWALSplit { private Path[] getLogForRegion(Path rootdir, TableName table, String region) throws IOException { Path tdir = FSUtils.getTableDir(rootdir, table); - @SuppressWarnings("deprecation") - Path editsdir = WALSplitter.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, - Bytes.toString(region.getBytes()))); + Path regiondir = HRegion.getNormalRegionDir(tdir, Bytes.toString(region.getBytes())); + Path editsdir = WALSplitter.getRegionDirRecoveredEditsDir(regiondir); FileStatus[] files = fs.listStatus(editsdir, new PathFilter() { @Override public boolean accept(Path p) { diff --git a/hbase-shell/src/main/ruby/hbase.rb b/hbase-shell/src/main/ruby/hbase.rb index aca1006..06afd67 100644 --- a/hbase-shell/src/main/ruby/hbase.rb +++ b/hbase-shell/src/main/ruby/hbase.rb @@ -67,6 +67,7 @@ module HBaseConstants VISIBILITY="VISIBILITY" AUTHORIZATIONS = "AUTHORIZATIONS" SKIP_FLUSH = 'SKIP_FLUSH' + HUMONGOUS = 'HUMONGOUS' CONSISTENCY = "CONSISTENCY" USER = 'USER' TABLE = 'TABLE' diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 9cdfe66..ba1618b 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -306,6 +306,7 @@ module Hbase end end htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(DURABILITY))) if arg[DURABILITY] + htd.setHumongous(JBoolean.valueOf(arg.delete(HUMONGOUS))) if arg[HUMONGOUS] set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA] set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] -- 1.7.10.2 (Apple Git-33)