diff --git a/dev-support/test-patch.properties b/dev-support/test-patch.properties index e9edecb..1aecd0c 100644 --- a/dev-support/test-patch.properties +++ b/dev-support/test-patch.properties @@ -20,7 +20,8 @@ MAVEN_OPTS="-Xmx3100M" OK_RELEASEAUDIT_WARNINGS=0 OK_FINDBUGS_WARNINGS=89 -# Allow two warnings. Javadoc complains about sun.misc.Unsafe use. See HBASE-7457 -OK_JAVADOC_WARNINGS=2 +# Allow three warnings. Javadoc complains about sun.misc.Unsafe use. +# See HBASE-7457, HBASE-13761 +OK_JAVADOC_WARNINGS=3 MAX_LINE_LENGTH=100 diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java index 3c7b2ce..175515c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java @@ -988,7 +988,7 @@ public class HRegionInfo implements Comparable { } /** - * Convert a HRegionInfo to a RegionInfo + * Convert a HRegionInfo to the protobuf RegionInfo * * @return the converted RegionInfo */ diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java index eb3553b..6ba25db 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java @@ -99,7 +99,11 @@ public class Get extends Query */ public Get(Get get) { this(get.getRow()); - this.filter = get.getFilter(); + // from Query + this.setFilter(get.getFilter()); + this.setReplicaId(get.getReplicaId()); + this.setConsistency(get.getConsistency()); + // from Get this.cacheBlocks = get.getCacheBlocks(); this.maxVersions = get.getMaxVersions(); this.storeLimit = get.getMaxResultsPerColumnFamily(); @@ -107,7 +111,18 @@ public class Get extends Query this.tr = get.getTimeRange(); this.checkExistenceOnly = get.isCheckExistenceOnly(); this.closestRowBefore = get.isClosestRowBefore(); - this.familyMap = get.getFamilyMap(); + Map> fams = get.getFamilyMap(); + for (Map.Entry> entry : fams.entrySet()) { + byte [] fam = entry.getKey(); + NavigableSet cols = entry.getValue(); + if (cols != null && cols.size() > 0) { + for (byte[] col : cols) { + addColumn(fam, col); + } + } else { + addFamily(fam); + } + } for (Map.Entry attr : get.getAttributesMap().entrySet()) { setAttribute(attr.getKey(), attr.getValue()); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java index f82e554..ec675b4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java @@ -70,6 +70,7 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescripto import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetTableDescriptorsResponse; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.hadoop.hbase.util.Threads; import com.google.common.annotations.VisibleForTesting; @@ -782,6 +783,7 @@ public class HTable implements HTableInterface, RegionLocator { if (scan.getBatch() > 0 && scan.isSmall()) { throw new IllegalArgumentException("Small scan should not be used with batching"); } + if (scan.getCaching() <= 0) { scan.setCaching(getScannerCaching()); } @@ -840,18 +842,28 @@ public class HTable implements HTableInterface, RegionLocator { */ @Override public Result get(final Get get) throws IOException { - if (get.getConsistency() == null){ - get.setConsistency(defaultConsistency); + return get(get, get.isCheckExistenceOnly()); + } + + private Result get(Get get, final boolean checkExistenceOnly) throws IOException { + // if we are changing settings to the get, clone it. + if (get.isCheckExistenceOnly() != checkExistenceOnly || get.getConsistency() == null) { + get = ReflectionUtils.newInstance(get.getClass(), get); + get.setCheckExistenceOnly(checkExistenceOnly); + if (get.getConsistency() == null){ + get.setConsistency(defaultConsistency); + } } if (get.getConsistency() == Consistency.STRONG) { // Good old call. + final Get getReq = get; RegionServerCallable callable = new RegionServerCallable(this.connection, getName(), get.getRow()) { @Override public Result call(int callTimeout) throws IOException { ClientProtos.GetRequest request = - RequestConverter.buildGetRequest(getLocation().getRegionInfo().getRegionName(), get); + RequestConverter.buildGetRequest(getLocation().getRegionInfo().getRegionName(), getReq); PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setPriority(tableName); controller.setCallTimeout(callTimeout); @@ -1362,8 +1374,7 @@ public class HTable implements HTableInterface, RegionLocator { */ @Override public boolean exists(final Get get) throws IOException { - get.setCheckExistenceOnly(true); - Result r = get(get); + Result r = get(get, true); assert r.getExists() != null; return r.getExists(); } @@ -1376,13 +1387,16 @@ public class HTable implements HTableInterface, RegionLocator { if (gets.isEmpty()) return new boolean[]{}; if (gets.size() == 1) return new boolean[]{exists(gets.get(0))}; + ArrayList exists = new ArrayList(gets.size()); for (Get g: gets){ - g.setCheckExistenceOnly(true); + Get ge = new Get(g); + ge.setCheckExistenceOnly(true); + exists.add(ge); } Object[] r1; try { - r1 = batch(gets); + r1 = batch(exists); } catch (InterruptedException e) { throw (InterruptedIOException)new InterruptedIOException().initCause(e); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Put.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Put.java index d72492c..1fdc4e2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Put.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Put.java @@ -123,7 +123,7 @@ public class Put extends Mutation implements HeapSize, Comparable { this(putToCopy.getRow(), putToCopy.ts); this.familyMap = new TreeMap>(Bytes.BYTES_COMPARATOR); for(Map.Entry> entry: putToCopy.getFamilyCellMap().entrySet()) { - this.familyMap.put(entry.getKey(), entry.getValue()); + this.familyMap.put(entry.getKey(), new ArrayList(entry.getValue())); } this.durability = putToCopy.durability; for (Map.Entry entry : putToCopy.getAttributesMap().entrySet()) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java index 69c9591..28af4d2 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java @@ -51,40 +51,33 @@ import org.apache.hadoop.hbase.util.Bytes; * and stopRow may be defined. If rows are not specified, the Scanner will * iterate over all rows. *

- * To scan everything for each row, instantiate a Scan object. + * To get all columns from all rows of a Table, create an instance with no constraints; use the + * {@link #Scan()} constructor. To constrain the scan to specific column families, + * call {@link #addFamily(byte[]) addFamily} for each family to retrieve on your Scan instance. *

- * To modify scanner caching for just this scan, use {@link #setCaching(int) setCaching}. - * If caching is NOT set, we will use the caching value of the hosting {@link Table}. - * In addition to row caching, it is possible to specify a - * maximum result size, using {@link #setMaxResultSize(long)}. When both are used, - * single server requests are limited by either number of rows or maximum result size, whichever - * limit comes first. - *

- * To further define the scope of what to get when scanning, perform additional - * methods as outlined below. - *

- * To get all columns from specific families, execute {@link #addFamily(byte[]) addFamily} - * for each family to retrieve. - *

- * To get specific columns, execute {@link #addColumn(byte[], byte[]) addColumn} + * To get specific columns, call {@link #addColumn(byte[], byte[]) addColumn} * for each column to retrieve. *

* To only retrieve columns within a specific range of version timestamps, - * execute {@link #setTimeRange(long, long) setTimeRange}. + * call {@link #setTimeRange(long, long) setTimeRange}. *

- * To only retrieve columns with a specific timestamp, execute + * To only retrieve columns with a specific timestamp, call * {@link #setTimeStamp(long) setTimestamp}. *

- * To limit the number of versions of each column to be returned, execute + * To limit the number of versions of each column to be returned, call * {@link #setMaxVersions(int) setMaxVersions}. *

* To limit the maximum number of values returned for each call to next(), - * execute {@link #setBatch(int) setBatch}. + * call {@link #setBatch(int) setBatch}. *

- * To add a filter, execute {@link #setFilter(org.apache.hadoop.hbase.filter.Filter) setFilter}. + * To add a filter, call {@link #setFilter(org.apache.hadoop.hbase.filter.Filter) setFilter}. *

* Expert: To explicitly disable server-side block caching for this scan, * execute {@link #setCacheBlocks(boolean)}. + *

Note: Usage alters Scan instances. Internally, attributes are updated as the Scan + * runs and if enabled, metrics accumulate in the Scan instance. Be aware this is the case when + * you go to clone a Scan instance or if you go to reuse a created Scan instance; safer is create + * a Scan instance per usage. */ @InterfaceAudience.Public @InterfaceStability.Stable diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java index 2ea5642..f112b2e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java @@ -19,52 +19,42 @@ package org.apache.hadoop.hbase.filter; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.List; -import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.InvalidProtocolBufferException; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.generated.FilterProtos; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair; import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hbase.util.UnsafeAccess; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.InvalidProtocolBufferException; /** - * Filters data based on fuzzy row key. Performs fast-forwards during scanning. - * It takes pairs (row key, fuzzy info) to match row keys. Where fuzzy info is - * a byte array with 0 or 1 as its values: + * This is optimized version of a standard FuzzyRowFilter Filters data based on fuzzy row key. + * Performs fast-forwards during scanning. It takes pairs (row key, fuzzy info) to match row keys. + * Where fuzzy info is a byte array with 0 or 1 as its values: *

    - *
  • - * 0 - means that this byte in provided row key is fixed, i.e. row key's byte at same position - * must match - *
  • - *
  • - * 1 - means that this byte in provided row key is NOT fixed, i.e. row key's byte at this - * position can be different from the one in provided row key - *
  • + *
  • 0 - means that this byte in provided row key is fixed, i.e. row key's byte at same position + * must match
  • + *
  • 1 - means that this byte in provided row key is NOT fixed, i.e. row key's byte at this + * position can be different from the one in provided row key
  • *
- * - * - * Example: - * Let's assume row key format is userId_actionId_year_month. Length of userId is fixed - * and is 4, length of actionId is 2 and year and month are 4 and 2 bytes long respectively. - * - * Let's assume that we need to fetch all users that performed certain action (encoded as "99") - * in Jan of any year. Then the pair (row key, fuzzy info) would be the following: - * row key = "????_99_????_01" (one can use any value instead of "?") - * fuzzy info = "\x01\x01\x01\x01\x00\x00\x00\x00\x01\x01\x01\x01\x00\x00\x00" - * - * I.e. fuzzy info tells the matching mask is "????_99_????_01", where at ? can be any value. - * + * Example: Let's assume row key format is userId_actionId_year_month. Length of userId is fixed and + * is 4, length of actionId is 2 and year and month are 4 and 2 bytes long respectively. Let's + * assume that we need to fetch all users that performed certain action (encoded as "99") in Jan of + * any year. Then the pair (row key, fuzzy info) would be the following: row key = "????_99_????_01" + * (one can use any value instead of "?") fuzzy info = + * "\x01\x01\x01\x01\x00\x00\x00\x00\x01\x01\x01\x01\x00\x00\x00" I.e. fuzzy info tells the matching + * mask is "????_99_????_01", where at ? can be any value. */ @InterfaceAudience.Public @InterfaceStability.Evolving @@ -72,83 +62,180 @@ public class FuzzyRowFilter extends FilterBase { private List> fuzzyKeysData; private boolean done = false; + /** + * The index of a last successfully found matching fuzzy string (in fuzzyKeysData). We will start + * matching next KV with this one. If they do not match then we will return back to the one-by-one + * iteration over fuzzyKeysData. + */ + private int lastFoundIndex = -1; + + /** + * Row tracker (keeps all next rows after SEEK_NEXT_USING_HINT was returned) + */ + private RowTracker tracker; + public FuzzyRowFilter(List> fuzzyKeysData) { Pair p; for (int i = 0; i < fuzzyKeysData.size(); i++) { p = fuzzyKeysData.get(i); if (p.getFirst().length != p.getSecond().length) { - Pair readable = new Pair( - Bytes.toStringBinary(p.getFirst()), - Bytes.toStringBinary(p.getSecond())); + Pair readable = + new Pair(Bytes.toStringBinary(p.getFirst()), Bytes.toStringBinary(p + .getSecond())); throw new IllegalArgumentException("Fuzzy pair lengths do not match: " + readable); } + // update mask ( 0 -> -1 (0xff), 1 -> 0) + p.setSecond(preprocessMask(p.getSecond())); + preprocessSearchKey(p); } this.fuzzyKeysData = fuzzyKeysData; + this.tracker = new RowTracker(); } - // TODO: possible improvement: save which fuzzy row key to use when providing a hint - @Override - public ReturnCode filterKeyValue(Cell cell) { - // assigning "worst" result first and looking for better options - SatisfiesCode bestOption = SatisfiesCode.NO_NEXT; - for (Pair fuzzyData : fuzzyKeysData) { - SatisfiesCode satisfiesCode = satisfies(isReversed(), cell.getRowArray(), - cell.getRowOffset(), cell.getRowLength(), fuzzyData.getFirst(), fuzzyData.getSecond()); - if (satisfiesCode == SatisfiesCode.YES) { - return ReturnCode.INCLUDE; - } + private void preprocessSearchKey(Pair p) { + if (UnsafeAccess.isAvailable() == false) { + return; + } + byte[] key = p.getFirst(); + byte[] mask = p.getSecond(); + for (int i = 0; i < mask.length; i++) { + // set non-fixed part of a search key to 0. + if (mask[i] == 0) key[i] = 0; + } + } - if (satisfiesCode == SatisfiesCode.NEXT_EXISTS) { - bestOption = SatisfiesCode.NEXT_EXISTS; + /** + * We need to preprocess mask array, as since we treat 0's as unfixed positions and -1 (0xff) as + * fixed positions + * @param mask + * @return mask array + */ + private byte[] preprocessMask(byte[] mask) { + if (UnsafeAccess.isAvailable() == false) { + return mask; + } + if (isPreprocessedMask(mask)) return mask; + for (int i = 0; i < mask.length; i++) { + if (mask[i] == 0) { + mask[i] = -1; // 0 -> -1 + } else if (mask[i] == 1) { + mask[i] = 0;// 1 -> 0 } } + return mask; + } - if (bestOption == SatisfiesCode.NEXT_EXISTS) { - return ReturnCode.SEEK_NEXT_USING_HINT; + private boolean isPreprocessedMask(byte[] mask) { + for (int i = 0; i < mask.length; i++) { + if (mask[i] != -1 && mask[i] != 0) { + return false; + } } - - // the only unhandled SatisfiesCode is NO_NEXT, i.e. we are done - done = true; - return ReturnCode.NEXT_ROW; + return true; } - // Override here explicitly as the method in super class FilterBase might do a KeyValue recreate. - // See HBASE-12068 @Override - public Cell transformCell(Cell v) { - return v; + public ReturnCode filterKeyValue(Cell c) { + final int startIndex = lastFoundIndex >= 0 ? lastFoundIndex : 0; + final int size = fuzzyKeysData.size(); + for (int i = startIndex; i < size + startIndex; i++) { + final int index = i % size; + Pair fuzzyData = fuzzyKeysData.get(index); + SatisfiesCode satisfiesCode = + satisfies(isReversed(), c.getRowArray(), c.getRowOffset(), c.getRowLength(), + fuzzyData.getFirst(), fuzzyData.getSecond()); + if (satisfiesCode == SatisfiesCode.YES) { + lastFoundIndex = index; + return ReturnCode.INCLUDE; + } + } + // NOT FOUND -> seek next using hint + lastFoundIndex = -1; + return ReturnCode.SEEK_NEXT_USING_HINT; + } @Override - public Cell getNextCellHint(Cell curCell) { - byte[] nextRowKey = null; - // Searching for the "smallest" row key that satisfies at least one fuzzy row key - for (Pair fuzzyData : fuzzyKeysData) { - byte[] nextRowKeyCandidate = getNextForFuzzyRule(isReversed(), curCell.getRowArray(), - curCell.getRowOffset(), curCell.getRowLength(), fuzzyData.getFirst(), - fuzzyData.getSecond()); - if (nextRowKeyCandidate == null) { - continue; - } - if (nextRowKey == null || - (reversed && Bytes.compareTo(nextRowKeyCandidate, nextRowKey) > 0) || - (!reversed && Bytes.compareTo(nextRowKeyCandidate, nextRowKey) < 0)) { - nextRowKey = nextRowKeyCandidate; + public Cell getNextCellHint(Cell currentCell) { + boolean result = true; + if (tracker.needsUpdate()) { + result = tracker.updateTracker(currentCell); + } + if (result == false) { + done = true; + return null; + } + byte[] nextRowKey = tracker.nextRow(); + // We need to compare nextRowKey with currentCell + int compareResult = + Bytes.compareTo(nextRowKey, 0, nextRowKey.length, currentCell.getRowArray(), + currentCell.getRowOffset(), currentCell.getRowLength()); + if ((reversed && compareResult > 0) || (!reversed && compareResult < 0)) { + // This can happen when we have multilpe filters and some other filter + // returns next row with hint which is larger (smaller for reverse) + // than the current (really?) + result = tracker.updateTracker(currentCell); + if (result == false) { + done = true; + return null; + } else { + nextRowKey = tracker.nextRow(); } } + return KeyValueUtil.createFirstOnRow(nextRowKey); + } + + /** + * If we have multiple fuzzy keys, row tracker should improve overall performance It calculates + * all next rows (one per every fuzzy key), sort them accordingly (ascending for regular and + * descending for reverse). Next time getNextCellHint is called we check row tracker first and + * return next row from the tracker if it exists, if there are no rows in the tracker we update + * tracker with a current cell and return first row. + */ + private class RowTracker { + private final List nextRows; + private int next = -1; + + RowTracker() { + nextRows = new ArrayList(); + } - if (!reversed && nextRowKey == null) { - // Should never happen for forward scanners; logic in filterKeyValue should return NO_NEXT. - // Can happen in reversed scanner when currentKV is just before the next possible match; in - // this case, fall back on scanner simply calling KeyValueHeap.next() - // TODO: is there a better way than throw exception? (stop the scanner?) - throw new IllegalStateException("No next row key that satisfies fuzzy exists when" + - " getNextKeyHint() is invoked." + - " Filter: " + this.toString() + - " currentKV: " + curCell); + boolean needsUpdate() { + return next == -1 || next == nextRows.size(); + } + + byte[] nextRow() { + if (next < 0 || next == nextRows.size()) return null; + return nextRows.get(next++); + } + + boolean updateTracker(Cell currentCell) { + nextRows.clear(); + for (Pair fuzzyData : fuzzyKeysData) { + byte[] nextRowKeyCandidate = + getNextForFuzzyRule(isReversed(), currentCell.getRowArray(), + currentCell.getRowOffset(), currentCell.getRowLength(), fuzzyData.getFirst(), + fuzzyData.getSecond()); + if (nextRowKeyCandidate == null) { + continue; + } + nextRows.add(nextRowKeyCandidate); + } + // Sort all next row candidates + Collections.sort(nextRows, new Comparator() { + @Override + public int compare(byte[] o1, byte[] o2) { + if (reversed) { + return -Bytes.compareTo(o1, o2); + } else { + return Bytes.compareTo(o1, o2); + } + } + }); + next = 0; + return nextRows.size() > 0; } - return nextRowKey == null ? null : KeyValueUtil.createFirstOnRow(nextRowKey); } @Override @@ -159,9 +246,8 @@ public class FuzzyRowFilter extends FilterBase { /** * @return The filter serialized using pb */ - public byte [] toByteArray() { - FilterProtos.FuzzyRowFilter.Builder builder = - FilterProtos.FuzzyRowFilter.newBuilder(); + public byte[] toByteArray() { + FilterProtos.FuzzyRowFilter.Builder builder = FilterProtos.FuzzyRowFilter.newBuilder(); for (Pair fuzzyData : fuzzyKeysData) { BytesBytesPair.Builder bbpBuilder = BytesBytesPair.newBuilder(); bbpBuilder.setFirst(ByteStringer.wrap(fuzzyData.getFirst())); @@ -177,8 +263,7 @@ public class FuzzyRowFilter extends FilterBase { * @throws DeserializationException * @see #toByteArray */ - public static FuzzyRowFilter parseFrom(final byte [] pbBytes) - throws DeserializationException { + public static FuzzyRowFilter parseFrom(final byte[] pbBytes) throws DeserializationException { FilterProtos.FuzzyRowFilter proto; try { proto = FilterProtos.FuzzyRowFilter.parseFrom(pbBytes); @@ -186,7 +271,7 @@ public class FuzzyRowFilter extends FilterBase { throw new DeserializationException(e); } int count = proto.getFuzzyKeysDataCount(); - ArrayList> fuzzyKeysData= new ArrayList>(count); + ArrayList> fuzzyKeysData = new ArrayList>(count); for (int i = 0; i < count; ++i) { BytesBytesPair current = proto.getFuzzyKeysData(i); byte[] keyBytes = current.getFirst().toByteArray(); @@ -227,12 +312,90 @@ public class FuzzyRowFilter extends FilterBase { @VisibleForTesting static SatisfiesCode satisfies(boolean reverse, byte[] row, byte[] fuzzyKeyBytes, - byte[] fuzzyKeyMeta) { + byte[] fuzzyKeyMeta) { return satisfies(reverse, row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta); } - private static SatisfiesCode satisfies(boolean reverse, byte[] row, int offset, int length, - byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) { + static SatisfiesCode satisfies(boolean reverse, byte[] row, int offset, int length, + byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) { + + if (UnsafeAccess.isAvailable() == false) { + return satisfiesNoUnsafe(reverse, row, offset, length, fuzzyKeyBytes, fuzzyKeyMeta); + } + + if (row == null) { + // do nothing, let scan to proceed + return SatisfiesCode.YES; + } + length = Math.min(length, fuzzyKeyBytes.length); + int numWords = length / Bytes.SIZEOF_LONG; + int offsetAdj = offset + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; + + int j = numWords << 3; // numWords * SIZEOF_LONG; + + for (int i = 0; i < j; i += Bytes.SIZEOF_LONG) { + + long fuzzyBytes = + UnsafeAccess.theUnsafe.getLong(fuzzyKeyBytes, UnsafeAccess.BYTE_ARRAY_BASE_OFFSET + + (long) i); + long fuzzyMeta = + UnsafeAccess.theUnsafe.getLong(fuzzyKeyMeta, UnsafeAccess.BYTE_ARRAY_BASE_OFFSET + + (long) i); + long rowValue = UnsafeAccess.theUnsafe.getLong(row, offsetAdj + (long) i); + if ((rowValue & fuzzyMeta) != (fuzzyBytes)) { + // We always return NEXT_EXISTS + return SatisfiesCode.NEXT_EXISTS; + } + } + + int off = j; + + if (length - off >= Bytes.SIZEOF_INT) { + int fuzzyBytes = + UnsafeAccess.theUnsafe.getInt(fuzzyKeyBytes, UnsafeAccess.BYTE_ARRAY_BASE_OFFSET + + (long) off); + int fuzzyMeta = + UnsafeAccess.theUnsafe.getInt(fuzzyKeyMeta, UnsafeAccess.BYTE_ARRAY_BASE_OFFSET + + (long) off); + int rowValue = UnsafeAccess.theUnsafe.getInt(row, offsetAdj + (long) off); + if ((rowValue & fuzzyMeta) != (fuzzyBytes)) { + // We always return NEXT_EXISTS + return SatisfiesCode.NEXT_EXISTS; + } + off += Bytes.SIZEOF_INT; + } + + if (length - off >= Bytes.SIZEOF_SHORT) { + short fuzzyBytes = + UnsafeAccess.theUnsafe.getShort(fuzzyKeyBytes, UnsafeAccess.BYTE_ARRAY_BASE_OFFSET + + (long) off); + short fuzzyMeta = + UnsafeAccess.theUnsafe.getShort(fuzzyKeyMeta, UnsafeAccess.BYTE_ARRAY_BASE_OFFSET + + (long) off); + short rowValue = UnsafeAccess.theUnsafe.getShort(row, offsetAdj + (long) off); + if ((rowValue & fuzzyMeta) != (fuzzyBytes)) { + // We always return NEXT_EXISTS + // even if it does not (in this case getNextForFuzzyRule + // will return null) + return SatisfiesCode.NEXT_EXISTS; + } + off += Bytes.SIZEOF_SHORT; + } + + if (length - off >= Bytes.SIZEOF_BYTE) { + int fuzzyBytes = fuzzyKeyBytes[off] & 0xff; + int fuzzyMeta = fuzzyKeyMeta[off] & 0xff; + int rowValue = row[offset + off] & 0xff; + if ((rowValue & fuzzyMeta) != (fuzzyBytes)) { + // We always return NEXT_EXISTS + return SatisfiesCode.NEXT_EXISTS; + } + } + return SatisfiesCode.YES; + } + + static SatisfiesCode satisfiesNoUnsafe(boolean reverse, byte[] row, int offset, + int length, byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) { if (row == null) { // do nothing, let scan to proceed return SatisfiesCode.YES; @@ -269,12 +432,11 @@ public class FuzzyRowFilter extends FilterBase { // bigger byte array that satisfies the rule we need to just increase this byte // (see the code of getNextForFuzzyRule below) by one. // Note: if non-fixed byte is already at biggest value, this doesn't allow us to say there's - // bigger one that satisfies the rule as it can't be increased. + // bigger one that satisfies the rule as it can't be increased. if (fuzzyKeyMeta[i] == 1 && !order.isMax(fuzzyKeyBytes[i])) { nextRowKeyCandidateExists = true; } } - return SatisfiesCode.YES; } @@ -285,7 +447,7 @@ public class FuzzyRowFilter extends FilterBase { @VisibleForTesting static byte[] getNextForFuzzyRule(boolean reverse, byte[] row, byte[] fuzzyKeyBytes, - byte[] fuzzyKeyMeta) { + byte[] fuzzyKeyMeta) { return getNextForFuzzyRule(reverse, row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta); } @@ -295,16 +457,20 @@ public class FuzzyRowFilter extends FilterBase { public boolean lt(int lhs, int rhs) { return lhs < rhs; } + public boolean gt(int lhs, int rhs) { return lhs > rhs; } + public byte inc(byte val) { // TODO: what about over/underflow? return (byte) (val + 1); } + public boolean isMax(byte val) { return val == (byte) 0xff; } + public byte min() { return 0; } @@ -313,16 +479,20 @@ public class FuzzyRowFilter extends FilterBase { public boolean lt(int lhs, int rhs) { return lhs > rhs; } + public boolean gt(int lhs, int rhs) { return lhs < rhs; } + public byte inc(byte val) { // TODO: what about over/underflow? return (byte) (val - 1); } + public boolean isMax(byte val) { return val == 0; } + public byte min() { return (byte) 0xFF; } @@ -334,33 +504,37 @@ public class FuzzyRowFilter extends FilterBase { /** Returns true when {@code lhs < rhs}. */ public abstract boolean lt(int lhs, int rhs); + /** Returns true when {@code lhs > rhs}. */ public abstract boolean gt(int lhs, int rhs); + /** Returns {@code val} incremented by 1. */ public abstract byte inc(byte val); + /** Return true when {@code val} is the maximum value */ public abstract boolean isMax(byte val); + /** Return the minimum value according to this ordering scheme. */ public abstract byte min(); } /** - * @return greater byte array than given (row) which satisfies the fuzzy rule if it exists, - * null otherwise + * @return greater byte array than given (row) which satisfies the fuzzy rule if it exists, null + * otherwise */ @VisibleForTesting static byte[] getNextForFuzzyRule(boolean reverse, byte[] row, int offset, int length, - byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) { + byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) { // To find out the next "smallest" byte array that satisfies fuzzy rule and "greater" than // the given one we do the following: // 1. setting values on all "fixed" positions to the values from fuzzyKeyBytes // 2. if during the first step given row did not increase, then we increase the value at - // the first "non-fixed" position (where it is not maximum already) + // the first "non-fixed" position (where it is not maximum already) // It is easier to perform this by using fuzzyKeyBytes copy and setting "non-fixed" position // values than otherwise. - byte[] result = Arrays.copyOf(fuzzyKeyBytes, - length > fuzzyKeyBytes.length ? length : fuzzyKeyBytes.length); + byte[] result = + Arrays.copyOf(fuzzyKeyBytes, length > fuzzyKeyBytes.length ? length : fuzzyKeyBytes.length); if (reverse && length > fuzzyKeyBytes.length) { // we need trailing 0xff's instead of trailing 0x00's for (int i = fuzzyKeyBytes.length; i < result.length; i++) { @@ -372,13 +546,13 @@ public class FuzzyRowFilter extends FilterBase { boolean increased = false; for (int i = 0; i < result.length; i++) { - if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 1) { + if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 0 /* non-fixed */) { result[i] = row[offset + i]; if (!order.isMax(row[offset + i])) { // this is "non-fixed" position and is not at max value, hence we can increase it toInc = i; } - } else if (i < fuzzyKeyMeta.length && fuzzyKeyMeta[i] == 0) { + } else if (i < fuzzyKeyMeta.length && fuzzyKeyMeta[i] == -1 /* fixed */) { if (order.lt((row[i + offset] & 0xFF), (fuzzyKeyBytes[i] & 0xFF))) { // if setting value for any fixed position increased the original array, // we are OK @@ -404,7 +578,7 @@ public class FuzzyRowFilter extends FilterBase { // Setting all "non-fixed" positions to zeroes to the right of the one we increased so // that found "next" row key is the smallest possible for (int i = toInc + 1; i < result.length; i++) { - if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 1) { + if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 0 /* non-fixed */) { result[i] = order.min(); } } @@ -414,20 +588,20 @@ public class FuzzyRowFilter extends FilterBase { } /** - * @return true if and only if the fields of the filter that are serialized - * are equal to the corresponding fields in other. Used for testing. + * @return true if and only if the fields of the filter that are serialized are equal to the + * corresponding fields in other. Used for testing. */ boolean areSerializedFieldsEqual(Filter o) { if (o == this) return true; if (!(o instanceof FuzzyRowFilter)) return false; - FuzzyRowFilter other = (FuzzyRowFilter)o; + FuzzyRowFilter other = (FuzzyRowFilter) o; if (this.fuzzyKeysData.size() != other.fuzzyKeysData.size()) return false; for (int i = 0; i < fuzzyKeysData.size(); ++i) { Pair thisData = this.fuzzyKeysData.get(i); Pair otherData = other.fuzzyKeysData.get(i); - if (!(Bytes.equals(thisData.getFirst(), otherData.getFirst()) - && Bytes.equals(thisData.getSecond(), otherData.getSecond()))) { + if (!(Bytes.equals(thisData.getFirst(), otherData.getFirst()) && Bytes.equals( + thisData.getSecond(), otherData.getSecond()))) { return false; } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java index 6a30511..0a6ba44 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java @@ -168,7 +168,7 @@ public class ReplicationQueuesZKImpl extends ReplicationStateZKBase implements R SortedMap> newQueues = new TreeMap>(); // check whether there is multi support. If yes, use it. if (conf.getBoolean(HConstants.ZOOKEEPER_USEMULTI, true)) { - LOG.info("Atomically moving " + regionserverZnode + "'s wals to my queue"); + LOG.info("Atomically moving " + regionserverZnode + "'s WALs to my queue"); newQueues = copyQueuesFromRSUsingMulti(regionserverZnode); } else { LOG.info("Moving " + regionserverZnode + "'s wals to my queue"); @@ -336,9 +336,9 @@ public class ReplicationQueuesZKImpl extends ReplicationStateZKBase implements R } // add delete op for dead rs listOfOps.add(ZKUtilOp.deleteNodeFailSilent(deadRSZnodePath)); - LOG.debug(" The multi list size is: " + listOfOps.size()); + if (LOG.isTraceEnabled()) LOG.trace(" The multi list size is: " + listOfOps.size()); ZKUtil.multiOrSequential(this.zookeeper, listOfOps, false); - LOG.info("Atomically moved the dead regionserver logs. "); + if (LOG.isTraceEnabled()) LOG.trace("Atomically moved the dead regionserver logs. "); } catch (KeeperException e) { // Multi call failed; it looks like some other regionserver took away the logs. LOG.warn("Got exception in copyQueuesFromRSUsingMulti: ", e); diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java index 54b6ef4..0975c14 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java @@ -17,21 +17,28 @@ */ package org.apache.hadoop.hbase.zookeeper; -import com.google.common.base.Stopwatch; -import com.google.protobuf.InvalidProtocolBufferException; +import java.io.EOFException; +import java.io.IOException; +import java.net.ConnectException; +import java.net.NoRouteToHostException; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.rmi.UnknownHostException; +import java.util.ArrayList; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.RetriesExhaustedException; import org.apache.hadoop.hbase.exceptions.DeserializationException; -import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ipc.FailedServerException; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; import org.apache.hadoop.hbase.master.RegionState; @@ -47,18 +54,8 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.ipc.RemoteException; import org.apache.zookeeper.KeeperException; -import java.io.EOFException; -import java.io.IOException; -import java.net.ConnectException; -import java.net.NoRouteToHostException; -import java.net.SocketException; -import java.net.SocketTimeoutException; -import java.rmi.UnknownHostException; - -import java.util.List; -import java.util.ArrayList; - -import javax.annotation.Nullable; +import com.google.common.base.Stopwatch; +import com.google.protobuf.InvalidProtocolBufferException; /** * Utility class to perform operation (get/wait for/verify/set/delete) on znode in ZooKeeper @@ -620,4 +617,4 @@ public class MetaTableLocator { stopped = true; } } -} +} \ No newline at end of file diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java index 5519dc5..0f57206 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java @@ -381,8 +381,8 @@ public class ZKUtil { * @return ensemble key with a name (if any) */ public static String getZooKeeperClusterKey(Configuration conf, String name) { - String ensemble = conf.get(HConstants.ZOOKEEPER_QUORUM.replaceAll( - "[\\t\\n\\x0B\\f\\r]", "")); + String ensemble = conf.get(HConstants.ZOOKEEPER_QUORUM).replaceAll( + "[\\t\\n\\x0B\\f\\r]", ""); StringBuilder builder = new StringBuilder(ensemble); builder.append(":"); builder.append(conf.get(HConstants.ZOOKEEPER_CLIENT_PORT)); diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java index 1f8ad49..fdb07d0 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestGet.java @@ -41,6 +41,8 @@ import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.KeyOnlyFilter; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; +import org.apache.hadoop.hbase.security.access.Permission; +import org.apache.hadoop.hbase.security.visibility.Authorizations; import org.apache.hadoop.hbase.util.Base64; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Assert; @@ -154,12 +156,48 @@ public class TestGet { Set qualifiers = get.getFamilyMap().get(family); Assert.assertEquals(1, qualifiers.size()); } - + @Test public void TestGetRowFromGetCopyConstructor() throws Exception { Get get = new Get(ROW); + get.setFilter(null); + get.setAuthorizations(new Authorizations("foo")); + get.setACL("u", new Permission(Permission.Action.READ)); + get.setConsistency(Consistency.TIMELINE); + get.setReplicaId(2); + get.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED); + get.setCheckExistenceOnly(true); + get.setClosestRowBefore(true); + get.setTimeRange(3, 4); + get.setMaxVersions(11); + get.setMaxResultsPerColumnFamily(10); + get.setRowOffsetPerColumnFamily(11); + get.setCacheBlocks(true); + Get copyGet = new Get(get); assertEquals(0, Bytes.compareTo(get.getRow(), copyGet.getRow())); + + // from OperationWithAttributes + assertEquals(get.getId(), copyGet.getId()); + + // from Query class + assertEquals(get.getFilter(), copyGet.getFilter()); + assertTrue(get.getAuthorizations().toString().equals(copyGet.getAuthorizations().toString())); + assertTrue(Bytes.equals(get.getACL(), copyGet.getACL())); + assertEquals(get.getConsistency(), copyGet.getConsistency()); + assertEquals(get.getReplicaId(), copyGet.getReplicaId()); + assertEquals(get.getIsolationLevel(), copyGet.getIsolationLevel()); + + // from Get class + assertEquals(get.isCheckExistenceOnly(), copyGet.isCheckExistenceOnly()); + assertEquals(get.isClosestRowBefore(), copyGet.isClosestRowBefore()); + assertTrue(get.getTimeRange().equals(copyGet.getTimeRange())); + assertEquals(get.isClosestRowBefore(), copyGet.isClosestRowBefore()); + assertEquals(get.getMaxVersions(), copyGet.getMaxVersions()); + assertEquals(get.getMaxResultsPerColumnFamily(), copyGet.getMaxResultsPerColumnFamily()); + assertEquals(get.getRowOffsetPerColumnFamily(), copyGet.getRowOffsetPerColumnFamily()); + assertEquals(get.getCacheBlocks(), copyGet.getCacheBlocks()); + assertEquals(get.getId(), copyGet.getId()); } @Test diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestPut.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestPut.java new file mode 100644 index 0000000..3a877dc --- /dev/null +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestPut.java @@ -0,0 +1,43 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.client; + +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +public class TestPut { + @Test + public void testCopyConstructor() { + Put origin = new Put(Bytes.toBytes("ROW-01")); + byte[] family = Bytes.toBytes("CF-01"); + byte[] qualifier = Bytes.toBytes("Q-01"); + + origin.addColumn(family, qualifier, Bytes.toBytes("V-01")); + Put clone = new Put(origin); + + assertEquals(origin.getCellList(family), clone.getCellList(family)); + origin.addColumn(family, qualifier, Bytes.toBytes("V-02")); + + //They should have different cell lists + assertNotEquals(origin.getCellList(family), clone.getCellList(family)); + + } +} diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java new file mode 100644 index 0000000..9e0f5ce --- /dev/null +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKUtil.java @@ -0,0 +1,43 @@ +/* + * Copyright 2015 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import org.apache.hadoop.conf.Configuration; + +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * + */ +@Category({SmallTests.class}) +public class TestZKUtil { + + @Test + public void testGetZooKeeperClusterKey() { + Configuration conf = HBaseConfiguration.create(); + conf.set(HConstants.ZOOKEEPER_QUORUM, "\tlocalhost\n"); + conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, "3333"); + conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "hbase"); + String clusterKey = ZKUtil.getZooKeeperClusterKey(conf, "test"); + Assert.assertTrue(!clusterKey.contains("\t") && !clusterKey.contains("\n")); + Assert.assertEquals("localhost:3333:hbase,test", clusterKey); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index ae0dad2..70562e2 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -280,7 +280,7 @@ public final class HConstants { "hbase.client.meta.operation.timeout"; /** Default HBase client operation timeout, which is tantamount to a blocking call */ - public static final int DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT = 60000; + public static final int DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT = 1200000; /** Used to construct the name of the log directory for a region server */ public static final String HREGION_LOGDIR_NAME = "WALs"; diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java index bae5fb4..26d215c 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java @@ -2498,22 +2498,11 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, * @return Created KeyValue OR if we find a length of zero, we will return null which * can be useful marking a stream as done. * @throws IOException + * @{@link Deprecated} Use {@link KeyValueUtil#iscreate(InputStream, boolean)} */ + @Deprecated public static KeyValue iscreate(final InputStream in) throws IOException { - byte [] intBytes = new byte[Bytes.SIZEOF_INT]; - int bytesRead = 0; - while (bytesRead < intBytes.length) { - int n = in.read(intBytes, bytesRead, intBytes.length - bytesRead); - if (n < 0) { - if (bytesRead == 0) return null; // EOF at start is ok - throw new IOException("Failed read of int, read " + bytesRead + " bytes"); - } - bytesRead += n; - } - // TODO: perhaps some sanity check is needed here. - byte [] bytes = new byte[Bytes.toInt(intBytes)]; - IOUtils.readFully(in, bytes, 0, bytes.length); - return new KeyValue(bytes, 0, bytes.length); + return KeyValueUtil.iscreate(in, true); } /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java index 8505fe9..adf0ad7 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java @@ -18,7 +18,9 @@ package org.apache.hadoop.hbase; +import java.io.DataInput; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -31,6 +33,7 @@ import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.IterableUtils; import org.apache.hadoop.hbase.util.SimpleMutableByteRange; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.WritableUtils; import com.google.common.base.Function; @@ -542,6 +545,39 @@ public class KeyValueUtil { return new ArrayList(lazyList); } + /** + * Create a KeyValue reading from the raw InputStream. Named + * iscreate so doesn't clash with {@link #create(DataInput)} + * + * @param in + * @param withTags + * whether the keyvalue should include tags are not + * @return Created KeyValue OR if we find a length of zero, we will return + * null which can be useful marking a stream as done. + * @throws IOException + */ + public static KeyValue iscreate(final InputStream in, boolean withTags) throws IOException { + byte[] intBytes = new byte[Bytes.SIZEOF_INT]; + int bytesRead = 0; + while (bytesRead < intBytes.length) { + int n = in.read(intBytes, bytesRead, intBytes.length - bytesRead); + if (n < 0) { + if (bytesRead == 0) + return null; // EOF at start is ok + throw new IOException("Failed read of int, read " + bytesRead + " bytes"); + } + bytesRead += n; + } + // TODO: perhaps some sanity check is needed here. + byte[] bytes = new byte[Bytes.toInt(intBytes)]; + IOUtils.readFully(in, bytes, 0, bytes.length); + if (withTags) { + return new KeyValue(bytes, 0, bytes.length); + } else { + return new NoTagsKeyValue(bytes, 0, bytes.length); + } + } + public static void oswrite(final Cell cell, final OutputStream out, final boolean withTags) throws IOException { if (cell instanceof KeyValue) { diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java index cfe9742..07fd838 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java @@ -65,7 +65,7 @@ public class KeyValueCodec implements Codec { } protected Cell parseCell() throws IOException { - return KeyValue.iscreate(in); + return KeyValueUtil.iscreate(in, false); } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java index 02158f4..5d34a46 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java @@ -71,7 +71,7 @@ public class KeyValueCodecWithTags implements Codec { } protected Cell parseCell() throws IOException { - return KeyValue.iscreate(in); + return KeyValueUtil.iscreate(in, true); } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java index 132abf5..c685a92 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/BoundedByteBufferPool.java @@ -19,12 +19,13 @@ package org.apache.hadoop.hbase.io; import java.nio.ByteBuffer; import java.util.Queue; -import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.BoundedArrayQueue; import com.google.common.annotations.VisibleForTesting; @@ -55,7 +56,8 @@ public class BoundedByteBufferPool { private final int maxByteBufferSizeToCache; // A running average only it only rises, it never recedes - private volatile int runningAverage; + @VisibleForTesting + volatile int runningAverage; // Scratch that keeps rough total size of pooled bytebuffers private volatile int totalReservoirCapacity; @@ -63,6 +65,8 @@ public class BoundedByteBufferPool { // For reporting private AtomicLong allocations = new AtomicLong(0); + private ReentrantLock lock = new ReentrantLock(); + /** * @param maxByteBufferSizeToCache * @param initialByteBufferSize @@ -72,15 +76,23 @@ public class BoundedByteBufferPool { final int maxToCache) { this.maxByteBufferSizeToCache = maxByteBufferSizeToCache; this.runningAverage = initialByteBufferSize; - this.buffers = new ArrayBlockingQueue(maxToCache, true); + this.buffers = new BoundedArrayQueue(maxToCache); } public ByteBuffer getBuffer() { - ByteBuffer bb = this.buffers.poll(); + ByteBuffer bb = null; + lock.lock(); + try { + bb = this.buffers.poll(); + if (bb != null) { + this.totalReservoirCapacity -= bb.capacity(); + } + } finally { + lock.unlock(); + } if (bb != null) { - // Clear sets limit == capacity. Postion == 0. + // Clear sets limit == capacity. Postion == 0. bb.clear(); - this.totalReservoirCapacity -= bb.capacity(); } else { bb = ByteBuffer.allocate(this.runningAverage); this.allocations.incrementAndGet(); @@ -96,15 +108,21 @@ public class BoundedByteBufferPool { public void putBuffer(ByteBuffer bb) { // If buffer is larger than we want to keep around, just let it go. if (bb.capacity() > this.maxByteBufferSizeToCache) return; - if (!this.buffers.offer(bb)) { + boolean success = false; + int average = 0; + lock.lock(); + try { + success = this.buffers.offer(bb); + if (success) { + this.totalReservoirCapacity += bb.capacity(); + average = this.totalReservoirCapacity / this.buffers.size(); // size will never be 0. + } + } finally { + lock.unlock(); + } + if (!success) { LOG.warn("At capacity: " + this.buffers.size()); } else { - int size = this.buffers.size(); // This size may be inexact. - this.totalReservoirCapacity += bb.capacity(); - int average = 0; - if (size != 0) { - average = this.totalReservoirCapacity / size; - } if (average > this.runningAverage && average < this.maxByteBufferSizeToCache) { this.runningAverage = average; } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/BoundedArrayQueue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/BoundedArrayQueue.java new file mode 100644 index 0000000..9db4c5c --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/BoundedArrayQueue.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.util.AbstractQueue; +import java.util.Iterator; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * A bounded non-thread safe implementation of {@link java.util.Queue}. + */ +@InterfaceAudience.Private +public class BoundedArrayQueue extends AbstractQueue { + + private Object[] items; + private int takeIndex, putIndex; + private int count; + + public BoundedArrayQueue(int maxElements) { + items = new Object[maxElements]; + } + + @Override + public int size() { + return count; + } + + /** + * Not implemented and will throw {@link UnsupportedOperationException} + */ + @Override + public Iterator iterator() { + // We don't need this. Leaving it as not implemented. + throw new UnsupportedOperationException(); + } + + @Override + public boolean offer(E e) { + if (count == items.length) return false; + items[putIndex] = e; + if (++putIndex == items.length) putIndex = 0; + count++; + return true; + } + + @Override + public E poll() { + return (count == 0) ? null : dequeue(); + } + + @SuppressWarnings("unchecked") + private E dequeue() { + E x = (E) items[takeIndex]; + items[takeIndex] = null; + if (++takeIndex == items.length) takeIndex = 0; + count--; + return x; + } + + @SuppressWarnings("unchecked") + @Override + public E peek() { + return (E) items[takeIndex]; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java index c59071c..ff54ebe 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java @@ -1186,7 +1186,7 @@ public class Bytes { * @param offset Offset into array at which vint begins. * @throws java.io.IOException e * @return deserialized long from buffer. - * @deprecated Use {@link #readAsVLong()} instead. + * @deprecated Use {@link #readAsVLong(byte[], int)} instead. */ @Deprecated public static long readVLong(final byte [] buffer, final int offset) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ReflectionUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ReflectionUtils.java index 5e8980d..650c544 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ReflectionUtils.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ReflectionUtils.java @@ -18,8 +18,17 @@ */ package org.apache.hadoop.hbase.util; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.nio.charset.Charset; + +import org.apache.commons.logging.Log; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -89,4 +98,93 @@ public class ReflectionUtils { throw new UnsupportedOperationException( "Unable to find suitable constructor for class " + type.getName()); } + + /* synchronized on ReflectionUtils.class */ + private static long previousLogTime = 0; + private static final ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); + + /** + * Log the current thread stacks at INFO level. + * @param log the logger that logs the stack trace + * @param title a descriptive title for the call stacks + * @param minInterval the minimum time from the last + */ + public static void logThreadInfo(Log log, + String title, + long minInterval) { + boolean dumpStack = false; + if (log.isInfoEnabled()) { + synchronized (ReflectionUtils.class) { + long now = System.currentTimeMillis(); + if (now - previousLogTime >= minInterval * 1000) { + previousLogTime = now; + dumpStack = true; + } + } + if (dumpStack) { + try { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + printThreadInfo(new PrintStream(buffer, false, "UTF-8"), title); + log.info(buffer.toString(Charset.defaultCharset().name())); + } catch (UnsupportedEncodingException ignored) { + log.warn("Could not write thread info about '" + title + + "' due to a string encoding issue."); + } + } + } + } + + /** + * Print all of the thread's information and stack traces. + * + * @param stream the stream to + * @param title a string title for the stack trace + */ + private static void printThreadInfo(PrintStream stream, + String title) { + final int STACK_DEPTH = 20; + boolean contention = threadBean.isThreadContentionMonitoringEnabled(); + long[] threadIds = threadBean.getAllThreadIds(); + stream.println("Process Thread Dump: " + title); + stream.println(threadIds.length + " active threads"); + for (long tid: threadIds) { + ThreadInfo info = threadBean.getThreadInfo(tid, STACK_DEPTH); + if (info == null) { + stream.println(" Inactive"); + continue; + } + stream.println("Thread " + + getTaskName(info.getThreadId(), + info.getThreadName()) + ":"); + Thread.State state = info.getThreadState(); + stream.println(" State: " + state); + stream.println(" Blocked count: " + info.getBlockedCount()); + stream.println(" Waited count: " + info.getWaitedCount()); + if (contention) { + stream.println(" Blocked time: " + info.getBlockedTime()); + stream.println(" Waited time: " + info.getWaitedTime()); + } + if (state == Thread.State.WAITING) { + stream.println(" Waiting on " + info.getLockName()); + } else if (state == Thread.State.BLOCKED) { + stream.println(" Blocked on " + info.getLockName()); + stream.println(" Blocked by " + + getTaskName(info.getLockOwnerId(), + info.getLockOwnerName())); + } + stream.println(" Stack:"); + for (StackTraceElement frame: info.getStackTrace()) { + stream.println(" " + frame.toString()); + } + } + stream.flush(); + } + + private static String getTaskName(long id, String name) { + if (name == null) { + return Long.toString(id); + } + return id + " (" + name + ")"; + } + } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java index 4c2748b..cae56af 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/RetryCounterFactory.java @@ -21,7 +21,7 @@ package org.apache.hadoop.hbase.util; import java.util.concurrent.TimeUnit; import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.util.RetryCounter.ExponentialBackoffPolicy; +import org.apache.hadoop.hbase.util.RetryCounter.ExponentialBackoffPolicyWithLimit; import org.apache.hadoop.hbase.util.RetryCounter.RetryConfig; @InterfaceAudience.Private @@ -29,12 +29,16 @@ public class RetryCounterFactory { private final RetryConfig retryConfig; public RetryCounterFactory(int maxAttempts, int sleepIntervalMillis) { + this(maxAttempts, sleepIntervalMillis, -1); + } + + public RetryCounterFactory(int maxAttempts, int sleepIntervalMillis, int maxSleepTime) { this(new RetryConfig( maxAttempts, sleepIntervalMillis, - -1, + maxSleepTime, TimeUnit.MILLISECONDS, - new ExponentialBackoffPolicy())); + new ExponentialBackoffPolicyWithLimit())); } public RetryCounterFactory(RetryConfig retryConfig) { diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/UnsafeAccess.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/UnsafeAccess.java new file mode 100644 index 0000000..680bee4 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/UnsafeAccess.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import java.lang.reflect.Field; +import java.nio.ByteOrder; +import java.security.AccessController; +import java.security.PrivilegedAction; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +import sun.misc.Unsafe; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public final class UnsafeAccess { + + private static final Log LOG = LogFactory.getLog(UnsafeAccess.class); + + public static final Unsafe theUnsafe; + + /** The offset to the first element in a byte array. */ + public static final int BYTE_ARRAY_BASE_OFFSET; + + static { + theUnsafe = (Unsafe) AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Object run() { + try { + Field f = Unsafe.class.getDeclaredField("theUnsafe"); + f.setAccessible(true); + return f.get(null); + } catch (Throwable e) { + LOG.warn("sun.misc.Unsafe is not accessible", e); + } + return null; + } + }); + + if(theUnsafe != null){ + BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class); + } else{ + BYTE_ARRAY_BASE_OFFSET = -1; + } + } + + private UnsafeAccess(){} + + public static boolean isAvailable() { + return theUnsafe != null; + } + + public static final boolean littleEndian = ByteOrder.nativeOrder() + .equals(ByteOrder.LITTLE_ENDIAN); +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestBoundedByteBufferPool.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestBoundedByteBufferPool.java index 79b9e68..5074b4c 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestBoundedByteBufferPool.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestBoundedByteBufferPool.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.io; import static org.junit.Assert.assertEquals; import java.nio.ByteBuffer; +import java.util.concurrent.ConcurrentLinkedDeque; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.junit.After; @@ -84,4 +85,64 @@ public class TestBoundedByteBufferPool { } assertEquals(maxToCache, this.reservoir.buffers.size()); } -} + + @Test + public void testBufferSizeGrowWithMultiThread() throws Exception { + final ConcurrentLinkedDeque bufferQueue = new ConcurrentLinkedDeque(); + int takeBufferThreadsCount = 30; + int putBufferThreadsCount = 1; + Thread takeBufferThreads[] = new Thread[takeBufferThreadsCount]; + for (int i = 0; i < takeBufferThreadsCount; i++) { + takeBufferThreads[i] = new Thread(new Runnable() { + @Override + public void run() { + while (true) { + ByteBuffer buffer = reservoir.getBuffer(); + try { + Thread.sleep(5); + } catch (InterruptedException e) { + break; + } + bufferQueue.offer(buffer); + if (Thread.currentThread().isInterrupted()) break; + } + } + }); + } + + Thread putBufferThread[] = new Thread[putBufferThreadsCount]; + for (int i = 0; i < putBufferThreadsCount; i++) { + putBufferThread[i] = new Thread(new Runnable() { + @Override + public void run() { + while (true) { + ByteBuffer buffer = bufferQueue.poll(); + if (buffer != null) { + reservoir.putBuffer(buffer); + } + if (Thread.currentThread().isInterrupted()) break; + } + } + }); + } + + for (int i = 0; i < takeBufferThreadsCount; i++) { + takeBufferThreads[i].start(); + } + for (int i = 0; i < putBufferThreadsCount; i++) { + putBufferThread[i].start(); + } + Thread.sleep(2 * 1000);// Let the threads run for 2 secs + for (int i = 0; i < takeBufferThreadsCount; i++) { + takeBufferThreads[i].interrupt(); + takeBufferThreads[i].join(); + } + for (int i = 0; i < putBufferThreadsCount; i++) { + putBufferThread[i].interrupt(); + putBufferThread[i].join(); + } + // None of the BBs we got from pool is growing while in use. So we should not change the + // runningAverage in pool + assertEquals(initialByteBufferSize, this.reservoir.runningAverage); + } +} \ No newline at end of file diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBoundedArrayQueue.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBoundedArrayQueue.java new file mode 100644 index 0000000..9570009 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBoundedArrayQueue.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestBoundedArrayQueue { + + private int qMaxElements = 5; + private BoundedArrayQueue queue = new BoundedArrayQueue(qMaxElements); + + @Test + public void testBoundedArrayQueueOperations() throws Exception { + assertEquals(0, queue.size()); + assertNull(queue.poll()); + assertNull(queue.peek()); + for(int i=0;i implements Comparable { * The main code of the procedure. It must be idempotent since execute() * may be called multiple time in case of machine failure in the middle * of the execution. + * @param env the environment passed to the ProcedureExecutor * @return a set of sub-procedures or null if there is nothing else to execute. + * @throws ProcedureYieldException the procedure will be added back to the queue and retried later + * @throws InterruptedException the procedure will be added back to the queue and retried later */ protected abstract Procedure[] execute(TEnvironment env) - throws ProcedureYieldException; + throws ProcedureYieldException, InterruptedException; /** * The code to undo what done by the execute() code. @@ -94,10 +97,12 @@ public abstract class Procedure implements Comparable { * the execute() call. The implementation must be idempotent since rollback() * may be called multiple time in case of machine failure in the middle * of the execution. + * @param env the environment passed to the ProcedureExecutor * @throws IOException temporary failure, the rollback will retry later + * @throws InterruptedException the procedure will be added back to the queue and retried later */ protected abstract void rollback(TEnvironment env) - throws IOException; + throws IOException, InterruptedException; /** * The abort() call is asynchronous and each procedure must decide how to deal @@ -168,6 +173,18 @@ public abstract class Procedure implements Comparable { // no-op } + /** + * By default, the executor will try ro run procedures start to finish. + * Return true to make the executor yield between each execution step to + * give other procedures time to run their steps. + * @param env the environment passed to the ProcedureExecutor + * @return Return true if the executor should yield on completion of an execution step. + * Defaults to return false. + */ + protected boolean isYieldAfterExecutionStep(final TEnvironment env) { + return false; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -394,7 +411,7 @@ public abstract class Procedure implements Comparable { */ @InterfaceAudience.Private protected Procedure[] doExecute(final TEnvironment env) - throws ProcedureYieldException { + throws ProcedureYieldException, InterruptedException { try { updateTimestamp(); return execute(env); @@ -408,7 +425,8 @@ public abstract class Procedure implements Comparable { * user-level code rollback(). */ @InterfaceAudience.Private - protected void doRollback(final TEnvironment env) throws IOException { + protected void doRollback(final TEnvironment env) + throws IOException, InterruptedException { try { updateTimestamp(); rollback(env); @@ -519,6 +537,20 @@ public abstract class Procedure implements Comparable { return (diff < 0) ? -1 : (diff > 0) ? 1 : 0; } + /** + * Get an hashcode for the specified Procedure ID + * @return the hashcode for the specified procId + */ + public static long getProcIdHashCode(final long procId) { + long h = procId; + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + return h; + } + /* * Helper to lookup the root Procedure ID given a specified procedure. */ @@ -677,4 +709,4 @@ public abstract class Procedure implements Comparable { return proc; } -} \ No newline at end of file +} diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java index 6e87997..d83d856 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java @@ -28,7 +28,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.HashSet; -import java.util.TreeSet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -43,6 +42,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; +import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator; import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.procedure2.util.TimeoutBlockingQueue; import org.apache.hadoop.hbase.procedure2.util.TimeoutBlockingQueue.TimeoutRetriever; @@ -148,8 +148,8 @@ public class ProcedureExecutor { public void periodicExecute(final TEnvironment env) { if (completed.isEmpty()) { - if (LOG.isDebugEnabled()) { - LOG.debug("No completed procedures to cleanup."); + if (LOG.isTraceEnabled()) { + LOG.trace("No completed procedures to cleanup."); } return; } @@ -264,45 +264,70 @@ public class ProcedureExecutor { this.conf = conf; } - private List> load() throws IOException { + private void load(final boolean abortOnCorruption) throws IOException { Preconditions.checkArgument(completed.isEmpty()); Preconditions.checkArgument(rollbackStack.isEmpty()); Preconditions.checkArgument(procedures.isEmpty()); Preconditions.checkArgument(waitingTimeout.isEmpty()); Preconditions.checkArgument(runnables.size() == 0); - // 1. Load the procedures - Iterator loader = store.load(); - if (loader == null) { - lastProcId.set(0); - return null; - } + store.load(new ProcedureStore.ProcedureLoader() { + @Override + public void setMaxProcId(long maxProcId) { + assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()"; + LOG.debug("load procedures maxProcId=" + maxProcId); + lastProcId.set(maxProcId); + } - long logMaxProcId = 0; - int runnablesCount = 0; - while (loader.hasNext()) { - Procedure proc = loader.next(); - proc.beforeReplay(getEnvironment()); - procedures.put(proc.getProcId(), proc); - logMaxProcId = Math.max(logMaxProcId, proc.getProcId()); - if (LOG.isDebugEnabled()) { - LOG.debug("Loading procedure state=" + proc.getState() + - " isFailed=" + proc.hasException() + ": " + proc); + @Override + public void load(ProcedureIterator procIter) throws IOException { + loadProcedures(procIter, abortOnCorruption); + } + + @Override + public void handleCorrupted(ProcedureIterator procIter) throws IOException { + int corruptedCount = 0; + while (procIter.hasNext()) { + Procedure proc = procIter.next(); + LOG.error("corrupted procedure: " + proc); + corruptedCount++; + } + if (abortOnCorruption && corruptedCount > 0) { + throw new IOException("found " + corruptedCount + " procedures on replay"); + } } + }); + } + + private void loadProcedures(final ProcedureIterator procIter, + final boolean abortOnCorruption) throws IOException { + // 1. Build the rollback stack + int runnablesCount = 0; + while (procIter.hasNext()) { + Procedure proc = procIter.next(); if (!proc.hasParent() && !proc.isFinished()) { rollbackStack.put(proc.getProcId(), new RootProcedureState()); } + // add the procedure to the map + proc.beforeReplay(getEnvironment()); + procedures.put(proc.getProcId(), proc); + if (proc.getState() == ProcedureState.RUNNABLE) { runnablesCount++; } } - assert lastProcId.get() < 0; - lastProcId.set(logMaxProcId); // 2. Initialize the stacks - TreeSet runnableSet = null; + ArrayList runnableList = new ArrayList(runnablesCount); HashSet waitingSet = null; - for (final Procedure proc: procedures.values()) { + procIter.reset(); + while (procIter.hasNext()) { + Procedure proc = procIter.next(); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("Loading procedure state=%s isFailed=%s: %s", + proc.getState(), proc.hasException(), proc)); + } + Long rootProcId = getRootProcedureId(proc); if (rootProcId == null) { // The 'proc' was ready to run but the root procedure was rolledback? @@ -312,10 +337,11 @@ public class ProcedureExecutor { if (!proc.hasParent() && proc.isFinished()) { if (LOG.isDebugEnabled()) { - LOG.debug("The procedure is completed state=" + proc.getState() + - " isFailed=" + proc.hasException() + ": " + proc); + LOG.debug(String.format("The procedure is completed state=%s isFailed=%s", + proc.getState(), proc.hasException())); } assert !rollbackStack.containsKey(proc.getProcId()); + procedures.remove(proc.getProcId()); completed.put(proc.getProcId(), newResultFromProcedure(proc)); continue; } @@ -333,10 +359,7 @@ public class ProcedureExecutor { switch (proc.getState()) { case RUNNABLE: - if (runnableSet == null) { - runnableSet = new TreeSet(); - } - runnableSet.add(proc); + runnableList.add(proc); break; case WAITING_TIMEOUT: if (waitingSet == null) { @@ -361,7 +384,7 @@ public class ProcedureExecutor { } // 3. Validate the stacks - List> corrupted = null; + int corruptedCount = 0; Iterator> itStack = rollbackStack.entrySet().iterator(); while (itStack.hasNext()) { Map.Entry entry = itStack.next(); @@ -369,32 +392,49 @@ public class ProcedureExecutor { if (procStack.isValid()) continue; for (Procedure proc: procStack.getSubprocedures()) { + LOG.error("corrupted procedure: " + proc); procedures.remove(proc.getProcId()); - if (runnableSet != null) runnableSet.remove(proc); + runnableList.remove(proc); if (waitingSet != null) waitingSet.remove(proc); + corruptedCount++; } itStack.remove(); - if (corrupted == null) { - corrupted = new ArrayList>(); - } - corrupted.add(entry); + } + + if (abortOnCorruption && corruptedCount > 0) { + throw new IOException("found " + corruptedCount + " procedures on replay"); } // 4. Push the runnables - if (runnableSet != null) { - // TODO: See ProcedureWALFormatReader.readInitEntry() some procedure - // may be started way before this stuff. - for (Procedure proc: runnableSet) { + if (!runnableList.isEmpty()) { + // TODO: See ProcedureWALFormatReader#hasFastStartSupport + // some procedure may be started way before this stuff. + for (int i = runnableList.size() - 1; i >= 0; --i) { + Procedure proc = runnableList.get(i); if (!proc.hasParent()) { sendProcedureLoadedNotification(proc.getProcId()); } - runnables.addBack(proc); + if (proc.wasExecuted()) { + runnables.addFront(proc); + } else { + // if it was not in execution, it can wait. + runnables.addBack(proc); + } } } - return corrupted; } - public void start(int numThreads) throws IOException { + /** + * Start the procedure executor. + * It calls ProcedureStore.recoverLease() and ProcedureStore.load() to + * recover the lease, and ensure a single executor, and start the procedure + * replay to resume and recover the previous pending and in-progress perocedures. + * + * @param numThreads number of threads available for procedure execution. + * @param abortOnCorruption true if you want to abort your service in case + * a corrupted procedure is found on replay. otherwise false. + */ + public void start(int numThreads, boolean abortOnCorruption) throws IOException { if (running.getAndSet(true)) { LOG.warn("Already running"); return; @@ -427,11 +467,11 @@ public class ProcedureExecutor { store.recoverLease(); // TODO: Split in two steps. - // TODO: Handle corrupted procedure returned (probably just a WARN) + // TODO: Handle corrupted procedures (currently just a warn) // The first one will make sure that we have the latest id, // so we can start the threads and accept new procedures. // The second step will do the actual load of old procedures. - load(); + load(abortOnCorruption); // Start the executors. Here we must have the lastProcId set. for (int i = 0; i < threads.length; ++i) { @@ -692,6 +732,12 @@ public class ProcedureExecutor { procedureFinished(proc); break; } + + // if the procedure is kind enough to pass the slot to someone else, yield + if (proc.isYieldAfterExecutionStep(getEnvironment())) { + runnables.yield(proc); + break; + } } while (procStack.isFailed()); } @@ -788,6 +834,11 @@ public class ProcedureExecutor { } subprocStack.remove(stackTail); + + // if the procedure is kind enough to pass the slot to someone else, yield + if (proc.isYieldAfterExecutionStep(getEnvironment())) { + return false; + } } // Finalize the procedure state @@ -811,6 +862,9 @@ public class ProcedureExecutor { LOG.debug("rollback attempt failed for " + proc, e); } return false; + } catch (InterruptedException e) { + handleInterruptedException(proc, e); + return false; } catch (Throwable e) { // Catch NullPointerExceptions or similar errors... LOG.fatal("CODE-BUG: Uncatched runtime exception for procedure: " + proc, e); @@ -819,9 +873,7 @@ public class ProcedureExecutor { // allows to kill the executor before something is stored to the wal. // useful to test the procedure recovery. if (testing != null && testing.shouldKillBeforeStoreUpdate()) { - if (LOG.isDebugEnabled()) { - LOG.debug("TESTING: Kill before store update"); - } + LOG.debug("TESTING: Kill before store update"); stop(); return false; } @@ -837,6 +889,7 @@ public class ProcedureExecutor { } else { store.update(proc); } + return true; } @@ -872,10 +925,14 @@ public class ProcedureExecutor { } } catch (ProcedureYieldException e) { if (LOG.isTraceEnabled()) { - LOG.trace("Yield procedure: " + procedure); + LOG.trace("Yield procedure: " + procedure + ": " + e.getMessage()); } runnables.yield(procedure); return; + } catch (InterruptedException e) { + handleInterruptedException(procedure, e); + runnables.yield(procedure); + return; } catch (Throwable e) { // Catch NullPointerExceptions or similar errors... String msg = "CODE-BUG: Uncatched runtime exception for procedure: " + procedure; @@ -934,9 +991,7 @@ public class ProcedureExecutor { // allows to kill the executor before something is stored to the wal. // useful to test the procedure recovery. if (testing != null && testing.shouldKillBeforeStoreUpdate()) { - if (LOG.isDebugEnabled()) { - LOG.debug("TESTING: Kill before store update"); - } + LOG.debug("TESTING: Kill before store update"); stop(); return; } @@ -959,6 +1014,11 @@ public class ProcedureExecutor { return; } + // if the procedure is kind enough to pass the slot to someone else, yield + if (reExecute && procedure.isYieldAfterExecutionStep(getEnvironment())) { + return; + } + assert (reExecute && subprocs == null) || !reExecute; } while (reExecute); @@ -995,6 +1055,18 @@ public class ProcedureExecutor { } } + private void handleInterruptedException(final Procedure proc, final InterruptedException e) { + if (LOG.isTraceEnabled()) { + LOG.trace("got an interrupt during " + proc + ". suspend and retry it later.", e); + } + + // NOTE: We don't call Thread.currentThread().interrupt() + // because otherwise all the subsequent calls e.g. Thread.sleep() will throw + // the InterruptedException. If the master is going down, we will be notified + // and the executor/store will be stopped. + // (The interrupted procedure will be retried on the next run) + } + private void sendProcedureLoadedNotification(final long procId) { if (!this.listeners.isEmpty()) { for (ProcedureExecutorListener listener: this.listeners) { @@ -1094,4 +1166,4 @@ public class ProcedureExecutor { } return new ProcedureResult(proc.getStartTime(), proc.getLastUpdate(), proc.getResult()); } -} \ No newline at end of file +} diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SequentialProcedure.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SequentialProcedure.java index 8ddb36e..bcb0424 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SequentialProcedure.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/SequentialProcedure.java @@ -42,7 +42,7 @@ public abstract class SequentialProcedure extends Procedure extends Procedure * Flow.HAS_MORE_STATE if there is another step. */ protected abstract Flow executeFromState(TEnvironment env, TState state) - throws ProcedureYieldException; + throws ProcedureYieldException, InterruptedException; /** * called to perform the rollback of the specified state @@ -65,7 +65,7 @@ public abstract class StateMachineProcedure * @throws IOException temporary failure, the rollback will retry later */ protected abstract void rollbackState(TEnvironment env, TState state) - throws IOException; + throws IOException, InterruptedException; /** * Convert an ordinal (or state id) to an Enum (or more descriptive) state object. @@ -95,12 +95,24 @@ public abstract class StateMachineProcedure setNextState(getStateId(state)); } + /** + * By default, the executor will try ro run all the steps of the procedure start to finish. + * Return true to make the executor yield between execution steps to + * give other procedures time to run their steps. + * @param state the state we are going to execute next. + * @return Return true if the executor should yield before the execution of the specified step. + * Defaults to return false. + */ + protected boolean isYieldBeforeExecuteFromState(TEnvironment env, TState state) { + return false; + } + @Override protected Procedure[] execute(final TEnvironment env) - throws ProcedureYieldException { + throws ProcedureYieldException, InterruptedException { updateTimestamp(); try { - TState state = stateCount > 0 ? getState(states[stateCount-1]) : getInitialState(); + TState state = getCurrentState(); if (stateCount == 0) { setNextState(getStateId(state)); } @@ -115,16 +127,26 @@ public abstract class StateMachineProcedure } @Override - protected void rollback(final TEnvironment env) throws IOException { + protected void rollback(final TEnvironment env) + throws IOException, InterruptedException { try { updateTimestamp(); - rollbackState(env, stateCount > 0 ? getState(states[stateCount-1]) : getInitialState()); + rollbackState(env, getCurrentState()); stateCount--; } finally { updateTimestamp(); } } + @Override + protected boolean isYieldAfterExecutionStep(final TEnvironment env) { + return isYieldBeforeExecuteFromState(env, getCurrentState()); + } + + private TState getCurrentState() { + return stateCount > 0 ? getState(states[stateCount-1]) : getInitialState(); + } + /** * Set the next state for the procedure. * @param stateId the ordinal() of the state enum (or state id) diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStore.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStore.java index 06bfa44..a05c115 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStore.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStore.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.procedure2.store; import java.io.IOException; -import java.util.Iterator; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; @@ -46,6 +45,57 @@ public interface ProcedureStore { } /** + * An Iterator over a collection of Procedure + */ + public interface ProcedureIterator { + /** + * Reset the Iterator by seeking to the beginning of the list. + */ + void reset(); + + /** + * Returns true if the iterator has more elements. + * (In other words, returns true if next() would return a Procedure + * rather than throwing an exception.) + * @return true if the iterator has more procedures + */ + boolean hasNext(); + + /** + * Returns the next procedure in the iteration. + * @throws IOException if there was an error fetching/deserializing the procedure + * @throws NoSuchElementException if the iteration has no more elements + * @return the next procedure in the iteration. + */ + Procedure next() throws IOException; + } + + /** + * Interface passed to the ProcedureStore.load() method to handle the store-load events. + */ + public interface ProcedureLoader { + /** + * Called by ProcedureStore.load() to notify about the maximum proc-id in the store. + * @param maxProcId the highest proc-id in the store + */ + void setMaxProcId(long maxProcId); + + /** + * Called by the ProcedureStore.load() every time a set of procedures are ready to be executed. + * The ProcedureIterator passed to the method, has the procedure sorted in replay-order. + * @param procIter iterator over the procedures ready to be added to the executor. + */ + void load(ProcedureIterator procIter) throws IOException; + + /** + * Called by the ProcedureStore.load() in case we have procedures not-ready to be added to + * the executor, which probably means they are corrupted since some information/link is missing. + * @param procIter iterator over the procedures not ready to be added to the executor, corrupted + */ + void handleCorrupted(ProcedureIterator procIter) throws IOException; + } + + /** * Add the listener to the notification list. * @param listener The AssignmentListener to register */ @@ -87,9 +137,9 @@ public interface ProcedureStore { /** * Load the Procedures in the store. - * @return the set of procedures present in the store + * @param loader the ProcedureLoader that will handle the store-load events */ - Iterator load() throws IOException; + void load(ProcedureLoader loader) throws IOException; /** * When a procedure is submitted to the executor insert(proc, null) will be called. diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.java index 17432ac..c75c141 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormat.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.hbase.io.util.StreamUtils; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker; +import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureLoader; import org.apache.hadoop.hbase.procedure2.util.ByteSlot; import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALEntry; import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALHeader; @@ -63,14 +64,14 @@ public final class ProcedureWALFormat { } } - interface Loader { + interface Loader extends ProcedureLoader { void removeLog(ProcedureWALFile log); void markCorruptedWAL(ProcedureWALFile log, IOException e); } private ProcedureWALFormat() {} - public static Iterator load(final Iterator logs, + public static void load(final Iterator logs, final ProcedureStoreTracker tracker, final Loader loader) throws IOException { ProcedureWALFormatReader reader = new ProcedureWALFormatReader(tracker); tracker.setKeepDeletes(true); @@ -84,14 +85,13 @@ public final class ProcedureWALFormat { log.close(); } } + reader.finalize(loader); // The tracker is now updated with all the procedures read from the logs tracker.setPartialFlag(false); tracker.resetUpdates(); } finally { tracker.setKeepDeletes(false); } - // TODO: Write compacted version? - return reader.getProcedures(); } public static void writeHeader(OutputStream stream, ProcedureWALHeader header) diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java index a60b8f5..76c0554 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/ProcedureWALFormatReader.java @@ -19,9 +19,6 @@ package org.apache.hadoop.hbase.procedure2.store.wal; import java.io.IOException; -import java.util.Iterator; -import java.util.Map; -import java.util.HashMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -30,6 +27,7 @@ import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker; +import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator; import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos; import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALEntry; @@ -41,17 +39,74 @@ import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureWALEn public class ProcedureWALFormatReader { private static final Log LOG = LogFactory.getLog(ProcedureWALFormatReader.class); - private final ProcedureStoreTracker tracker; - //private final long compactionLogId; - - private final Map procedures = new HashMap(); - private final Map localProcedures = - new HashMap(); + // ============================================================================================== + // We read the WALs in reverse order. from the newest to the oldest. + // We have different entry types: + // - INIT: Procedure submitted by the user (also known as 'root procedure') + // - INSERT: Children added to the procedure :[, ...] + // - UPDATE: The specified procedure was updated + // - DELETE: The procedure was removed (completed/rolledback and result TTL expired) + // + // In the WAL we can find multiple times the same procedure as UPDATE or INSERT. + // We read the WAL from top to bottom, so every time we find an entry of the + // same procedure, that will be the "latest" update. + // + // We keep two in-memory maps: + // - localProcedureMap: is the map containing the entries in the WAL we are processing + // - procedureMap: is the map containing all the procedures we found up to the WAL in process. + // localProcedureMap is merged with the procedureMap once we reach the WAL EOF. + // + // Since we are reading the WALs in reverse order (newest to oldest), + // if we find an entry related to a procedure we already have in 'procedureMap' we can discard it. + // + // The WAL is append-only so the last procedure in the WAL is the one that + // was in execution at the time we crashed/closed the server. + // given that, the procedure replay order can be inferred by the WAL order. + // + // Example: + // WAL-2: [A, B, A, C, D] + // WAL-1: [F, G, A, F, B] + // Replay-Order: [D, C, A, B, F, G] + // + // The "localProcedureMap" keeps a "replayOrder" list. Every time we add the + // record to the map that record is moved to the head of the "replayOrder" list. + // Using the example above: + // WAL-2 localProcedureMap.replayOrder is [D, C, A, B] + // WAL-1 localProcedureMap.replayOrder is [F, G] + // + // each time we reach the WAL-EOF, the "replayOrder" list is merged/appended in 'procedureMap' + // so using the example above we end up with: [D, C, A, B] + [F, G] as replay order. + // + // Fast Start: INIT/INSERT record and StackIDs + // --------------------------------------------- + // We have to special record, INIT and INSERT that tracks the first time + // the procedure was added to the WAL. We can use that information to be able + // to start procedures before reaching the end of the WAL, or before reading all the WALs. + // but in some cases the WAL with that record can be already gone. + // In alternative we can use the stackIds on each procedure, + // to identify when a procedure is ready to start. + // If there are gaps in the sum of the stackIds we need to read more WALs. + // + // Example (all procs child of A): + // WAL-2: [A, B] A stackIds = [0, 4], B stackIds = [1, 5] + // WAL-1: [A, B, C, D] + // + // In the case above we need to read one more WAL to be able to consider + // the root procedure A and all children as ready. + // ============================================================================================== + private final WalProcedureMap localProcedureMap = new WalProcedureMap(1024); + private final WalProcedureMap procedureMap = new WalProcedureMap(1024); + //private long compactionLogId; private long maxProcId = 0; + private final ProcedureStoreTracker tracker; + private final boolean hasFastStartSupport; + public ProcedureWALFormatReader(final ProcedureStoreTracker tracker) { this.tracker = tracker; + // we support fast-start only if we have a clean shutdown. + this.hasFastStartSupport = !tracker.isEmpty(); } public void read(ProcedureWALFile log, ProcedureWALFormat.Loader loader) throws IOException { @@ -91,58 +146,65 @@ public class ProcedureWALFormatReader { loader.markCorruptedWAL(log, e); } - if (localProcedures.isEmpty()) { + if (localProcedureMap.isEmpty()) { LOG.info("No active entry found in state log " + log + ". removing it"); loader.removeLog(log); } else { - Iterator> itd = - localProcedures.entrySet().iterator(); - while (itd.hasNext()) { - Map.Entry entry = itd.next(); - itd.remove(); + procedureMap.mergeTail(localProcedureMap); - // Deserialize the procedure - Procedure proc = Procedure.convert(entry.getValue()); - procedures.put(entry.getKey(), proc); - } - - // TODO: Some procedure may be already runnables (see readInitEntry()) - // (we can also check the "update map" in the log trackers) + //if (hasFastStartSupport) { + // TODO: Some procedure may be already runnables (see readInitEntry()) + // (we can also check the "update map" in the log trackers) + // -------------------------------------------------- + //EntryIterator iter = procedureMap.fetchReady(); + //if (iter != null) loader.load(iter); + // -------------------------------------------------- + //} } } - public Iterator getProcedures() { - return procedures.values().iterator(); + public void finalize(ProcedureWALFormat.Loader loader) throws IOException { + // notify the loader about the max proc ID + loader.setMaxProcId(maxProcId); + + // fetch the procedure ready to run. + ProcedureIterator procIter = procedureMap.fetchReady(); + if (procIter != null) loader.load(procIter); + + // remaining procedures have missing link or dependencies + // consider them as corrupted, manual fix is probably required. + procIter = procedureMap.fetchAll(); + if (procIter != null) loader.handleCorrupted(procIter); } - private void loadEntries(final ProcedureWALEntry entry) { - for (ProcedureProtos.Procedure proc: entry.getProcedureList()) { - maxProcId = Math.max(maxProcId, proc.getProcId()); - if (isRequired(proc.getProcId())) { - if (LOG.isTraceEnabled()) { - LOG.trace("read " + entry.getType() + " entry " + proc.getProcId()); - } - localProcedures.put(proc.getProcId(), proc); - tracker.setDeleted(proc.getProcId(), false); + private void loadProcedure(final ProcedureWALEntry entry, final ProcedureProtos.Procedure proc) { + maxProcId = Math.max(maxProcId, proc.getProcId()); + if (isRequired(proc.getProcId())) { + if (LOG.isTraceEnabled()) { + LOG.trace("read " + entry.getType() + " entry " + proc.getProcId()); } + localProcedureMap.add(proc); + tracker.setDeleted(proc.getProcId(), false); } } private void readInitEntry(final ProcedureWALEntry entry) throws IOException { assert entry.getProcedureCount() == 1 : "Expected only one procedure"; - // TODO: Make it runnable, before reading other files - loadEntries(entry); + loadProcedure(entry, entry.getProcedure(0)); } private void readInsertEntry(final ProcedureWALEntry entry) throws IOException { assert entry.getProcedureCount() >= 1 : "Expected one or more procedures"; - loadEntries(entry); + loadProcedure(entry, entry.getProcedure(0)); + for (int i = 1; i < entry.getProcedureCount(); ++i) { + loadProcedure(entry, entry.getProcedure(i)); + } } private void readUpdateEntry(final ProcedureWALEntry entry) throws IOException { assert entry.getProcedureCount() == 1 : "Expected only one procedure"; - loadEntries(entry); + loadProcedure(entry, entry.getProcedure(0)); } private void readDeleteEntry(final ProcedureWALEntry entry) throws IOException { @@ -152,7 +214,7 @@ public class ProcedureWALFormatReader { LOG.trace("read delete entry " + entry.getProcId()); } maxProcId = Math.max(maxProcId, entry.getProcId()); - localProcedures.remove(entry.getProcId()); + localProcedureMap.remove(entry.getProcId()); tracker.setDeleted(entry.getProcId(), true); } @@ -161,6 +223,458 @@ public class ProcedureWALFormatReader { } private boolean isRequired(final long procId) { - return !isDeleted(procId) && !procedures.containsKey(procId); + return !isDeleted(procId) && !procedureMap.contains(procId); + } + + // ========================================================================== + // We keep an in-memory map of the procedures sorted by replay order. + // (see the details in the beginning of the file) + // _______________________________________________ + // procedureMap = | A | | E | | C | | | | | G | | | + // D B + // replayOrderHead = C <-> B <-> E <-> D <-> A <-> G + // + // We also have a lazy grouping by "root procedure", and a list of + // unlinked procedure. If after reading all the WALs we have unlinked + // procedures it means that we had a missing WAL or a corruption. + // rootHead = A <-> D <-> G + // B E + // C + // unlinkFromLinkList = None + // ========================================================================== + private static class Entry { + // hash-table next + protected Entry hashNext; + // child head + protected Entry childHead; + // double-link for rootHead or childHead + protected Entry linkNext; + protected Entry linkPrev; + // replay double-linked-list + protected Entry replayNext; + protected Entry replayPrev; + // procedure-infos + protected Procedure procedure; + protected ProcedureProtos.Procedure proto; + protected boolean ready = false; + + public Entry(Entry hashNext) { this.hashNext = hashNext; } + + public long getProcId() { return proto.getProcId(); } + public long getParentId() { return proto.getParentId(); } + public boolean hasParent() { return proto.hasParentId(); } + public boolean isReady() { return ready; } + + public Procedure convert() throws IOException { + if (procedure == null) { + procedure = Procedure.convert(proto); + } + return procedure; + } + + @Override + public String toString() { + return "Entry(" + getProcId() + ", parentId=" + getParentId() + ")"; + } + } + + private static class EntryIterator implements ProcedureIterator { + private final Entry replayHead; + private Entry current; + + public EntryIterator(Entry replayHead) { + this.replayHead = replayHead; + this.current = replayHead; + } + + @Override + public void reset() { + this.current = replayHead; + } + + @Override + public boolean hasNext() { + return current != null; + } + + @Override + public Procedure next() throws IOException { + try { + return current.convert(); + } finally { + current = current.replayNext; + } + } + } + + private static class WalProcedureMap { + // procedure hash table + private Entry[] procedureMap; + + // replay-order double-linked-list + private Entry replayOrderHead; + private Entry replayOrderTail; + + // root linked-list + private Entry rootHead; + + // pending unlinked children (root not present yet) + private Entry childUnlinkedHead; + + public WalProcedureMap(int size) { + procedureMap = new Entry[size]; + replayOrderHead = null; + replayOrderTail = null; + rootHead = null; + childUnlinkedHead = null; + } + + public void add(ProcedureProtos.Procedure procProto) { + Entry entry = addToMap(procProto.getProcId(), procProto.hasParentId()); + boolean isNew = entry.proto == null; + entry.proto = procProto; + addToReplayList(entry); + + if (isNew) { + if (procProto.hasParentId()) { + childUnlinkedHead = addToLinkList(entry, childUnlinkedHead); + } else { + rootHead = addToLinkList(entry, rootHead); + } + } + } + + public boolean remove(long procId) { + Entry entry = removeFromMap(procId); + if (entry != null) { + unlinkFromReplayList(entry); + unlinkFromLinkList(entry); + return true; + } + return false; + } + + public boolean contains(long procId) { + return getProcedure(procId) != null; + } + + public boolean isEmpty() { + return replayOrderHead == null; + } + + public void clear() { + for (int i = 0; i < procedureMap.length; ++i) { + procedureMap[i] = null; + } + replayOrderHead = null; + replayOrderTail = null; + rootHead = null; + childUnlinkedHead = null; + } + + /* + * Merges two WalProcedureMap, + * the target is the "global" map, the source is the "local" map. + * - The entries in the hashtables are guaranteed to be unique. + * On replay we don't load procedures that already exist in the "global" + * map (the one we are merging the "local" in to). + * - The replayOrderList of the "local" nao will be appended to the "global" + * map replay list. + * - The "local" map will be cleared at the end of the operation. + */ + public void mergeTail(WalProcedureMap other) { + for (Entry p = other.replayOrderHead; p != null; p = p.replayNext) { + int slotIndex = getMapSlot(p.getProcId()); + p.hashNext = procedureMap[slotIndex]; + procedureMap[slotIndex] = p; + } + + if (replayOrderHead == null) { + replayOrderHead = other.replayOrderHead; + replayOrderTail = other.replayOrderTail; + rootHead = other.rootHead; + childUnlinkedHead = other.childUnlinkedHead; + } else { + // append replay list + assert replayOrderTail.replayNext == null; + assert other.replayOrderHead.replayPrev == null; + replayOrderTail.replayNext = other.replayOrderHead; + other.replayOrderHead.replayPrev = replayOrderTail; + replayOrderTail = other.replayOrderTail; + + // merge rootHead + if (rootHead == null) { + rootHead = other.rootHead; + } else if (other.rootHead != null) { + Entry otherTail = findLinkListTail(other.rootHead); + otherTail.linkNext = rootHead; + rootHead.linkPrev = otherTail; + rootHead = other.rootHead; + } + + // merge childUnlinkedHead + if (childUnlinkedHead == null) { + childUnlinkedHead = other.childUnlinkedHead; + } else if (other.childUnlinkedHead != null) { + Entry otherTail = findLinkListTail(other.childUnlinkedHead); + otherTail.linkNext = childUnlinkedHead; + childUnlinkedHead.linkPrev = otherTail; + childUnlinkedHead = other.childUnlinkedHead; + } + } + + other.clear(); + } + + /* + * Returns an EntryIterator with the list of procedures ready + * to be added to the executor. + * A Procedure is ready if its children and parent are ready. + */ + public EntryIterator fetchReady() { + buildGraph(); + + Entry readyHead = null; + Entry readyTail = null; + Entry p = replayOrderHead; + while (p != null) { + Entry next = p.replayNext; + if (p.isReady()) { + unlinkFromReplayList(p); + if (readyTail != null) { + readyTail.replayNext = p; + p.replayPrev = readyTail; + } else { + p.replayPrev = null; + readyHead = p; + } + readyTail = p; + p.replayNext = null; + } + p = next; + } + // we need the hash-table lookups for parents, so this must be done + // out of the loop where we check isReadyToRun() + for (p = readyHead; p != null; p = p.replayNext) { + removeFromMap(p.getProcId()); + unlinkFromLinkList(p); + } + return readyHead != null ? new EntryIterator(readyHead) : null; + } + + /* + * Drain this map and return all procedures in it. + */ + public EntryIterator fetchAll() { + Entry head = replayOrderHead; + for (Entry p = head; p != null; p = p.replayNext) { + removeFromMap(p.getProcId()); + } + for (int i = 0; i < procedureMap.length; ++i) { + assert procedureMap[i] == null : "map not empty i=" + i; + } + replayOrderHead = null; + replayOrderTail = null; + childUnlinkedHead = null; + rootHead = null; + return head != null ? new EntryIterator(head) : null; + } + + private void buildGraph() { + Entry p = childUnlinkedHead; + while (p != null) { + Entry next = p.linkNext; + Entry rootProc = getRootProcedure(p); + if (rootProc != null) { + rootProc.childHead = addToLinkList(p, rootProc.childHead); + } + p = next; + } + + for (p = rootHead; p != null; p = p.linkNext) { + checkReadyToRun(p); + } + } + + private Entry getRootProcedure(Entry entry) { + while (entry != null && entry.hasParent()) { + entry = getProcedure(entry.getParentId()); + } + return entry; + } + + /* + * (see the comprehensive explaination in the beginning of the file) + * A Procedure is ready when parent and children are ready. + * "ready" means that we all the information that we need in-memory. + * + * Example-1: + * We have two WALs, we start reading fronm the newest (wal-2) + * wal-2 | C B | + * wal-1 | A B C | + * + * If C and B don't depend on A (A is not the parent), we can start them + * before reading wal-1. If B is the only one with parent A we can start C + * and read one more WAL before being able to start B. + * + * How do we know with the only information in B that we are not ready. + * - easy case, the parent is missing from the global map + * - more complex case we look at the Stack IDs + * + * The Stack-IDs are added to the procedure order as incremental index + * tracking how many times that procedure was executed, which is equivalent + * at the number of times we wrote the procedure to the WAL. + * In the example above: + * wal-2: B has stackId = [1, 2] + * wal-1: B has stackId = [1] + * wal-1: A has stackId = [0] + * + * Since we know that the Stack-IDs are incremental for a Procedure, + * we notice that there is a gap in the stackIds of B, so something was + * executed before. + * To identify when a Procedure is ready we do the sum of the stackIds of + * the procedure and the parent. if the stackIdSum is equals to the + * sum of {1..maxStackId} then everything we need is avaiable. + * + * Example-2 + * wal-2 | A | A stackIds = [0, 2] + * wal-1 | A B | B stackIds = [1] + * + * There is a gap between A stackIds so something was executed in between. + */ + private boolean checkReadyToRun(Entry rootEntry) { + int stackIdSum = 0; + int maxStackId = 0; + for (int i = 0; i < rootEntry.proto.getStackIdCount(); ++i) { + int stackId = 1 + rootEntry.proto.getStackId(i); + maxStackId = Math.max(maxStackId, stackId); + stackIdSum += stackId; + } + + for (Entry p = rootEntry.childHead; p != null; p = p.linkNext) { + for (int i = 0; i < p.proto.getStackIdCount(); ++i) { + int stackId = 1 + p.proto.getStackId(i); + maxStackId = Math.max(maxStackId, stackId); + stackIdSum += stackId; + } + } + final int cmpStackIdSum = (maxStackId * (maxStackId + 1) / 2); + if (cmpStackIdSum == stackIdSum) { + rootEntry.ready = true; + for (Entry p = rootEntry.childHead; p != null; p = p.linkNext) { + p.ready = true; + } + return true; + } + return false; + } + + private void unlinkFromReplayList(Entry entry) { + if (replayOrderHead == entry) { + replayOrderHead = entry.replayNext; + } + if (replayOrderTail == entry) { + replayOrderTail = entry.replayPrev; + } + if (entry.replayPrev != null) { + entry.replayPrev.replayNext = entry.replayNext; + } + if (entry.replayNext != null) { + entry.replayNext.replayPrev = entry.replayPrev; + } + } + + private void addToReplayList(final Entry entry) { + unlinkFromReplayList(entry); + entry.replayNext = replayOrderHead; + entry.replayPrev = null; + if (replayOrderHead != null) { + replayOrderHead.replayPrev = entry; + } else { + replayOrderTail = entry; + } + replayOrderHead = entry; + } + + private void unlinkFromLinkList(Entry entry) { + if (entry == rootHead) { + rootHead = entry.linkNext; + } else if (entry == childUnlinkedHead) { + childUnlinkedHead = entry.linkNext; + } + if (entry.linkPrev != null) { + entry.linkPrev.linkNext = entry.linkNext; + } + if (entry.linkNext != null) { + entry.linkNext.linkPrev = entry.linkPrev; + } + } + + private Entry addToLinkList(Entry entry, Entry linkHead) { + unlinkFromLinkList(entry); + entry.linkNext = linkHead; + entry.linkPrev = null; + if (linkHead != null) { + linkHead.linkPrev = entry; + } + return entry; + } + + private Entry findLinkListTail(Entry linkHead) { + Entry tail = linkHead; + while (tail.linkNext != null) { + tail = tail.linkNext; + } + return tail; + } + + private Entry addToMap(final long procId, final boolean hasParent) { + int slotIndex = getMapSlot(procId); + Entry entry = getProcedure(slotIndex, procId); + if (entry != null) return entry; + + entry = new Entry(procedureMap[slotIndex]); + procedureMap[slotIndex] = entry; + return entry; + } + + private Entry removeFromMap(final long procId) { + int slotIndex = getMapSlot(procId); + Entry prev = null; + Entry entry = procedureMap[slotIndex]; + while (entry != null) { + if (procId == entry.getProcId()) { + if (prev != null) { + prev.hashNext = entry.hashNext; + } else { + procedureMap[slotIndex] = entry.hashNext; + } + entry.hashNext = null; + return entry; + } + prev = entry; + entry = entry.hashNext; + } + return null; + } + + private Entry getProcedure(final long procId) { + return getProcedure(getMapSlot(procId), procId); + } + + private Entry getProcedure(final int slotIndex, final long procId) { + Entry entry = procedureMap[slotIndex]; + while (entry != null) { + if (procId == entry.getProcId()) { + return entry; + } + entry = entry.hashNext; + } + return null; + } + + private int getMapSlot(final long procId) { + return (int)(Procedure.getProcIdHashCode(procId) % procedureMap.length); + } } } \ No newline at end of file diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java index 1884adc..f4a52b1 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/wal/WALProcedureStore.java @@ -215,13 +215,12 @@ public class WALProcedureStore implements ProcedureStore { FileStatus[] oldLogs = getLogFiles(); while (running.get()) { // Get Log-MaxID and recover lease on old logs - flushLogId = initOldLogs(oldLogs) + 1; + flushLogId = initOldLogs(oldLogs); // Create new state-log - if (!rollWriter(flushLogId)) { - if (LOG.isDebugEnabled()) { - LOG.debug("Someone else has already created log: " + flushLogId); - } + if (!rollWriter(flushLogId + 1)) { + // someone else has already created this log + LOG.debug("someone else has already created log " + flushLogId); continue; } @@ -241,7 +240,7 @@ public class WALProcedureStore implements ProcedureStore { } @Override - public Iterator load() throws IOException { + public void load(final ProcedureLoader loader) throws IOException { if (logs.isEmpty()) { throw new RuntimeException("recoverLease() must be called before loading data"); } @@ -251,7 +250,8 @@ public class WALProcedureStore implements ProcedureStore { if (LOG.isDebugEnabled()) { LOG.debug("No state logs to replay."); } - return null; + loader.setMaxProcId(0); + return; } // Load the old logs @@ -259,7 +259,22 @@ public class WALProcedureStore implements ProcedureStore { Iterator it = logs.descendingIterator(); it.next(); // Skip the current log try { - return ProcedureWALFormat.load(it, storeTracker, new ProcedureWALFormat.Loader() { + ProcedureWALFormat.load(it, storeTracker, new ProcedureWALFormat.Loader() { + @Override + public void setMaxProcId(long maxProcId) { + loader.setMaxProcId(maxProcId); + } + + @Override + public void load(ProcedureIterator procIter) throws IOException { + loader.load(procIter); + } + + @Override + public void handleCorrupted(ProcedureIterator procIter) throws IOException { + loader.handleCorrupted(procIter); + } + @Override public void removeLog(ProcedureWALFile log) { toRemove.add(log); @@ -301,7 +316,7 @@ public class WALProcedureStore implements ProcedureStore { } // Push the transaction data and wait until it is persisted - logId = pushData(slot); + pushData(slot); } catch (IOException e) { // We are not able to serialize the procedure. // this is a code error, and we are not able to go on. @@ -383,7 +398,7 @@ public class WALProcedureStore implements ProcedureStore { storeTracker.delete(procId); if (logId == flushLogId) { if (storeTracker.isEmpty() && totalSynced.get() > rollThreshold) { - removeOldLogs = rollWriterOrDie(logId + 1); + removeOldLogs = rollWriterOrDie(); } } } @@ -541,9 +556,9 @@ public class WALProcedureStore implements ProcedureStore { } } - private boolean rollWriterOrDie(final long logId) { + private boolean rollWriterOrDie() { try { - return rollWriter(logId); + return rollWriter(); } catch (IOException e) { LOG.warn("Unable to roll the log", e); sendAbortProcessSignal(); @@ -551,7 +566,13 @@ public class WALProcedureStore implements ProcedureStore { } } + protected boolean rollWriter() throws IOException { + return rollWriter(flushLogId + 1); + } + private boolean rollWriter(final long logId) throws IOException { + assert logId > flushLogId : "logId=" + logId + " flushLogId=" + flushLogId; + ProcedureWALHeader header = ProcedureWALHeader.newBuilder() .setVersion(ProcedureWALFormat.HEADER_VERSION) .setType(ProcedureWALFormat.LOG_TYPE_STREAM) diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java index 7b9fc69..ddea9d2 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java @@ -57,11 +57,11 @@ public class ProcedureTestingUtility { public static void restart(ProcedureExecutor procExecutor) throws Exception { - restart(procExecutor, null); + restart(procExecutor, null, true); } public static void restart(ProcedureExecutor procExecutor, - Runnable beforeStartAction) throws Exception { + Runnable beforeStartAction, boolean failOnCorrupted) throws Exception { ProcedureStore procStore = procExecutor.getStore(); int storeThreads = procExecutor.getNumThreads(); int execThreads = procExecutor.getNumThreads(); @@ -75,7 +75,7 @@ public class ProcedureTestingUtility { } // re-start procStore.start(storeThreads); - procExecutor.start(execThreads); + procExecutor.start(execThreads, failOnCorrupted); } public static void setKillBeforeStoreUpdate(ProcedureExecutor procExecutor, diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java index 022f8ad..0b2a364 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureExecution.java @@ -70,7 +70,7 @@ public class TestProcedureExecution { procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), fs, logDir); procExecutor = new ProcedureExecutor(htu.getConfiguration(), null, procStore); procStore.start(PROCEDURE_EXECUTOR_SLOTS); - procExecutor.start(PROCEDURE_EXECUTOR_SLOTS); + procExecutor.start(PROCEDURE_EXECUTOR_SLOTS, true); } @After diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java index f21b6fa..7735b63 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureRecovery.java @@ -74,7 +74,7 @@ public class TestProcedureRecovery { procExecutor = new ProcedureExecutor(htu.getConfiguration(), null, procStore); procExecutor.testing = new ProcedureExecutor.Testing(); procStore.start(PROCEDURE_EXECUTOR_SLOTS); - procExecutor.start(PROCEDURE_EXECUTOR_SLOTS); + procExecutor.start(PROCEDURE_EXECUTOR_SLOTS, true); procSleepInterval = 0; } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureReplayOrder.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureReplayOrder.java index 8a7c1a1..61c58e1 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureReplayOrder.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestProcedureReplayOrder.java @@ -19,13 +19,17 @@ package org.apache.hadoop.hbase.procedure2; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseCommonTestingUtility; +import org.apache.hadoop.hbase.io.util.StreamUtils; import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -45,7 +49,7 @@ import static org.junit.Assert.fail; public class TestProcedureReplayOrder { private static final Log LOG = LogFactory.getLog(TestProcedureReplayOrder.class); - private static final Procedure NULL_PROC = null; + private static final int NUM_THREADS = 16; private ProcedureExecutor procExecutor; private TestProcedureEnv procEnv; @@ -59,7 +63,7 @@ public class TestProcedureReplayOrder { @Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); - htu.getConfiguration().setInt("hbase.procedure.store.wal.sync.wait.msec", 10); + htu.getConfiguration().setInt("hbase.procedure.store.wal.sync.wait.msec", 25); testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); @@ -69,8 +73,8 @@ public class TestProcedureReplayOrder { procEnv = new TestProcedureEnv(); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), fs, logDir); procExecutor = new ProcedureExecutor(htu.getConfiguration(), procEnv, procStore); - procStore.start(24); - procExecutor.start(1); + procStore.start(NUM_THREADS); + procExecutor.start(1, true); } @After @@ -81,47 +85,45 @@ public class TestProcedureReplayOrder { } @Test(timeout=90000) - public void testSingleStepReplyOrder() throws Exception { - // avoid the procedure to be runnable - procEnv.setAcquireLock(false); + public void testSingleStepReplayOrder() throws Exception { + final int NUM_PROC_XTHREAD = 32; + final int NUM_PROCS = NUM_THREADS * NUM_PROC_XTHREAD; // submit the procedures - submitProcedures(16, 25, TestSingleStepProcedure.class); + submitProcedures(NUM_THREADS, NUM_PROC_XTHREAD, TestSingleStepProcedure.class); + + while (procEnv.getExecId() < NUM_PROCS) { + Thread.sleep(100); + } // restart the executor and allow the procedures to run - ProcedureTestingUtility.restart(procExecutor, new Runnable() { - @Override - public void run() { - procEnv.setAcquireLock(true); - } - }); + ProcedureTestingUtility.restart(procExecutor); // wait the execution of all the procedures and // assert that the execution order was sorted by procId ProcedureTestingUtility.waitNoProcedureRunning(procExecutor); - procEnv.assertSortedExecList(); - - // TODO: FIXME: This should be revisited + procEnv.assertSortedExecList(NUM_PROCS); } - @Ignore @Test(timeout=90000) - public void testMultiStepReplyOrder() throws Exception { - // avoid the procedure to be runnable - procEnv.setAcquireLock(false); + public void testMultiStepReplayOrder() throws Exception { + final int NUM_PROC_XTHREAD = 24; + final int NUM_PROCS = NUM_THREADS * (NUM_PROC_XTHREAD * 2); // submit the procedures - submitProcedures(16, 10, TestTwoStepProcedure.class); + submitProcedures(NUM_THREADS, NUM_PROC_XTHREAD, TestTwoStepProcedure.class); + + while (procEnv.getExecId() < NUM_PROCS) { + Thread.sleep(100); + } // restart the executor and allow the procedures to run - ProcedureTestingUtility.restart(procExecutor, new Runnable() { - @Override - public void run() { - procEnv.setAcquireLock(true); - } - }); + ProcedureTestingUtility.restart(procExecutor); - fail("TODO: FIXME: NOT IMPLEMENT REPLAY ORDER"); + // wait the execution of all the procedures and + // assert that the execution order was sorted by procId + ProcedureTestingUtility.waitNoProcedureRunning(procExecutor); + procEnv.assertSortedExecList(NUM_PROCS); } private void submitProcedures(final int nthreads, final int nprocPerThread, @@ -153,46 +155,38 @@ public class TestProcedureReplayOrder { } private static class TestProcedureEnv { - private ArrayList execList = new ArrayList(); - private boolean acquireLock = true; - - public void setAcquireLock(boolean acquireLock) { - this.acquireLock = acquireLock; - } + private ArrayList execList = new ArrayList(); + private AtomicLong execTimestamp = new AtomicLong(0); - public boolean canAcquireLock() { - return acquireLock; + public long getExecId() { + return execTimestamp.get(); } - public void addToExecList(final Procedure proc) { - execList.add(proc.getProcId()); + public long nextExecId() { + return execTimestamp.incrementAndGet(); } - public ArrayList getExecList() { - return execList; + public void addToExecList(final TestProcedure proc) { + execList.add(proc); } - public void assertSortedExecList() { + public void assertSortedExecList(int numProcs) { + assertEquals(numProcs, execList.size()); LOG.debug("EXEC LIST: " + execList); - for (int i = 1; i < execList.size(); ++i) { - assertTrue("exec list not sorted: " + execList.get(i-1) + " >= " + execList.get(i), - execList.get(i-1) < execList.get(i)); + for (int i = 0; i < execList.size() - 1; ++i) { + TestProcedure a = execList.get(i); + TestProcedure b = execList.get(i + 1); + assertTrue("exec list not sorted: " + a + " < " + b, a.getExecId() > b.getExecId()); } } } - public static class TestSingleStepProcedure extends SequentialProcedure { - public TestSingleStepProcedure() { } - - @Override - protected Procedure[] execute(TestProcedureEnv env) { - LOG.debug("execute procedure " + this); - env.addToExecList(this); - return null; - } + public static abstract class TestProcedure extends Procedure { + protected long execId = 0; + protected int step = 0; - protected boolean acquireLock(final TestProcedureEnv env) { - return env.canAcquireLock(); + public long getExecId() { + return execId; } @Override @@ -200,26 +194,62 @@ public class TestProcedureReplayOrder { @Override protected boolean abort(TestProcedureEnv env) { return true; } + + @Override + protected void serializeStateData(final OutputStream stream) throws IOException { + StreamUtils.writeLong(stream, execId); + } + + @Override + protected void deserializeStateData(final InputStream stream) throws IOException { + execId = StreamUtils.readLong(stream); + step = 2; + } } - public static class TestTwoStepProcedure extends SequentialProcedure { - public TestTwoStepProcedure() { } + public static class TestSingleStepProcedure extends TestProcedure { + public TestSingleStepProcedure() { } @Override - protected Procedure[] execute(TestProcedureEnv env) { - LOG.debug("execute procedure " + this); - env.addToExecList(this); - return new Procedure[] { new TestSingleStepProcedure() }; + protected Procedure[] execute(TestProcedureEnv env) throws ProcedureYieldException { + LOG.trace("execute procedure step=" + step + ": " + this); + if (step == 0) { + step = 1; + execId = env.nextExecId(); + return new Procedure[] { this }; + } else if (step == 2) { + env.addToExecList(this); + return null; + } + throw new ProcedureYieldException(); } - protected boolean acquireLock(final TestProcedureEnv env) { - return true; + @Override + public String toString() { + return "SingleStep(procId=" + getProcId() + " execId=" + execId + ")"; } + } + + public static class TestTwoStepProcedure extends TestProcedure { + public TestTwoStepProcedure() { } @Override - protected void rollback(TestProcedureEnv env) { } + protected Procedure[] execute(TestProcedureEnv env) throws ProcedureYieldException { + LOG.trace("execute procedure step=" + step + ": " + this); + if (step == 0) { + step = 1; + execId = env.nextExecId(); + return new Procedure[] { new TestSingleStepProcedure() }; + } else if (step == 2) { + env.addToExecList(this); + return null; + } + throw new ProcedureYieldException(); + } @Override - protected boolean abort(TestProcedureEnv env) { return true; } + public String toString() { + return "TwoStep(procId=" + getProcId() + " execId=" + execId + ")"; + } } } diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestYieldProcedures.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestYieldProcedures.java new file mode 100644 index 0000000..7ae76c4 --- /dev/null +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/TestYieldProcedures.java @@ -0,0 +1,285 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.procedure2; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseCommonTestingUtility; +import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; +import org.apache.hadoop.hbase.protobuf.generated.ProcedureProtos.ProcedureState; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; + +import org.junit.After; +import org.junit.Before; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@Category(SmallTests.class) +public class TestYieldProcedures { + private static final Log LOG = LogFactory.getLog(TestYieldProcedures.class); + + private static final int PROCEDURE_EXECUTOR_SLOTS = 1; + private static final Procedure NULL_PROC = null; + + private ProcedureExecutor procExecutor; + private ProcedureStore procStore; + + private HBaseCommonTestingUtility htu; + private FileSystem fs; + private Path testDir; + private Path logDir; + + @Before + public void setUp() throws IOException { + htu = new HBaseCommonTestingUtility(); + testDir = htu.getDataTestDir(); + fs = testDir.getFileSystem(htu.getConfiguration()); + assertTrue(testDir.depth() > 1); + + logDir = new Path(testDir, "proc-logs"); + procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), fs, logDir); + procExecutor = new ProcedureExecutor(htu.getConfiguration(), new TestProcEnv(), procStore); + procStore.start(PROCEDURE_EXECUTOR_SLOTS); + procExecutor.start(PROCEDURE_EXECUTOR_SLOTS, true); + } + + @After + public void tearDown() throws IOException { + procExecutor.stop(); + procStore.stop(false); + fs.delete(logDir, true); + } + + @Test + public void testYieldEachExecutionStep() throws Exception { + final int NUM_STATES = 3; + + TestStateMachineProcedure[] procs = new TestStateMachineProcedure[3]; + for (int i = 0; i < procs.length; ++i) { + procs[i] = new TestStateMachineProcedure(true, false); + procExecutor.submitProcedure(procs[i]); + } + ProcedureTestingUtility.waitNoProcedureRunning(procExecutor); + + // verify yield during execute() + long prevTimestamp = 0; + for (int execStep = 0; execStep < NUM_STATES; ++execStep) { + for (int i = 0; i < procs.length; ++i) { + assertEquals(NUM_STATES * 2, procs[i].getExecutionInfo().size()); + TestStateMachineProcedure.ExecutionInfo info = procs[i].getExecutionInfo().get(execStep); + LOG.info("i=" + i + " execStep=" + execStep + " timestamp=" + info.getTimestamp()); + assertEquals(false, info.isRollback()); + assertEquals(execStep, info.getStep().ordinal()); + assertEquals(prevTimestamp + 1, info.getTimestamp()); + prevTimestamp++; + } + } + + // verify yield during rollback() + int count = NUM_STATES; + for (int execStep = NUM_STATES - 1; execStep >= 0; --execStep) { + for (int i = 0; i < procs.length; ++i) { + assertEquals(NUM_STATES * 2, procs[i].getExecutionInfo().size()); + TestStateMachineProcedure.ExecutionInfo info = procs[i].getExecutionInfo().get(count); + LOG.info("i=" + i + " execStep=" + execStep + " timestamp=" + info.getTimestamp()); + assertEquals(true, info.isRollback()); + assertEquals(execStep, info.getStep().ordinal()); + assertEquals(prevTimestamp + 1, info.getTimestamp()); + prevTimestamp++; + } + count++; + } + } + + @Test + public void testYieldOnInterrupt() throws Exception { + final int NUM_STATES = 3; + int count = 0; + + TestStateMachineProcedure proc = new TestStateMachineProcedure(true, true); + ProcedureTestingUtility.submitAndWait(procExecutor, proc); + + // test execute (we execute steps twice, one has the IE the other completes) + assertEquals(NUM_STATES * 4, proc.getExecutionInfo().size()); + for (int i = 0; i < NUM_STATES; ++i) { + TestStateMachineProcedure.ExecutionInfo info = proc.getExecutionInfo().get(count++); + assertEquals(false, info.isRollback()); + assertEquals(i, info.getStep().ordinal()); + + info = proc.getExecutionInfo().get(count++); + assertEquals(false, info.isRollback()); + assertEquals(i, info.getStep().ordinal()); + } + + // test rollback (we execute steps twice, one has the IE the other completes) + for (int i = NUM_STATES - 1; i >= 0; --i) { + TestStateMachineProcedure.ExecutionInfo info = proc.getExecutionInfo().get(count++); + assertEquals(true, info.isRollback()); + assertEquals(i, info.getStep().ordinal()); + + info = proc.getExecutionInfo().get(count++); + assertEquals(true, info.isRollback()); + assertEquals(i, info.getStep().ordinal()); + } + } + + private static class TestProcEnv { + public final AtomicLong timestamp = new AtomicLong(0); + + public long nextTimestamp() { + return timestamp.incrementAndGet(); + } + } + + public static class TestStateMachineProcedure + extends StateMachineProcedure { + enum State { STATE_1, STATE_2, STATE_3 } + + public class ExecutionInfo { + private final boolean rollback; + private final long timestamp; + private final State step; + + public ExecutionInfo(long timestamp, State step, boolean isRollback) { + this.timestamp = timestamp; + this.step = step; + this.rollback = isRollback; + } + + public State getStep() { return step; } + public long getTimestamp() { return timestamp; } + public boolean isRollback() { return rollback; } + } + + private final ArrayList executionInfo = new ArrayList(); + private final AtomicBoolean aborted = new AtomicBoolean(false); + private final boolean throwInterruptOnceOnEachStep; + private final boolean abortOnFinalStep; + + public TestStateMachineProcedure() { + this(false, false); + } + + public TestStateMachineProcedure(boolean abortOnFinalStep, + boolean throwInterruptOnceOnEachStep) { + this.abortOnFinalStep = abortOnFinalStep; + this.throwInterruptOnceOnEachStep = throwInterruptOnceOnEachStep; + } + + public ArrayList getExecutionInfo() { + return executionInfo; + } + + @Override + protected StateMachineProcedure.Flow executeFromState(TestProcEnv env, State state) + throws InterruptedException { + LOG.info("execute step " + state); + executionInfo.add(new ExecutionInfo(env.nextTimestamp(), state, false)); + Thread.sleep(150); + + if (throwInterruptOnceOnEachStep && ((executionInfo.size() - 1) % 2) == 0) { + LOG.debug("THROW INTERRUPT"); + throw new InterruptedException("test interrupt"); + } + + switch (state) { + case STATE_1: + setNextState(State.STATE_2); + break; + case STATE_2: + setNextState(State.STATE_3); + break; + case STATE_3: + if (abortOnFinalStep) { + setFailure("test", new IOException("Requested abort on final step")); + } + return Flow.NO_MORE_STATE; + default: + throw new UnsupportedOperationException(); + } + return Flow.HAS_MORE_STATE; + } + + @Override + protected void rollbackState(TestProcEnv env, final State state) + throws InterruptedException { + LOG.debug("rollback state " + state); + executionInfo.add(new ExecutionInfo(env.nextTimestamp(), state, true)); + Thread.sleep(150); + + if (throwInterruptOnceOnEachStep && ((executionInfo.size() - 1) % 2) == 0) { + LOG.debug("THROW INTERRUPT"); + throw new InterruptedException("test interrupt"); + } + + switch (state) { + case STATE_1: + break; + case STATE_2: + break; + case STATE_3: + break; + default: + throw new UnsupportedOperationException(); + } + } + + @Override + protected State getState(final int stateId) { + return State.values()[stateId]; + } + + @Override + protected int getStateId(final State state) { + return state.ordinal(); + } + + @Override + protected State getInitialState() { + return State.STATE_1; + } + + @Override + protected boolean isYieldBeforeExecuteFromState(TestProcEnv env, State state) { + return true; + } + + @Override + protected boolean abort(TestProcEnv env) { + aborted.set(true); + return true; + } + } +} diff --git a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java index 1829d4b..19a9ea4 100644 --- a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java +++ b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java @@ -22,6 +22,10 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import java.util.Iterator; import java.util.HashSet; import java.util.Set; @@ -35,6 +39,8 @@ import org.apache.hadoop.hbase.HBaseCommonTestingUtility; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.procedure2.SequentialProcedure; +import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; +import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.IOUtils; @@ -83,17 +89,20 @@ public class TestWALProcedureStore { fs.delete(logDir, true); } - private Iterator storeRestart() throws Exception { + private void storeRestart(ProcedureStore.ProcedureLoader loader) throws Exception { procStore.stop(false); procStore.start(PROCEDURE_STORE_SLOTS); procStore.recoverLease(); - return procStore.load(); + procStore.load(loader); } @Test public void testEmptyLogLoad() throws Exception { - Iterator loader = storeRestart(); - assertEquals(0, countProcedures(loader)); + LoadCounter loader = new LoadCounter(); + storeRestart(loader); + assertEquals(0, loader.getMaxProcId()); + assertEquals(0, loader.getLoadedCount()); + assertEquals(0, loader.getCorruptedCount()); } @Test @@ -152,8 +161,10 @@ public class TestWALProcedureStore { assertEquals(1, logs.length); corruptLog(logs[0], 4); - int count = countProcedures(storeRestart()); - assertEquals(100, count); + LoadCounter loader = new LoadCounter(); + storeRestart(loader); + assertEquals(100, loader.getLoadedCount()); + assertEquals(0, loader.getCorruptedCount()); } @Test @@ -172,10 +183,205 @@ public class TestWALProcedureStore { assertEquals(1, logs.length); corruptLog(logs[0], 1823); - int count = countProcedures(storeRestart()); + LoadCounter loader = new LoadCounter(); + storeRestart(loader); assertTrue(procStore.getCorruptedLogs() != null); assertEquals(1, procStore.getCorruptedLogs().size()); - assertEquals(85, count); + assertEquals(85, loader.getLoadedCount()); + assertEquals(0, loader.getCorruptedCount()); + } + + @Test + public void testCorruptedProcedures() throws Exception { + // Insert root-procedures + TestProcedure[] rootProcs = new TestProcedure[10]; + for (int i = 1; i <= rootProcs.length; i++) { + rootProcs[i-1] = new TestProcedure(i, 0); + procStore.insert(rootProcs[i-1], null); + rootProcs[i-1].addStackId(0); + procStore.update(rootProcs[i-1]); + } + // insert root-child txn + procStore.rollWriter(); + for (int i = 1; i <= rootProcs.length; i++) { + TestProcedure b = new TestProcedure(rootProcs.length + i, i); + rootProcs[i-1].addStackId(1); + procStore.insert(rootProcs[i-1], new Procedure[] { b }); + } + // insert child updates + procStore.rollWriter(); + for (int i = 1; i <= rootProcs.length; i++) { + procStore.update(new TestProcedure(rootProcs.length + i, i)); + } + + // Stop the store + procStore.stop(false); + + // Remove 4 byte from the trailer + FileStatus[] logs = fs.listStatus(logDir); + assertEquals(3, logs.length); + Arrays.sort(logs, new Comparator() { + @Override + public int compare(FileStatus o1, FileStatus o2) { + return o1.getPath().getName().compareTo(o2.getPath().getName()); + } + }); + + // Remove the first log, we have insert-txn and updates in the others so everything is fine. + fs.delete(logs[0].getPath(), false); + LoadCounter loader = new LoadCounter(); + storeRestart(loader); + assertEquals(rootProcs.length * 2, loader.getLoadedCount()); + assertEquals(0, loader.getCorruptedCount()); + + // Remove the second log, we have lost any root/parent references + fs.delete(logs[1].getPath(), false); + loader.reset(); + storeRestart(loader); + assertEquals(0, loader.getLoadedCount()); + assertEquals(rootProcs.length, loader.getCorruptedCount()); + for (Procedure proc: loader.getCorrupted()) { + assertTrue(proc.toString(), proc.getParentProcId() <= rootProcs.length); + assertTrue(proc.toString(), + proc.getProcId() > rootProcs.length && + proc.getProcId() <= (rootProcs.length * 2)); + } + } + + @Test(timeout=60000) + public void testWalReplayOrder_AB_A() throws Exception { + /* + * | A B | -> | A | + */ + TestProcedure a = new TestProcedure(1, 0); + TestProcedure b = new TestProcedure(2, 1); + + procStore.insert(a, null); + a.addStackId(0); + procStore.update(a); + + procStore.insert(a, new Procedure[] { b }); + b.addStackId(1); + procStore.update(b); + + procStore.rollWriter(); + + a.addStackId(2); + procStore.update(a); + + storeRestart(new ProcedureStore.ProcedureLoader() { + @Override + public void setMaxProcId(long maxProcId) { + assertEquals(2, maxProcId); + } + + @Override + public void load(ProcedureIterator procIter) throws IOException { + assertTrue(procIter.hasNext()); + assertEquals(1, procIter.next().getProcId()); + assertTrue(procIter.hasNext()); + assertEquals(2, procIter.next().getProcId()); + assertFalse(procIter.hasNext()); + } + + @Override + public void handleCorrupted(ProcedureIterator procIter) throws IOException { + assertFalse(procIter.hasNext()); + } + }); + } + + @Test(timeout=60000) + public void testWalReplayOrder_ABC_BAD() throws Exception { + /* + * | A B C | -> | B A D | + */ + TestProcedure a = new TestProcedure(1, 0); + TestProcedure b = new TestProcedure(2, 1); + TestProcedure c = new TestProcedure(3, 2); + TestProcedure d = new TestProcedure(4, 0); + + procStore.insert(a, null); + a.addStackId(0); + procStore.update(a); + + procStore.insert(a, new Procedure[] { b }); + b.addStackId(1); + procStore.update(b); + + procStore.insert(b, new Procedure[] { c }); + b.addStackId(2); + procStore.update(b); + + procStore.rollWriter(); + + b.addStackId(3); + procStore.update(b); + + a.addStackId(4); + procStore.update(a); + + procStore.insert(d, null); + d.addStackId(0); + procStore.update(d); + + storeRestart(new ProcedureStore.ProcedureLoader() { + @Override + public void setMaxProcId(long maxProcId) { + assertEquals(4, maxProcId); + } + + @Override + public void load(ProcedureIterator procIter) throws IOException { + assertTrue(procIter.hasNext()); + assertEquals(4, procIter.next().getProcId()); + // TODO: This will be multiple call once we do fast-start + //assertFalse(procIter.hasNext()); + + assertTrue(procIter.hasNext()); + assertEquals(1, procIter.next().getProcId()); + assertTrue(procIter.hasNext()); + assertEquals(2, procIter.next().getProcId()); + assertTrue(procIter.hasNext()); + assertEquals(3, procIter.next().getProcId()); + assertFalse(procIter.hasNext()); + } + + @Override + public void handleCorrupted(ProcedureIterator procIter) throws IOException { + assertFalse(procIter.hasNext()); + } + }); + } + + public static class TestProcedure extends Procedure { + public TestProcedure() {} + + public TestProcedure(long procId, long parentId) { + setProcId(procId); + if (parentId > 0) { + setParentProcId(parentId); + } + } + + public void addStackId(final int index) { + addStackIndex(index); + } + + @Override + protected Procedure[] execute(Void env) { return null; } + + @Override + protected void rollback(Void env) { } + + @Override + protected boolean abort(Void env) { return false; } + + @Override + protected void serializeStateData(final OutputStream stream) throws IOException { } + + @Override + protected void deserializeStateData(final InputStream stream) throws IOException { } } private void corruptLog(final FileStatus logFile, final long dropBytes) @@ -191,29 +397,11 @@ public class TestWALProcedureStore { } private void verifyProcIdsOnRestart(final Set procIds) throws Exception { - int count = 0; - Iterator loader = storeRestart(); - while (loader.hasNext()) { - Procedure proc = loader.next(); - LOG.debug("loading procId=" + proc.getProcId()); - assertTrue("procId=" + proc.getProcId() + " unexpected", procIds.contains(proc.getProcId())); - count++; - } - assertEquals(procIds.size(), count); - } - - private void assertIsEmpty(Iterator iterator) { - assertEquals(0, countProcedures(iterator)); - } - - private int countProcedures(Iterator iterator) { - int count = 0; - while (iterator.hasNext()) { - Procedure proc = iterator.next(); - LOG.trace("loading procId=" + proc.getProcId()); - count++; - } - return count; + LOG.debug("expected: " + procIds); + LoadCounter loader = new LoadCounter(); + storeRestart(loader); + assertEquals(procIds.size(), loader.getLoadedCount()); + assertEquals(0, loader.getCorruptedCount()); } private void assertEmptyLogDir() { @@ -263,4 +451,78 @@ public class TestWALProcedureStore { } } } + + private class LoadCounter implements ProcedureStore.ProcedureLoader { + private final ArrayList corrupted = new ArrayList(); + private final ArrayList loaded = new ArrayList(); + + private Set procIds; + private long maxProcId = 0; + + public LoadCounter() { + this(null); + } + + public LoadCounter(final Set procIds) { + this.procIds = procIds; + } + + public void reset() { + reset(null); + } + + public void reset(final Set procIds) { + corrupted.clear(); + loaded.clear(); + this.procIds = procIds; + this.maxProcId = 0; + } + + public long getMaxProcId() { + return maxProcId; + } + + public ArrayList getLoaded() { + return loaded; + } + + public int getLoadedCount() { + return loaded.size(); + } + + public ArrayList getCorrupted() { + return corrupted; + } + + public int getCorruptedCount() { + return corrupted.size(); + } + + @Override + public void setMaxProcId(long maxProcId) { + maxProcId = maxProcId; + } + + @Override + public void load(ProcedureIterator procIter) throws IOException { + while (procIter.hasNext()) { + Procedure proc = procIter.next(); + LOG.debug("loading procId=" + proc.getProcId() + ": " + proc); + if (procIds != null) { + assertTrue("procId=" + proc.getProcId() + " unexpected", + procIds.contains(proc.getProcId())); + } + loaded.add(proc); + } + } + + @Override + public void handleCorrupted(ProcedureIterator procIter) throws IOException { + while (procIter.hasNext()) { + Procedure proc = procIter.next(); + LOG.debug("corrupted procId=" + proc.getProcId() + ": " + proc); + corrupted.add(proc); + } + } + } } diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProcedureProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProcedureProtos.java index e0a4775..0a44199 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProcedureProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProcedureProtos.java @@ -1070,6 +1070,160 @@ public final class MasterProcedureProtos { // @@protoc_insertion_point(enum_scope:DisableTableState) } + /** + * Protobuf enum {@code ServerCrashState} + */ + public enum ServerCrashState + implements com.google.protobuf.ProtocolMessageEnum { + /** + * SERVER_CRASH_START = 1; + */ + SERVER_CRASH_START(0, 1), + /** + * SERVER_CRASH_PROCESS_META = 2; + */ + SERVER_CRASH_PROCESS_META(1, 2), + /** + * SERVER_CRASH_GET_REGIONS = 3; + */ + SERVER_CRASH_GET_REGIONS(2, 3), + /** + * SERVER_CRASH_NO_SPLIT_LOGS = 4; + */ + SERVER_CRASH_NO_SPLIT_LOGS(3, 4), + /** + * SERVER_CRASH_SPLIT_LOGS = 5; + */ + SERVER_CRASH_SPLIT_LOGS(4, 5), + /** + * SERVER_CRASH_PREPARE_LOG_REPLAY = 6; + */ + SERVER_CRASH_PREPARE_LOG_REPLAY(5, 6), + /** + * SERVER_CRASH_CALC_REGIONS_TO_ASSIGN = 7; + */ + SERVER_CRASH_CALC_REGIONS_TO_ASSIGN(6, 7), + /** + * SERVER_CRASH_ASSIGN = 8; + */ + SERVER_CRASH_ASSIGN(7, 8), + /** + * SERVER_CRASH_WAIT_ON_ASSIGN = 9; + */ + SERVER_CRASH_WAIT_ON_ASSIGN(8, 9), + /** + * SERVER_CRASH_FINISH = 100; + */ + SERVER_CRASH_FINISH(9, 100), + ; + + /** + * SERVER_CRASH_START = 1; + */ + public static final int SERVER_CRASH_START_VALUE = 1; + /** + * SERVER_CRASH_PROCESS_META = 2; + */ + public static final int SERVER_CRASH_PROCESS_META_VALUE = 2; + /** + * SERVER_CRASH_GET_REGIONS = 3; + */ + public static final int SERVER_CRASH_GET_REGIONS_VALUE = 3; + /** + * SERVER_CRASH_NO_SPLIT_LOGS = 4; + */ + public static final int SERVER_CRASH_NO_SPLIT_LOGS_VALUE = 4; + /** + * SERVER_CRASH_SPLIT_LOGS = 5; + */ + public static final int SERVER_CRASH_SPLIT_LOGS_VALUE = 5; + /** + * SERVER_CRASH_PREPARE_LOG_REPLAY = 6; + */ + public static final int SERVER_CRASH_PREPARE_LOG_REPLAY_VALUE = 6; + /** + * SERVER_CRASH_CALC_REGIONS_TO_ASSIGN = 7; + */ + public static final int SERVER_CRASH_CALC_REGIONS_TO_ASSIGN_VALUE = 7; + /** + * SERVER_CRASH_ASSIGN = 8; + */ + public static final int SERVER_CRASH_ASSIGN_VALUE = 8; + /** + * SERVER_CRASH_WAIT_ON_ASSIGN = 9; + */ + public static final int SERVER_CRASH_WAIT_ON_ASSIGN_VALUE = 9; + /** + * SERVER_CRASH_FINISH = 100; + */ + public static final int SERVER_CRASH_FINISH_VALUE = 100; + + + public final int getNumber() { return value; } + + public static ServerCrashState valueOf(int value) { + switch (value) { + case 1: return SERVER_CRASH_START; + case 2: return SERVER_CRASH_PROCESS_META; + case 3: return SERVER_CRASH_GET_REGIONS; + case 4: return SERVER_CRASH_NO_SPLIT_LOGS; + case 5: return SERVER_CRASH_SPLIT_LOGS; + case 6: return SERVER_CRASH_PREPARE_LOG_REPLAY; + case 7: return SERVER_CRASH_CALC_REGIONS_TO_ASSIGN; + case 8: return SERVER_CRASH_ASSIGN; + case 9: return SERVER_CRASH_WAIT_ON_ASSIGN; + case 100: return SERVER_CRASH_FINISH; + default: return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static com.google.protobuf.Internal.EnumLiteMap + internalValueMap = + new com.google.protobuf.Internal.EnumLiteMap() { + public ServerCrashState findValueByNumber(int number) { + return ServerCrashState.valueOf(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor + getValueDescriptor() { + return getDescriptor().getValues().get(index); + } + public final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptorForType() { + return getDescriptor(); + } + public static final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.getDescriptor().getEnumTypes().get(9); + } + + private static final ServerCrashState[] VALUES = values(); + + public static ServerCrashState valueOf( + com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException( + "EnumValueDescriptor is not for this type."); + } + return VALUES[desc.getIndex()]; + } + + private final int index; + private final int value; + + private ServerCrashState(int index, int value) { + this.index = index; + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:ServerCrashState) + } + public interface CreateTableStateDataOrBuilder extends com.google.protobuf.MessageOrBuilder { @@ -11200,181 +11354,1791 @@ public final class MasterProcedureProtos { // @@protoc_insertion_point(class_scope:DisableTableStateData) } - private static com.google.protobuf.Descriptors.Descriptor - internal_static_CreateTableStateData_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_CreateTableStateData_fieldAccessorTable; - private static com.google.protobuf.Descriptors.Descriptor - internal_static_ModifyTableStateData_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_ModifyTableStateData_fieldAccessorTable; - private static com.google.protobuf.Descriptors.Descriptor - internal_static_TruncateTableStateData_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_TruncateTableStateData_fieldAccessorTable; - private static com.google.protobuf.Descriptors.Descriptor - internal_static_DeleteTableStateData_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_DeleteTableStateData_fieldAccessorTable; - private static com.google.protobuf.Descriptors.Descriptor - internal_static_AddColumnFamilyStateData_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_AddColumnFamilyStateData_fieldAccessorTable; - private static com.google.protobuf.Descriptors.Descriptor - internal_static_ModifyColumnFamilyStateData_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_ModifyColumnFamilyStateData_fieldAccessorTable; - private static com.google.protobuf.Descriptors.Descriptor - internal_static_DeleteColumnFamilyStateData_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_DeleteColumnFamilyStateData_fieldAccessorTable; - private static com.google.protobuf.Descriptors.Descriptor - internal_static_EnableTableStateData_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_EnableTableStateData_fieldAccessorTable; - private static com.google.protobuf.Descriptors.Descriptor - internal_static_DisableTableStateData_descriptor; - private static - com.google.protobuf.GeneratedMessage.FieldAccessorTable - internal_static_DisableTableStateData_fieldAccessorTable; + public interface ServerCrashStateDataOrBuilder + extends com.google.protobuf.MessageOrBuilder { - public static com.google.protobuf.Descriptors.FileDescriptor - getDescriptor() { - return descriptor; + // required .ServerName server_name = 1; + /** + * required .ServerName server_name = 1; + */ + boolean hasServerName(); + /** + * required .ServerName server_name = 1; + */ + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName getServerName(); + /** + * required .ServerName server_name = 1; + */ + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder getServerNameOrBuilder(); + + // optional bool distributed_log_replay = 2; + /** + * optional bool distributed_log_replay = 2; + */ + boolean hasDistributedLogReplay(); + /** + * optional bool distributed_log_replay = 2; + */ + boolean getDistributedLogReplay(); + + // repeated .RegionInfo regions_on_crashed_server = 3; + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + java.util.List + getRegionsOnCrashedServerList(); + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo getRegionsOnCrashedServer(int index); + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + int getRegionsOnCrashedServerCount(); + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + java.util.List + getRegionsOnCrashedServerOrBuilderList(); + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder getRegionsOnCrashedServerOrBuilder( + int index); + + // repeated .RegionInfo regions_to_assign = 4; + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + java.util.List + getRegionsToAssignList(); + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo getRegionsToAssign(int index); + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + int getRegionsToAssignCount(); + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + java.util.List + getRegionsToAssignOrBuilderList(); + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder getRegionsToAssignOrBuilder( + int index); + + // optional bool carrying_meta = 5; + /** + * optional bool carrying_meta = 5; + */ + boolean hasCarryingMeta(); + /** + * optional bool carrying_meta = 5; + */ + boolean getCarryingMeta(); + + // optional bool should_split_wal = 6 [default = true]; + /** + * optional bool should_split_wal = 6 [default = true]; + */ + boolean hasShouldSplitWal(); + /** + * optional bool should_split_wal = 6 [default = true]; + */ + boolean getShouldSplitWal(); } - private static com.google.protobuf.Descriptors.FileDescriptor - descriptor; - static { - java.lang.String[] descriptorData = { - "\n\025MasterProcedure.proto\032\013HBase.proto\032\tRP" + - "C.proto\"\201\001\n\024CreateTableStateData\022#\n\tuser" + - "_info\030\001 \002(\0132\020.UserInformation\022\"\n\014table_s" + - "chema\030\002 \002(\0132\014.TableSchema\022 \n\013region_info" + - "\030\003 \003(\0132\013.RegionInfo\"\277\001\n\024ModifyTableState" + - "Data\022#\n\tuser_info\030\001 \002(\0132\020.UserInformatio" + - "n\022-\n\027unmodified_table_schema\030\002 \001(\0132\014.Tab" + - "leSchema\022+\n\025modified_table_schema\030\003 \002(\0132" + - "\014.TableSchema\022&\n\036delete_column_family_in" + - "_modify\030\004 \002(\010\"\274\001\n\026TruncateTableStateData", - "\022#\n\tuser_info\030\001 \002(\0132\020.UserInformation\022\027\n" + - "\017preserve_splits\030\002 \002(\010\022\036\n\ntable_name\030\003 \001" + - "(\0132\n.TableName\022\"\n\014table_schema\030\004 \001(\0132\014.T" + - "ableSchema\022 \n\013region_info\030\005 \003(\0132\013.Region" + - "Info\"}\n\024DeleteTableStateData\022#\n\tuser_inf" + - "o\030\001 \002(\0132\020.UserInformation\022\036\n\ntable_name\030" + - "\002 \002(\0132\n.TableName\022 \n\013region_info\030\003 \003(\0132\013" + - ".RegionInfo\"\300\001\n\030AddColumnFamilyStateData" + - "\022#\n\tuser_info\030\001 \002(\0132\020.UserInformation\022\036\n" + - "\ntable_name\030\002 \002(\0132\n.TableName\0220\n\023columnf", - "amily_schema\030\003 \002(\0132\023.ColumnFamilySchema\022" + - "-\n\027unmodified_table_schema\030\004 \001(\0132\014.Table" + - "Schema\"\303\001\n\033ModifyColumnFamilyStateData\022#" + - "\n\tuser_info\030\001 \002(\0132\020.UserInformation\022\036\n\nt" + - "able_name\030\002 \002(\0132\n.TableName\0220\n\023columnfam" + - "ily_schema\030\003 \002(\0132\023.ColumnFamilySchema\022-\n" + - "\027unmodified_table_schema\030\004 \001(\0132\014.TableSc" + - "hema\"\254\001\n\033DeleteColumnFamilyStateData\022#\n\t" + - "user_info\030\001 \002(\0132\020.UserInformation\022\036\n\ntab" + - "le_name\030\002 \002(\0132\n.TableName\022\031\n\021columnfamil", - "y_name\030\003 \002(\014\022-\n\027unmodified_table_schema\030" + - "\004 \001(\0132\014.TableSchema\"{\n\024EnableTableStateD" + - "ata\022#\n\tuser_info\030\001 \002(\0132\020.UserInformation" + - "\022\036\n\ntable_name\030\002 \002(\0132\n.TableName\022\036\n\026skip" + - "_table_state_check\030\003 \002(\010\"|\n\025DisableTable" + - "StateData\022#\n\tuser_info\030\001 \002(\0132\020.UserInfor" + - "mation\022\036\n\ntable_name\030\002 \002(\0132\n.TableName\022\036" + - "\n\026skip_table_state_check\030\003 \002(\010*\330\001\n\020Creat" + - "eTableState\022\036\n\032CREATE_TABLE_PRE_OPERATIO" + - "N\020\001\022 \n\034CREATE_TABLE_WRITE_FS_LAYOUT\020\002\022\034\n", - "\030CREATE_TABLE_ADD_TO_META\020\003\022\037\n\033CREATE_TA" + - "BLE_ASSIGN_REGIONS\020\004\022\"\n\036CREATE_TABLE_UPD" + - "ATE_DESC_CACHE\020\005\022\037\n\033CREATE_TABLE_POST_OP" + - "ERATION\020\006*\207\002\n\020ModifyTableState\022\030\n\024MODIFY" + - "_TABLE_PREPARE\020\001\022\036\n\032MODIFY_TABLE_PRE_OPE" + - "RATION\020\002\022(\n$MODIFY_TABLE_UPDATE_TABLE_DE" + - "SCRIPTOR\020\003\022&\n\"MODIFY_TABLE_REMOVE_REPLIC" + - "A_COLUMN\020\004\022!\n\035MODIFY_TABLE_DELETE_FS_LAY" + - "OUT\020\005\022\037\n\033MODIFY_TABLE_POST_OPERATION\020\006\022#" + - "\n\037MODIFY_TABLE_REOPEN_ALL_REGIONS\020\007*\212\002\n\022", - "TruncateTableState\022 \n\034TRUNCATE_TABLE_PRE" + - "_OPERATION\020\001\022#\n\037TRUNCATE_TABLE_REMOVE_FR" + - "OM_META\020\002\022\"\n\036TRUNCATE_TABLE_CLEAR_FS_LAY" + - "OUT\020\003\022#\n\037TRUNCATE_TABLE_CREATE_FS_LAYOUT" + - "\020\004\022\036\n\032TRUNCATE_TABLE_ADD_TO_META\020\005\022!\n\035TR" + - "UNCATE_TABLE_ASSIGN_REGIONS\020\006\022!\n\035TRUNCAT" + - "E_TABLE_POST_OPERATION\020\007*\337\001\n\020DeleteTable" + - "State\022\036\n\032DELETE_TABLE_PRE_OPERATION\020\001\022!\n" + - "\035DELETE_TABLE_REMOVE_FROM_META\020\002\022 \n\034DELE" + - "TE_TABLE_CLEAR_FS_LAYOUT\020\003\022\"\n\036DELETE_TAB", - "LE_UPDATE_DESC_CACHE\020\004\022!\n\035DELETE_TABLE_U" + - "NASSIGN_REGIONS\020\005\022\037\n\033DELETE_TABLE_POST_O" + - "PERATION\020\006*\331\001\n\024AddColumnFamilyState\022\035\n\031A" + - "DD_COLUMN_FAMILY_PREPARE\020\001\022#\n\037ADD_COLUMN" + - "_FAMILY_PRE_OPERATION\020\002\022-\n)ADD_COLUMN_FA" + - "MILY_UPDATE_TABLE_DESCRIPTOR\020\003\022$\n ADD_CO" + - "LUMN_FAMILY_POST_OPERATION\020\004\022(\n$ADD_COLU" + - "MN_FAMILY_REOPEN_ALL_REGIONS\020\005*\353\001\n\027Modif" + - "yColumnFamilyState\022 \n\034MODIFY_COLUMN_FAMI" + - "LY_PREPARE\020\001\022&\n\"MODIFY_COLUMN_FAMILY_PRE", - "_OPERATION\020\002\0220\n,MODIFY_COLUMN_FAMILY_UPD" + - "ATE_TABLE_DESCRIPTOR\020\003\022\'\n#MODIFY_COLUMN_" + - "FAMILY_POST_OPERATION\020\004\022+\n\'MODIFY_COLUMN" + - "_FAMILY_REOPEN_ALL_REGIONS\020\005*\226\002\n\027DeleteC" + - "olumnFamilyState\022 \n\034DELETE_COLUMN_FAMILY" + - "_PREPARE\020\001\022&\n\"DELETE_COLUMN_FAMILY_PRE_O" + - "PERATION\020\002\0220\n,DELETE_COLUMN_FAMILY_UPDAT" + - "E_TABLE_DESCRIPTOR\020\003\022)\n%DELETE_COLUMN_FA" + - "MILY_DELETE_FS_LAYOUT\020\004\022\'\n#DELETE_COLUMN" + - "_FAMILY_POST_OPERATION\020\005\022+\n\'DELETE_COLUM", - "N_FAMILY_REOPEN_ALL_REGIONS\020\006*\350\001\n\020Enable" + - "TableState\022\030\n\024ENABLE_TABLE_PREPARE\020\001\022\036\n\032" + - "ENABLE_TABLE_PRE_OPERATION\020\002\022)\n%ENABLE_T" + - "ABLE_SET_ENABLING_TABLE_STATE\020\003\022$\n ENABL" + - "E_TABLE_MARK_REGIONS_ONLINE\020\004\022(\n$ENABLE_" + - "TABLE_SET_ENABLED_TABLE_STATE\020\005\022\037\n\033ENABL" + - "E_TABLE_POST_OPERATION\020\006*\362\001\n\021DisableTabl" + - "eState\022\031\n\025DISABLE_TABLE_PREPARE\020\001\022\037\n\033DIS" + - "ABLE_TABLE_PRE_OPERATION\020\002\022+\n\'DISABLE_TA" + - "BLE_SET_DISABLING_TABLE_STATE\020\003\022&\n\"DISAB", - "LE_TABLE_MARK_REGIONS_OFFLINE\020\004\022*\n&DISAB" + - "LE_TABLE_SET_DISABLED_TABLE_STATE\020\005\022 \n\034D" + - "ISABLE_TABLE_POST_OPERATION\020\006BK\n*org.apa" + - "che.hadoop.hbase.protobuf.generatedB\025Mas" + - "terProcedureProtosH\001\210\001\001\240\001\001" - }; - com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = - new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { - public com.google.protobuf.ExtensionRegistry assignDescriptors( - com.google.protobuf.Descriptors.FileDescriptor root) { - descriptor = root; - internal_static_CreateTableStateData_descriptor = - getDescriptor().getMessageTypes().get(0); - internal_static_CreateTableStateData_fieldAccessorTable = new - com.google.protobuf.GeneratedMessage.FieldAccessorTable( - internal_static_CreateTableStateData_descriptor, - new java.lang.String[] { "UserInfo", "TableSchema", "RegionInfo", }); - internal_static_ModifyTableStateData_descriptor = - getDescriptor().getMessageTypes().get(1); - internal_static_ModifyTableStateData_fieldAccessorTable = new - com.google.protobuf.GeneratedMessage.FieldAccessorTable( - internal_static_ModifyTableStateData_descriptor, - new java.lang.String[] { "UserInfo", "UnmodifiedTableSchema", "ModifiedTableSchema", "DeleteColumnFamilyInModify", }); - internal_static_TruncateTableStateData_descriptor = - getDescriptor().getMessageTypes().get(2); - internal_static_TruncateTableStateData_fieldAccessorTable = new - com.google.protobuf.GeneratedMessage.FieldAccessorTable( - internal_static_TruncateTableStateData_descriptor, - new java.lang.String[] { "UserInfo", "PreserveSplits", "TableName", "TableSchema", "RegionInfo", }); - internal_static_DeleteTableStateData_descriptor = - getDescriptor().getMessageTypes().get(3); + /** + * Protobuf type {@code ServerCrashStateData} + */ + public static final class ServerCrashStateData extends + com.google.protobuf.GeneratedMessage + implements ServerCrashStateDataOrBuilder { + // Use ServerCrashStateData.newBuilder() to construct. + private ServerCrashStateData(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private ServerCrashStateData(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final ServerCrashStateData defaultInstance; + public static ServerCrashStateData getDefaultInstance() { + return defaultInstance; + } + + public ServerCrashStateData getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private ServerCrashStateData( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder subBuilder = null; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + subBuilder = serverName_.toBuilder(); + } + serverName_ = input.readMessage(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(serverName_); + serverName_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000001; + break; + } + case 16: { + bitField0_ |= 0x00000002; + distributedLogReplay_ = input.readBool(); + break; + } + case 26: { + if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + regionsOnCrashedServer_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000004; + } + regionsOnCrashedServer_.add(input.readMessage(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.PARSER, extensionRegistry)); + break; + } + case 34: { + if (!((mutable_bitField0_ & 0x00000008) == 0x00000008)) { + regionsToAssign_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000008; + } + regionsToAssign_.add(input.readMessage(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.PARSER, extensionRegistry)); + break; + } + case 40: { + bitField0_ |= 0x00000004; + carryingMeta_ = input.readBool(); + break; + } + case 48: { + bitField0_ |= 0x00000008; + shouldSplitWal_ = input.readBool(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + regionsOnCrashedServer_ = java.util.Collections.unmodifiableList(regionsOnCrashedServer_); + } + if (((mutable_bitField0_ & 0x00000008) == 0x00000008)) { + regionsToAssign_ = java.util.Collections.unmodifiableList(regionsToAssign_); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.internal_static_ServerCrashStateData_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.internal_static_ServerCrashStateData_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData.class, org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public ServerCrashStateData parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new ServerCrashStateData(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + private int bitField0_; + // required .ServerName server_name = 1; + public static final int SERVER_NAME_FIELD_NUMBER = 1; + private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName serverName_; + /** + * required .ServerName server_name = 1; + */ + public boolean hasServerName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required .ServerName server_name = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName getServerName() { + return serverName_; + } + /** + * required .ServerName server_name = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder getServerNameOrBuilder() { + return serverName_; + } + + // optional bool distributed_log_replay = 2; + public static final int DISTRIBUTED_LOG_REPLAY_FIELD_NUMBER = 2; + private boolean distributedLogReplay_; + /** + * optional bool distributed_log_replay = 2; + */ + public boolean hasDistributedLogReplay() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + /** + * optional bool distributed_log_replay = 2; + */ + public boolean getDistributedLogReplay() { + return distributedLogReplay_; + } + + // repeated .RegionInfo regions_on_crashed_server = 3; + public static final int REGIONS_ON_CRASHED_SERVER_FIELD_NUMBER = 3; + private java.util.List regionsOnCrashedServer_; + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public java.util.List getRegionsOnCrashedServerList() { + return regionsOnCrashedServer_; + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public java.util.List + getRegionsOnCrashedServerOrBuilderList() { + return regionsOnCrashedServer_; + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public int getRegionsOnCrashedServerCount() { + return regionsOnCrashedServer_.size(); + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo getRegionsOnCrashedServer(int index) { + return regionsOnCrashedServer_.get(index); + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder getRegionsOnCrashedServerOrBuilder( + int index) { + return regionsOnCrashedServer_.get(index); + } + + // repeated .RegionInfo regions_to_assign = 4; + public static final int REGIONS_TO_ASSIGN_FIELD_NUMBER = 4; + private java.util.List regionsToAssign_; + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public java.util.List getRegionsToAssignList() { + return regionsToAssign_; + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public java.util.List + getRegionsToAssignOrBuilderList() { + return regionsToAssign_; + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public int getRegionsToAssignCount() { + return regionsToAssign_.size(); + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo getRegionsToAssign(int index) { + return regionsToAssign_.get(index); + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder getRegionsToAssignOrBuilder( + int index) { + return regionsToAssign_.get(index); + } + + // optional bool carrying_meta = 5; + public static final int CARRYING_META_FIELD_NUMBER = 5; + private boolean carryingMeta_; + /** + * optional bool carrying_meta = 5; + */ + public boolean hasCarryingMeta() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + /** + * optional bool carrying_meta = 5; + */ + public boolean getCarryingMeta() { + return carryingMeta_; + } + + // optional bool should_split_wal = 6 [default = true]; + public static final int SHOULD_SPLIT_WAL_FIELD_NUMBER = 6; + private boolean shouldSplitWal_; + /** + * optional bool should_split_wal = 6 [default = true]; + */ + public boolean hasShouldSplitWal() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + /** + * optional bool should_split_wal = 6 [default = true]; + */ + public boolean getShouldSplitWal() { + return shouldSplitWal_; + } + + private void initFields() { + serverName_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.getDefaultInstance(); + distributedLogReplay_ = false; + regionsOnCrashedServer_ = java.util.Collections.emptyList(); + regionsToAssign_ = java.util.Collections.emptyList(); + carryingMeta_ = false; + shouldSplitWal_ = true; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasServerName()) { + memoizedIsInitialized = 0; + return false; + } + if (!getServerName().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + for (int i = 0; i < getRegionsOnCrashedServerCount(); i++) { + if (!getRegionsOnCrashedServer(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + for (int i = 0; i < getRegionsToAssignCount(); i++) { + if (!getRegionsToAssign(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeMessage(1, serverName_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBool(2, distributedLogReplay_); + } + for (int i = 0; i < regionsOnCrashedServer_.size(); i++) { + output.writeMessage(3, regionsOnCrashedServer_.get(i)); + } + for (int i = 0; i < regionsToAssign_.size(); i++) { + output.writeMessage(4, regionsToAssign_.get(i)); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeBool(5, carryingMeta_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeBool(6, shouldSplitWal_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, serverName_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(2, distributedLogReplay_); + } + for (int i = 0; i < regionsOnCrashedServer_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(3, regionsOnCrashedServer_.get(i)); + } + for (int i = 0; i < regionsToAssign_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(4, regionsToAssign_.get(i)); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(5, carryingMeta_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(6, shouldSplitWal_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData)) { + return super.equals(obj); + } + org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData other = (org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData) obj; + + boolean result = true; + result = result && (hasServerName() == other.hasServerName()); + if (hasServerName()) { + result = result && getServerName() + .equals(other.getServerName()); + } + result = result && (hasDistributedLogReplay() == other.hasDistributedLogReplay()); + if (hasDistributedLogReplay()) { + result = result && (getDistributedLogReplay() + == other.getDistributedLogReplay()); + } + result = result && getRegionsOnCrashedServerList() + .equals(other.getRegionsOnCrashedServerList()); + result = result && getRegionsToAssignList() + .equals(other.getRegionsToAssignList()); + result = result && (hasCarryingMeta() == other.hasCarryingMeta()); + if (hasCarryingMeta()) { + result = result && (getCarryingMeta() + == other.getCarryingMeta()); + } + result = result && (hasShouldSplitWal() == other.hasShouldSplitWal()); + if (hasShouldSplitWal()) { + result = result && (getShouldSplitWal() + == other.getShouldSplitWal()); + } + result = result && + getUnknownFields().equals(other.getUnknownFields()); + return result; + } + + private int memoizedHashCode = 0; + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptorForType().hashCode(); + if (hasServerName()) { + hash = (37 * hash) + SERVER_NAME_FIELD_NUMBER; + hash = (53 * hash) + getServerName().hashCode(); + } + if (hasDistributedLogReplay()) { + hash = (37 * hash) + DISTRIBUTED_LOG_REPLAY_FIELD_NUMBER; + hash = (53 * hash) + hashBoolean(getDistributedLogReplay()); + } + if (getRegionsOnCrashedServerCount() > 0) { + hash = (37 * hash) + REGIONS_ON_CRASHED_SERVER_FIELD_NUMBER; + hash = (53 * hash) + getRegionsOnCrashedServerList().hashCode(); + } + if (getRegionsToAssignCount() > 0) { + hash = (37 * hash) + REGIONS_TO_ASSIGN_FIELD_NUMBER; + hash = (53 * hash) + getRegionsToAssignList().hashCode(); + } + if (hasCarryingMeta()) { + hash = (37 * hash) + CARRYING_META_FIELD_NUMBER; + hash = (53 * hash) + hashBoolean(getCarryingMeta()); + } + if (hasShouldSplitWal()) { + hash = (37 * hash) + SHOULD_SPLIT_WAL_FIELD_NUMBER; + hash = (53 * hash) + hashBoolean(getShouldSplitWal()); + } + hash = (29 * hash) + getUnknownFields().hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code ServerCrashStateData} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateDataOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.internal_static_ServerCrashStateData_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.internal_static_ServerCrashStateData_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData.class, org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData.Builder.class); + } + + // Construct using org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getServerNameFieldBuilder(); + getRegionsOnCrashedServerFieldBuilder(); + getRegionsToAssignFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + if (serverNameBuilder_ == null) { + serverName_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.getDefaultInstance(); + } else { + serverNameBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000001); + distributedLogReplay_ = false; + bitField0_ = (bitField0_ & ~0x00000002); + if (regionsOnCrashedServerBuilder_ == null) { + regionsOnCrashedServer_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000004); + } else { + regionsOnCrashedServerBuilder_.clear(); + } + if (regionsToAssignBuilder_ == null) { + regionsToAssign_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000008); + } else { + regionsToAssignBuilder_.clear(); + } + carryingMeta_ = false; + bitField0_ = (bitField0_ & ~0x00000010); + shouldSplitWal_ = true; + bitField0_ = (bitField0_ & ~0x00000020); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.internal_static_ServerCrashStateData_descriptor; + } + + public org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData getDefaultInstanceForType() { + return org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData build() { + org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData buildPartial() { + org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData result = new org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + if (serverNameBuilder_ == null) { + result.serverName_ = serverName_; + } else { + result.serverName_ = serverNameBuilder_.build(); + } + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.distributedLogReplay_ = distributedLogReplay_; + if (regionsOnCrashedServerBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004)) { + regionsOnCrashedServer_ = java.util.Collections.unmodifiableList(regionsOnCrashedServer_); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.regionsOnCrashedServer_ = regionsOnCrashedServer_; + } else { + result.regionsOnCrashedServer_ = regionsOnCrashedServerBuilder_.build(); + } + if (regionsToAssignBuilder_ == null) { + if (((bitField0_ & 0x00000008) == 0x00000008)) { + regionsToAssign_ = java.util.Collections.unmodifiableList(regionsToAssign_); + bitField0_ = (bitField0_ & ~0x00000008); + } + result.regionsToAssign_ = regionsToAssign_; + } else { + result.regionsToAssign_ = regionsToAssignBuilder_.build(); + } + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000004; + } + result.carryingMeta_ = carryingMeta_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000008; + } + result.shouldSplitWal_ = shouldSplitWal_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData) { + return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData other) { + if (other == org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData.getDefaultInstance()) return this; + if (other.hasServerName()) { + mergeServerName(other.getServerName()); + } + if (other.hasDistributedLogReplay()) { + setDistributedLogReplay(other.getDistributedLogReplay()); + } + if (regionsOnCrashedServerBuilder_ == null) { + if (!other.regionsOnCrashedServer_.isEmpty()) { + if (regionsOnCrashedServer_.isEmpty()) { + regionsOnCrashedServer_ = other.regionsOnCrashedServer_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensureRegionsOnCrashedServerIsMutable(); + regionsOnCrashedServer_.addAll(other.regionsOnCrashedServer_); + } + onChanged(); + } + } else { + if (!other.regionsOnCrashedServer_.isEmpty()) { + if (regionsOnCrashedServerBuilder_.isEmpty()) { + regionsOnCrashedServerBuilder_.dispose(); + regionsOnCrashedServerBuilder_ = null; + regionsOnCrashedServer_ = other.regionsOnCrashedServer_; + bitField0_ = (bitField0_ & ~0x00000004); + regionsOnCrashedServerBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getRegionsOnCrashedServerFieldBuilder() : null; + } else { + regionsOnCrashedServerBuilder_.addAllMessages(other.regionsOnCrashedServer_); + } + } + } + if (regionsToAssignBuilder_ == null) { + if (!other.regionsToAssign_.isEmpty()) { + if (regionsToAssign_.isEmpty()) { + regionsToAssign_ = other.regionsToAssign_; + bitField0_ = (bitField0_ & ~0x00000008); + } else { + ensureRegionsToAssignIsMutable(); + regionsToAssign_.addAll(other.regionsToAssign_); + } + onChanged(); + } + } else { + if (!other.regionsToAssign_.isEmpty()) { + if (regionsToAssignBuilder_.isEmpty()) { + regionsToAssignBuilder_.dispose(); + regionsToAssignBuilder_ = null; + regionsToAssign_ = other.regionsToAssign_; + bitField0_ = (bitField0_ & ~0x00000008); + regionsToAssignBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getRegionsToAssignFieldBuilder() : null; + } else { + regionsToAssignBuilder_.addAllMessages(other.regionsToAssign_); + } + } + } + if (other.hasCarryingMeta()) { + setCarryingMeta(other.getCarryingMeta()); + } + if (other.hasShouldSplitWal()) { + setShouldSplitWal(other.getShouldSplitWal()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasServerName()) { + + return false; + } + if (!getServerName().isInitialized()) { + + return false; + } + for (int i = 0; i < getRegionsOnCrashedServerCount(); i++) { + if (!getRegionsOnCrashedServer(i).isInitialized()) { + + return false; + } + } + for (int i = 0; i < getRegionsToAssignCount(); i++) { + if (!getRegionsToAssign(i).isInitialized()) { + + return false; + } + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashStateData) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // required .ServerName server_name = 1; + private org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName serverName_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.getDefaultInstance(); + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder> serverNameBuilder_; + /** + * required .ServerName server_name = 1; + */ + public boolean hasServerName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required .ServerName server_name = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName getServerName() { + if (serverNameBuilder_ == null) { + return serverName_; + } else { + return serverNameBuilder_.getMessage(); + } + } + /** + * required .ServerName server_name = 1; + */ + public Builder setServerName(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName value) { + if (serverNameBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + serverName_ = value; + onChanged(); + } else { + serverNameBuilder_.setMessage(value); + } + bitField0_ |= 0x00000001; + return this; + } + /** + * required .ServerName server_name = 1; + */ + public Builder setServerName( + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder builderForValue) { + if (serverNameBuilder_ == null) { + serverName_ = builderForValue.build(); + onChanged(); + } else { + serverNameBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000001; + return this; + } + /** + * required .ServerName server_name = 1; + */ + public Builder mergeServerName(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName value) { + if (serverNameBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001) && + serverName_ != org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.getDefaultInstance()) { + serverName_ = + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.newBuilder(serverName_).mergeFrom(value).buildPartial(); + } else { + serverName_ = value; + } + onChanged(); + } else { + serverNameBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000001; + return this; + } + /** + * required .ServerName server_name = 1; + */ + public Builder clearServerName() { + if (serverNameBuilder_ == null) { + serverName_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.getDefaultInstance(); + onChanged(); + } else { + serverNameBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + /** + * required .ServerName server_name = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder getServerNameBuilder() { + bitField0_ |= 0x00000001; + onChanged(); + return getServerNameFieldBuilder().getBuilder(); + } + /** + * required .ServerName server_name = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder getServerNameOrBuilder() { + if (serverNameBuilder_ != null) { + return serverNameBuilder_.getMessageOrBuilder(); + } else { + return serverName_; + } + } + /** + * required .ServerName server_name = 1; + */ + private com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder> + getServerNameFieldBuilder() { + if (serverNameBuilder_ == null) { + serverNameBuilder_ = new com.google.protobuf.SingleFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder>( + serverName_, + getParentForChildren(), + isClean()); + serverName_ = null; + } + return serverNameBuilder_; + } + + // optional bool distributed_log_replay = 2; + private boolean distributedLogReplay_ ; + /** + * optional bool distributed_log_replay = 2; + */ + public boolean hasDistributedLogReplay() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + /** + * optional bool distributed_log_replay = 2; + */ + public boolean getDistributedLogReplay() { + return distributedLogReplay_; + } + /** + * optional bool distributed_log_replay = 2; + */ + public Builder setDistributedLogReplay(boolean value) { + bitField0_ |= 0x00000002; + distributedLogReplay_ = value; + onChanged(); + return this; + } + /** + * optional bool distributed_log_replay = 2; + */ + public Builder clearDistributedLogReplay() { + bitField0_ = (bitField0_ & ~0x00000002); + distributedLogReplay_ = false; + onChanged(); + return this; + } + + // repeated .RegionInfo regions_on_crashed_server = 3; + private java.util.List regionsOnCrashedServer_ = + java.util.Collections.emptyList(); + private void ensureRegionsOnCrashedServerIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + regionsOnCrashedServer_ = new java.util.ArrayList(regionsOnCrashedServer_); + bitField0_ |= 0x00000004; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder> regionsOnCrashedServerBuilder_; + + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public java.util.List getRegionsOnCrashedServerList() { + if (regionsOnCrashedServerBuilder_ == null) { + return java.util.Collections.unmodifiableList(regionsOnCrashedServer_); + } else { + return regionsOnCrashedServerBuilder_.getMessageList(); + } + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public int getRegionsOnCrashedServerCount() { + if (regionsOnCrashedServerBuilder_ == null) { + return regionsOnCrashedServer_.size(); + } else { + return regionsOnCrashedServerBuilder_.getCount(); + } + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo getRegionsOnCrashedServer(int index) { + if (regionsOnCrashedServerBuilder_ == null) { + return regionsOnCrashedServer_.get(index); + } else { + return regionsOnCrashedServerBuilder_.getMessage(index); + } + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public Builder setRegionsOnCrashedServer( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo value) { + if (regionsOnCrashedServerBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRegionsOnCrashedServerIsMutable(); + regionsOnCrashedServer_.set(index, value); + onChanged(); + } else { + regionsOnCrashedServerBuilder_.setMessage(index, value); + } + return this; + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public Builder setRegionsOnCrashedServer( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder builderForValue) { + if (regionsOnCrashedServerBuilder_ == null) { + ensureRegionsOnCrashedServerIsMutable(); + regionsOnCrashedServer_.set(index, builderForValue.build()); + onChanged(); + } else { + regionsOnCrashedServerBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public Builder addRegionsOnCrashedServer(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo value) { + if (regionsOnCrashedServerBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRegionsOnCrashedServerIsMutable(); + regionsOnCrashedServer_.add(value); + onChanged(); + } else { + regionsOnCrashedServerBuilder_.addMessage(value); + } + return this; + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public Builder addRegionsOnCrashedServer( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo value) { + if (regionsOnCrashedServerBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRegionsOnCrashedServerIsMutable(); + regionsOnCrashedServer_.add(index, value); + onChanged(); + } else { + regionsOnCrashedServerBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public Builder addRegionsOnCrashedServer( + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder builderForValue) { + if (regionsOnCrashedServerBuilder_ == null) { + ensureRegionsOnCrashedServerIsMutable(); + regionsOnCrashedServer_.add(builderForValue.build()); + onChanged(); + } else { + regionsOnCrashedServerBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public Builder addRegionsOnCrashedServer( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder builderForValue) { + if (regionsOnCrashedServerBuilder_ == null) { + ensureRegionsOnCrashedServerIsMutable(); + regionsOnCrashedServer_.add(index, builderForValue.build()); + onChanged(); + } else { + regionsOnCrashedServerBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public Builder addAllRegionsOnCrashedServer( + java.lang.Iterable values) { + if (regionsOnCrashedServerBuilder_ == null) { + ensureRegionsOnCrashedServerIsMutable(); + super.addAll(values, regionsOnCrashedServer_); + onChanged(); + } else { + regionsOnCrashedServerBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public Builder clearRegionsOnCrashedServer() { + if (regionsOnCrashedServerBuilder_ == null) { + regionsOnCrashedServer_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + } else { + regionsOnCrashedServerBuilder_.clear(); + } + return this; + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public Builder removeRegionsOnCrashedServer(int index) { + if (regionsOnCrashedServerBuilder_ == null) { + ensureRegionsOnCrashedServerIsMutable(); + regionsOnCrashedServer_.remove(index); + onChanged(); + } else { + regionsOnCrashedServerBuilder_.remove(index); + } + return this; + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder getRegionsOnCrashedServerBuilder( + int index) { + return getRegionsOnCrashedServerFieldBuilder().getBuilder(index); + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder getRegionsOnCrashedServerOrBuilder( + int index) { + if (regionsOnCrashedServerBuilder_ == null) { + return regionsOnCrashedServer_.get(index); } else { + return regionsOnCrashedServerBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public java.util.List + getRegionsOnCrashedServerOrBuilderList() { + if (regionsOnCrashedServerBuilder_ != null) { + return regionsOnCrashedServerBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(regionsOnCrashedServer_); + } + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder addRegionsOnCrashedServerBuilder() { + return getRegionsOnCrashedServerFieldBuilder().addBuilder( + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.getDefaultInstance()); + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder addRegionsOnCrashedServerBuilder( + int index) { + return getRegionsOnCrashedServerFieldBuilder().addBuilder( + index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.getDefaultInstance()); + } + /** + * repeated .RegionInfo regions_on_crashed_server = 3; + */ + public java.util.List + getRegionsOnCrashedServerBuilderList() { + return getRegionsOnCrashedServerFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder> + getRegionsOnCrashedServerFieldBuilder() { + if (regionsOnCrashedServerBuilder_ == null) { + regionsOnCrashedServerBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder>( + regionsOnCrashedServer_, + ((bitField0_ & 0x00000004) == 0x00000004), + getParentForChildren(), + isClean()); + regionsOnCrashedServer_ = null; + } + return regionsOnCrashedServerBuilder_; + } + + // repeated .RegionInfo regions_to_assign = 4; + private java.util.List regionsToAssign_ = + java.util.Collections.emptyList(); + private void ensureRegionsToAssignIsMutable() { + if (!((bitField0_ & 0x00000008) == 0x00000008)) { + regionsToAssign_ = new java.util.ArrayList(regionsToAssign_); + bitField0_ |= 0x00000008; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder> regionsToAssignBuilder_; + + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public java.util.List getRegionsToAssignList() { + if (regionsToAssignBuilder_ == null) { + return java.util.Collections.unmodifiableList(regionsToAssign_); + } else { + return regionsToAssignBuilder_.getMessageList(); + } + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public int getRegionsToAssignCount() { + if (regionsToAssignBuilder_ == null) { + return regionsToAssign_.size(); + } else { + return regionsToAssignBuilder_.getCount(); + } + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo getRegionsToAssign(int index) { + if (regionsToAssignBuilder_ == null) { + return regionsToAssign_.get(index); + } else { + return regionsToAssignBuilder_.getMessage(index); + } + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public Builder setRegionsToAssign( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo value) { + if (regionsToAssignBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRegionsToAssignIsMutable(); + regionsToAssign_.set(index, value); + onChanged(); + } else { + regionsToAssignBuilder_.setMessage(index, value); + } + return this; + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public Builder setRegionsToAssign( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder builderForValue) { + if (regionsToAssignBuilder_ == null) { + ensureRegionsToAssignIsMutable(); + regionsToAssign_.set(index, builderForValue.build()); + onChanged(); + } else { + regionsToAssignBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public Builder addRegionsToAssign(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo value) { + if (regionsToAssignBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRegionsToAssignIsMutable(); + regionsToAssign_.add(value); + onChanged(); + } else { + regionsToAssignBuilder_.addMessage(value); + } + return this; + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public Builder addRegionsToAssign( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo value) { + if (regionsToAssignBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureRegionsToAssignIsMutable(); + regionsToAssign_.add(index, value); + onChanged(); + } else { + regionsToAssignBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public Builder addRegionsToAssign( + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder builderForValue) { + if (regionsToAssignBuilder_ == null) { + ensureRegionsToAssignIsMutable(); + regionsToAssign_.add(builderForValue.build()); + onChanged(); + } else { + regionsToAssignBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public Builder addRegionsToAssign( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder builderForValue) { + if (regionsToAssignBuilder_ == null) { + ensureRegionsToAssignIsMutable(); + regionsToAssign_.add(index, builderForValue.build()); + onChanged(); + } else { + regionsToAssignBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public Builder addAllRegionsToAssign( + java.lang.Iterable values) { + if (regionsToAssignBuilder_ == null) { + ensureRegionsToAssignIsMutable(); + super.addAll(values, regionsToAssign_); + onChanged(); + } else { + regionsToAssignBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public Builder clearRegionsToAssign() { + if (regionsToAssignBuilder_ == null) { + regionsToAssign_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000008); + onChanged(); + } else { + regionsToAssignBuilder_.clear(); + } + return this; + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public Builder removeRegionsToAssign(int index) { + if (regionsToAssignBuilder_ == null) { + ensureRegionsToAssignIsMutable(); + regionsToAssign_.remove(index); + onChanged(); + } else { + regionsToAssignBuilder_.remove(index); + } + return this; + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder getRegionsToAssignBuilder( + int index) { + return getRegionsToAssignFieldBuilder().getBuilder(index); + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder getRegionsToAssignOrBuilder( + int index) { + if (regionsToAssignBuilder_ == null) { + return regionsToAssign_.get(index); } else { + return regionsToAssignBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public java.util.List + getRegionsToAssignOrBuilderList() { + if (regionsToAssignBuilder_ != null) { + return regionsToAssignBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(regionsToAssign_); + } + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder addRegionsToAssignBuilder() { + return getRegionsToAssignFieldBuilder().addBuilder( + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.getDefaultInstance()); + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder addRegionsToAssignBuilder( + int index) { + return getRegionsToAssignFieldBuilder().addBuilder( + index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.getDefaultInstance()); + } + /** + * repeated .RegionInfo regions_to_assign = 4; + */ + public java.util.List + getRegionsToAssignBuilderList() { + return getRegionsToAssignFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder> + getRegionsToAssignFieldBuilder() { + if (regionsToAssignBuilder_ == null) { + regionsToAssignBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfoOrBuilder>( + regionsToAssign_, + ((bitField0_ & 0x00000008) == 0x00000008), + getParentForChildren(), + isClean()); + regionsToAssign_ = null; + } + return regionsToAssignBuilder_; + } + + // optional bool carrying_meta = 5; + private boolean carryingMeta_ ; + /** + * optional bool carrying_meta = 5; + */ + public boolean hasCarryingMeta() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + /** + * optional bool carrying_meta = 5; + */ + public boolean getCarryingMeta() { + return carryingMeta_; + } + /** + * optional bool carrying_meta = 5; + */ + public Builder setCarryingMeta(boolean value) { + bitField0_ |= 0x00000010; + carryingMeta_ = value; + onChanged(); + return this; + } + /** + * optional bool carrying_meta = 5; + */ + public Builder clearCarryingMeta() { + bitField0_ = (bitField0_ & ~0x00000010); + carryingMeta_ = false; + onChanged(); + return this; + } + + // optional bool should_split_wal = 6 [default = true]; + private boolean shouldSplitWal_ = true; + /** + * optional bool should_split_wal = 6 [default = true]; + */ + public boolean hasShouldSplitWal() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + /** + * optional bool should_split_wal = 6 [default = true]; + */ + public boolean getShouldSplitWal() { + return shouldSplitWal_; + } + /** + * optional bool should_split_wal = 6 [default = true]; + */ + public Builder setShouldSplitWal(boolean value) { + bitField0_ |= 0x00000020; + shouldSplitWal_ = value; + onChanged(); + return this; + } + /** + * optional bool should_split_wal = 6 [default = true]; + */ + public Builder clearShouldSplitWal() { + bitField0_ = (bitField0_ & ~0x00000020); + shouldSplitWal_ = true; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:ServerCrashStateData) + } + + static { + defaultInstance = new ServerCrashStateData(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:ServerCrashStateData) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_CreateTableStateData_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_CreateTableStateData_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_ModifyTableStateData_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_ModifyTableStateData_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_TruncateTableStateData_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_TruncateTableStateData_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_DeleteTableStateData_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_DeleteTableStateData_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_AddColumnFamilyStateData_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_AddColumnFamilyStateData_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_ModifyColumnFamilyStateData_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_ModifyColumnFamilyStateData_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_DeleteColumnFamilyStateData_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_DeleteColumnFamilyStateData_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_EnableTableStateData_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_EnableTableStateData_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_DisableTableStateData_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_DisableTableStateData_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_ServerCrashStateData_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_ServerCrashStateData_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\025MasterProcedure.proto\032\013HBase.proto\032\tRP" + + "C.proto\"\201\001\n\024CreateTableStateData\022#\n\tuser" + + "_info\030\001 \002(\0132\020.UserInformation\022\"\n\014table_s" + + "chema\030\002 \002(\0132\014.TableSchema\022 \n\013region_info" + + "\030\003 \003(\0132\013.RegionInfo\"\277\001\n\024ModifyTableState" + + "Data\022#\n\tuser_info\030\001 \002(\0132\020.UserInformatio" + + "n\022-\n\027unmodified_table_schema\030\002 \001(\0132\014.Tab" + + "leSchema\022+\n\025modified_table_schema\030\003 \002(\0132" + + "\014.TableSchema\022&\n\036delete_column_family_in" + + "_modify\030\004 \002(\010\"\274\001\n\026TruncateTableStateData", + "\022#\n\tuser_info\030\001 \002(\0132\020.UserInformation\022\027\n" + + "\017preserve_splits\030\002 \002(\010\022\036\n\ntable_name\030\003 \001" + + "(\0132\n.TableName\022\"\n\014table_schema\030\004 \001(\0132\014.T" + + "ableSchema\022 \n\013region_info\030\005 \003(\0132\013.Region" + + "Info\"}\n\024DeleteTableStateData\022#\n\tuser_inf" + + "o\030\001 \002(\0132\020.UserInformation\022\036\n\ntable_name\030" + + "\002 \002(\0132\n.TableName\022 \n\013region_info\030\003 \003(\0132\013" + + ".RegionInfo\"\300\001\n\030AddColumnFamilyStateData" + + "\022#\n\tuser_info\030\001 \002(\0132\020.UserInformation\022\036\n" + + "\ntable_name\030\002 \002(\0132\n.TableName\0220\n\023columnf", + "amily_schema\030\003 \002(\0132\023.ColumnFamilySchema\022" + + "-\n\027unmodified_table_schema\030\004 \001(\0132\014.Table" + + "Schema\"\303\001\n\033ModifyColumnFamilyStateData\022#" + + "\n\tuser_info\030\001 \002(\0132\020.UserInformation\022\036\n\nt" + + "able_name\030\002 \002(\0132\n.TableName\0220\n\023columnfam" + + "ily_schema\030\003 \002(\0132\023.ColumnFamilySchema\022-\n" + + "\027unmodified_table_schema\030\004 \001(\0132\014.TableSc" + + "hema\"\254\001\n\033DeleteColumnFamilyStateData\022#\n\t" + + "user_info\030\001 \002(\0132\020.UserInformation\022\036\n\ntab" + + "le_name\030\002 \002(\0132\n.TableName\022\031\n\021columnfamil", + "y_name\030\003 \002(\014\022-\n\027unmodified_table_schema\030" + + "\004 \001(\0132\014.TableSchema\"{\n\024EnableTableStateD" + + "ata\022#\n\tuser_info\030\001 \002(\0132\020.UserInformation" + + "\022\036\n\ntable_name\030\002 \002(\0132\n.TableName\022\036\n\026skip" + + "_table_state_check\030\003 \002(\010\"|\n\025DisableTable" + + "StateData\022#\n\tuser_info\030\001 \002(\0132\020.UserInfor" + + "mation\022\036\n\ntable_name\030\002 \002(\0132\n.TableName\022\036" + + "\n\026skip_table_state_check\030\003 \002(\010\"\347\001\n\024Serve" + + "rCrashStateData\022 \n\013server_name\030\001 \002(\0132\013.S" + + "erverName\022\036\n\026distributed_log_replay\030\002 \001(", + "\010\022.\n\031regions_on_crashed_server\030\003 \003(\0132\013.R" + + "egionInfo\022&\n\021regions_to_assign\030\004 \003(\0132\013.R" + + "egionInfo\022\025\n\rcarrying_meta\030\005 \001(\010\022\036\n\020shou" + + "ld_split_wal\030\006 \001(\010:\004true*\330\001\n\020CreateTable" + + "State\022\036\n\032CREATE_TABLE_PRE_OPERATION\020\001\022 \n" + + "\034CREATE_TABLE_WRITE_FS_LAYOUT\020\002\022\034\n\030CREAT" + + "E_TABLE_ADD_TO_META\020\003\022\037\n\033CREATE_TABLE_AS" + + "SIGN_REGIONS\020\004\022\"\n\036CREATE_TABLE_UPDATE_DE" + + "SC_CACHE\020\005\022\037\n\033CREATE_TABLE_POST_OPERATIO" + + "N\020\006*\207\002\n\020ModifyTableState\022\030\n\024MODIFY_TABLE", + "_PREPARE\020\001\022\036\n\032MODIFY_TABLE_PRE_OPERATION" + + "\020\002\022(\n$MODIFY_TABLE_UPDATE_TABLE_DESCRIPT" + + "OR\020\003\022&\n\"MODIFY_TABLE_REMOVE_REPLICA_COLU" + + "MN\020\004\022!\n\035MODIFY_TABLE_DELETE_FS_LAYOUT\020\005\022" + + "\037\n\033MODIFY_TABLE_POST_OPERATION\020\006\022#\n\037MODI" + + "FY_TABLE_REOPEN_ALL_REGIONS\020\007*\212\002\n\022Trunca" + + "teTableState\022 \n\034TRUNCATE_TABLE_PRE_OPERA" + + "TION\020\001\022#\n\037TRUNCATE_TABLE_REMOVE_FROM_MET" + + "A\020\002\022\"\n\036TRUNCATE_TABLE_CLEAR_FS_LAYOUT\020\003\022" + + "#\n\037TRUNCATE_TABLE_CREATE_FS_LAYOUT\020\004\022\036\n\032", + "TRUNCATE_TABLE_ADD_TO_META\020\005\022!\n\035TRUNCATE" + + "_TABLE_ASSIGN_REGIONS\020\006\022!\n\035TRUNCATE_TABL" + + "E_POST_OPERATION\020\007*\337\001\n\020DeleteTableState\022" + + "\036\n\032DELETE_TABLE_PRE_OPERATION\020\001\022!\n\035DELET" + + "E_TABLE_REMOVE_FROM_META\020\002\022 \n\034DELETE_TAB" + + "LE_CLEAR_FS_LAYOUT\020\003\022\"\n\036DELETE_TABLE_UPD" + + "ATE_DESC_CACHE\020\004\022!\n\035DELETE_TABLE_UNASSIG" + + "N_REGIONS\020\005\022\037\n\033DELETE_TABLE_POST_OPERATI" + + "ON\020\006*\331\001\n\024AddColumnFamilyState\022\035\n\031ADD_COL" + + "UMN_FAMILY_PREPARE\020\001\022#\n\037ADD_COLUMN_FAMIL", + "Y_PRE_OPERATION\020\002\022-\n)ADD_COLUMN_FAMILY_U" + + "PDATE_TABLE_DESCRIPTOR\020\003\022$\n ADD_COLUMN_F" + + "AMILY_POST_OPERATION\020\004\022(\n$ADD_COLUMN_FAM" + + "ILY_REOPEN_ALL_REGIONS\020\005*\353\001\n\027ModifyColum" + + "nFamilyState\022 \n\034MODIFY_COLUMN_FAMILY_PRE" + + "PARE\020\001\022&\n\"MODIFY_COLUMN_FAMILY_PRE_OPERA" + + "TION\020\002\0220\n,MODIFY_COLUMN_FAMILY_UPDATE_TA" + + "BLE_DESCRIPTOR\020\003\022\'\n#MODIFY_COLUMN_FAMILY" + + "_POST_OPERATION\020\004\022+\n\'MODIFY_COLUMN_FAMIL" + + "Y_REOPEN_ALL_REGIONS\020\005*\226\002\n\027DeleteColumnF", + "amilyState\022 \n\034DELETE_COLUMN_FAMILY_PREPA" + + "RE\020\001\022&\n\"DELETE_COLUMN_FAMILY_PRE_OPERATI" + + "ON\020\002\0220\n,DELETE_COLUMN_FAMILY_UPDATE_TABL" + + "E_DESCRIPTOR\020\003\022)\n%DELETE_COLUMN_FAMILY_D" + + "ELETE_FS_LAYOUT\020\004\022\'\n#DELETE_COLUMN_FAMIL" + + "Y_POST_OPERATION\020\005\022+\n\'DELETE_COLUMN_FAMI" + + "LY_REOPEN_ALL_REGIONS\020\006*\350\001\n\020EnableTableS" + + "tate\022\030\n\024ENABLE_TABLE_PREPARE\020\001\022\036\n\032ENABLE" + + "_TABLE_PRE_OPERATION\020\002\022)\n%ENABLE_TABLE_S" + + "ET_ENABLING_TABLE_STATE\020\003\022$\n ENABLE_TABL", + "E_MARK_REGIONS_ONLINE\020\004\022(\n$ENABLE_TABLE_" + + "SET_ENABLED_TABLE_STATE\020\005\022\037\n\033ENABLE_TABL" + + "E_POST_OPERATION\020\006*\362\001\n\021DisableTableState" + + "\022\031\n\025DISABLE_TABLE_PREPARE\020\001\022\037\n\033DISABLE_T" + + "ABLE_PRE_OPERATION\020\002\022+\n\'DISABLE_TABLE_SE" + + "T_DISABLING_TABLE_STATE\020\003\022&\n\"DISABLE_TAB" + + "LE_MARK_REGIONS_OFFLINE\020\004\022*\n&DISABLE_TAB" + + "LE_SET_DISABLED_TABLE_STATE\020\005\022 \n\034DISABLE" + + "_TABLE_POST_OPERATION\020\006*\305\002\n\020ServerCrashS" + + "tate\022\026\n\022SERVER_CRASH_START\020\001\022\035\n\031SERVER_C", + "RASH_PROCESS_META\020\002\022\034\n\030SERVER_CRASH_GET_" + + "REGIONS\020\003\022\036\n\032SERVER_CRASH_NO_SPLIT_LOGS\020" + + "\004\022\033\n\027SERVER_CRASH_SPLIT_LOGS\020\005\022#\n\037SERVER" + + "_CRASH_PREPARE_LOG_REPLAY\020\006\022\'\n#SERVER_CR" + + "ASH_CALC_REGIONS_TO_ASSIGN\020\007\022\027\n\023SERVER_C" + + "RASH_ASSIGN\020\010\022\037\n\033SERVER_CRASH_WAIT_ON_AS" + + "SIGN\020\t\022\027\n\023SERVER_CRASH_FINISH\020dBK\n*org.a" + + "pache.hadoop.hbase.protobuf.generatedB\025M" + + "asterProcedureProtosH\001\210\001\001\240\001\001" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_CreateTableStateData_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_CreateTableStateData_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_CreateTableStateData_descriptor, + new java.lang.String[] { "UserInfo", "TableSchema", "RegionInfo", }); + internal_static_ModifyTableStateData_descriptor = + getDescriptor().getMessageTypes().get(1); + internal_static_ModifyTableStateData_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_ModifyTableStateData_descriptor, + new java.lang.String[] { "UserInfo", "UnmodifiedTableSchema", "ModifiedTableSchema", "DeleteColumnFamilyInModify", }); + internal_static_TruncateTableStateData_descriptor = + getDescriptor().getMessageTypes().get(2); + internal_static_TruncateTableStateData_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_TruncateTableStateData_descriptor, + new java.lang.String[] { "UserInfo", "PreserveSplits", "TableName", "TableSchema", "RegionInfo", }); + internal_static_DeleteTableStateData_descriptor = + getDescriptor().getMessageTypes().get(3); internal_static_DeleteTableStateData_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_DeleteTableStateData_descriptor, @@ -11409,6 +13173,12 @@ public final class MasterProcedureProtos { com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_DisableTableStateData_descriptor, new java.lang.String[] { "UserInfo", "TableName", "SkipTableStateCheck", }); + internal_static_ServerCrashStateData_descriptor = + getDescriptor().getMessageTypes().get(9); + internal_static_ServerCrashStateData_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_ServerCrashStateData_descriptor, + new java.lang.String[] { "ServerName", "DistributedLogReplay", "RegionsOnCrashedServer", "RegionsToAssign", "CarryingMeta", "ShouldSplitWal", }); return null; } }; diff --git a/hbase-protocol/src/main/protobuf/MasterProcedure.proto b/hbase-protocol/src/main/protobuf/MasterProcedure.proto index e1c6880..5e94721 100644 --- a/hbase-protocol/src/main/protobuf/MasterProcedure.proto +++ b/hbase-protocol/src/main/protobuf/MasterProcedure.proto @@ -183,3 +183,25 @@ message DisableTableStateData { required TableName table_name = 2; required bool skip_table_state_check = 3; } + +message ServerCrashStateData { + required ServerName server_name = 1; + optional bool distributed_log_replay = 2; + repeated RegionInfo regions_on_crashed_server = 3; + repeated RegionInfo regions_to_assign = 4; + optional bool carrying_meta = 5; + optional bool should_split_wal = 6 [default = true]; +} + +enum ServerCrashState { + SERVER_CRASH_START = 1; + SERVER_CRASH_PROCESS_META = 2; + SERVER_CRASH_GET_REGIONS = 3; + SERVER_CRASH_NO_SPLIT_LOGS = 4; + SERVER_CRASH_SPLIT_LOGS = 5; + SERVER_CRASH_PREPARE_LOG_REPLAY = 6; + SERVER_CRASH_CALC_REGIONS_TO_ASSIGN = 7; + SERVER_CRASH_ASSIGN = 8; + SERVER_CRASH_WAIT_ON_ASSIGN = 9; + SERVER_CRASH_FINISH = 100; +} diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon index 6d736eb..e8ca96b 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon @@ -216,7 +216,7 @@ AssignmentManager assignmentManager = master.getAssignmentManager();
<& BackupMasterStatusTmpl; master = master &>
- +
@@ -242,7 +242,7 @@ AssignmentManager assignmentManager = master.getAssignmentManager(); HBase Source Checksum - <% org.apache.hadoop.util.VersionInfo.getSrcChecksum() %> + <% org.apache.hadoop.hbase.util.VersionInfo.getSrcChecksum() %> HBase source MD5 checksum @@ -256,6 +256,21 @@ AssignmentManager assignmentManager = master.getAssignmentManager(); When Hadoop version was compiled and by whom + Hadoop Source Checksum + <% org.apache.hadoop.util.VersionInfo.getSrcChecksum() %> + Hadoop source MD5 checksum + + + ZooKeeper Client Version + <% org.apache.zookeeper.Version.getVersion() %>, revision=<% org.apache.zookeeper.Version.getRevision() %> + ZooKeeper client version and revision + + + ZooKeeper Client Compiled + <% org.apache.zookeeper.Version.getBuildDate() %> + When ZooKeeper client version was compiled + + Zookeeper Quorum <%escape #n> <% formatZKString() %> @@ -368,7 +383,7 @@ AssignmentManager assignmentManager = master.getAssignmentManager(); <%def userTables> <%java> - HTableDescriptor[] tables = null; + HTableDescriptor[] tables = null; try (Admin admin = master.getConnection().getAdmin()) { tables = master.isInitialized() ? admin.listTables() : null; } @@ -390,9 +405,10 @@ AssignmentManager assignmentManager = master.getAssignmentManager(); <%for HTableDescriptor htDesc : tables%> <%java> + TableName tableName = htDesc.getTableName(); Map> tableRegions = master.getAssignmentManager().getRegionStates() - .getRegionByStateOfTable(htDesc.getTableName()); + .getRegionByStateOfTable(tableName); int openRegionsCount = tableRegions.get(RegionState.State.OPEN).size(); int offlineRegionsCount = tableRegions.get(RegionState.State.OFFLINE).size(); int splitRegionsCount = tableRegions.get(RegionState.State.SPLIT).size(); @@ -408,10 +424,10 @@ AssignmentManager assignmentManager = master.getAssignmentManager(); - splitRegionsCount; - <% htDesc.getTableName().getNamespaceAsString() %> - ><% htDesc.getTableName().getQualifierAsString() %> + <% tableName.getNamespaceAsString() %> + ><% tableName.getQualifierAsString() %> <%if (frags != null) %> - <% frags.get(htDesc.getTableName().getNameAsString()) != null ? frags.get(htDesc.getTableName().getQualifierAsString()).intValue() + "%" : "n/a" %> + <% frags.get(tableName.getNameAsString()) != null ? frags.get(tableName.getNameAsString()).intValue() + "%" : "n/a" %> <% openRegionsCount %> <% offlineRegionsCount %> diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon index 49c3b5e..3d47643 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon @@ -142,7 +142,7 @@ org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; HBase Source Checksum - <% org.apache.hadoop.util.VersionInfo.getSrcChecksum() %> + <% org.apache.hadoop.hbase.util.VersionInfo.getSrcChecksum() %> HBase source MD5 checksum @@ -156,6 +156,21 @@ org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; When Hadoop version was compiled and by whom + Hadoop Source Checksum + <% org.apache.hadoop.util.VersionInfo.getSrcChecksum() %> + Hadoop source MD5 checksum + + + ZooKeeper Client Version + <% org.apache.zookeeper.Version.getVersion() %>, revision=<% org.apache.zookeeper.Version.getRevision() %> + ZooKeeper client version and revision + + + ZooKeeper Client Compiled + <% org.apache.zookeeper.Version.getBuildDate() %> + When ZooKeeper client version was compiled + + Zookeeper Quorum <% regionServer.getZooKeeper().getQuorum() %> Addresses of all registered ZK servers diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java index acdcf60..0ae3a00 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZKSplitLogManagerCoordination.java @@ -69,7 +69,7 @@ import org.apache.zookeeper.data.Stat; /** * ZooKeeper based implementation of - * {@link org.apache.hadoop.hbase.master.SplitLogManagerCoordination} + * {@link SplitLogManagerCoordination} */ @InterfaceAudience.Private public class ZKSplitLogManagerCoordination extends ZooKeeperListener implements @@ -645,9 +645,9 @@ public class ZKSplitLogManagerCoordination extends ZooKeeperListener implements lastSequenceId = lastRecordedFlushedSequenceId; } ZKUtil.createSetData(this.watcher, nodePath, - ZKUtil.regionSequenceIdsToByteArray(lastSequenceId, null)); + ZKUtil.regionSequenceIdsToByteArray(lastSequenceId, null)); if (LOG.isDebugEnabled()) { - LOG.debug("Marked " + regionEncodeName + " as recovering from " + serverName + + LOG.debug("Marked " + regionEncodeName + " recovering from " + serverName + ": " + nodePath); } // break retry loop @@ -684,7 +684,7 @@ public class ZKSplitLogManagerCoordination extends ZooKeeperListener implements /** * ZooKeeper implementation of - * {@link org.apache.hadoop.hbase.master. + * {@link org.apache.hadoop.hbase.coordination. * SplitLogManagerCoordination#removeStaleRecoveringRegions(Set)} */ @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java index 637920b..b682764 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coordination/ZkSplitLogWorkerCoordination.java @@ -104,7 +104,7 @@ public class ZkSplitLogWorkerCoordination extends ZooKeeperListener implements @Override public void nodeChildrenChanged(String path) { if (path.equals(watcher.splitLogZNode)) { - LOG.debug("tasks arrived or departed"); + if (LOG.isTraceEnabled()) LOG.trace("tasks arrived or departed on " + path); synchronized (taskReadyLock) { taskReadySeq++; taskReadyLock.notify(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java index e2688b1..84c3548 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java @@ -57,12 +57,12 @@ import org.apache.hadoop.hbase.http.conf.ConfServlet; import org.apache.hadoop.hbase.http.jmx.JMXJsonServlet; import org.apache.hadoop.hbase.http.log.LogLevel; import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.util.ReflectionUtils; import org.apache.hadoop.metrics.MetricsServlet; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authorize.AccessControlList; -import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Shell; import org.mortbay.io.Buffer; import org.mortbay.jetty.Connector; @@ -617,8 +617,7 @@ public class HttpServer implements FilterContainer { FilterInitializer[] initializers = new FilterInitializer[classes.length]; for(int i = 0; i < classes.length; i++) { - initializers[i] = (FilterInitializer)ReflectionUtils.newInstance( - classes[i], conf); + initializers[i] = (FilterInitializer)ReflectionUtils.newInstance(classes[i]); } return initializers; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/MultiTableSnapshotInputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/MultiTableSnapshotInputFormat.java new file mode 100644 index 0000000..ab27edd --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/MultiTableSnapshotInputFormat.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.mapred; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.mapreduce.MultiTableSnapshotInputFormatImpl; +import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormatImpl; +import org.apache.hadoop.mapred.InputFormat; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.RecordReader; +import org.apache.hadoop.mapred.Reporter; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * MultiTableSnapshotInputFormat generalizes {@link org.apache.hadoop.hbase.mapred + * .TableSnapshotInputFormat} + * allowing a MapReduce job to run over one or more table snapshots, with one or more scans + * configured for each. + * Internally, the input format delegates to {@link org.apache.hadoop.hbase.mapreduce + * .TableSnapshotInputFormat} + * and thus has the same performance advantages; see {@link org.apache.hadoop.hbase.mapreduce + * .TableSnapshotInputFormat} for + * more details. + * Usage is similar to TableSnapshotInputFormat, with the following exception: + * initMultiTableSnapshotMapperJob takes in a map + * from snapshot name to a collection of scans. For each snapshot in the map, each corresponding + * scan will be applied; + * the overall dataset for the job is defined by the concatenation of the regions and tables + * included in each snapshot/scan + * pair. + * {@link org.apache.hadoop.hbase.mapred.TableMapReduceUtil#initMultiTableSnapshotMapperJob(Map, + * Class, Class, Class, JobConf, boolean, Path)} + * can be used to configure the job. + *
{@code
+ * Job job = new Job(conf);
+ * Map> snapshotScans = ImmutableMap.of(
+ *    "snapshot1", ImmutableList.of(new Scan(Bytes.toBytes("a"), Bytes.toBytes("b"))),
+ *    "snapshot2", ImmutableList.of(new Scan(Bytes.toBytes("1"), Bytes.toBytes("2")))
+ * );
+ * Path restoreDir = new Path("/tmp/snapshot_restore_dir")
+ * TableMapReduceUtil.initTableSnapshotMapperJob(
+ *     snapshotScans, MyTableMapper.class, MyMapKeyOutput.class,
+ *      MyMapOutputValueWritable.class, job, true, restoreDir);
+ * }
+ * 
+ * Internally, this input format restores each snapshot into a subdirectory of the given tmp + * directory. Input splits and + * record readers are created as described in {@link org.apache.hadoop.hbase.mapreduce + * .TableSnapshotInputFormat} + * (one per region). + * See {@link org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat} for more notes on + * permissioning; the + * same caveats apply here. + * + * @see org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat + * @see org.apache.hadoop.hbase.client.TableSnapshotScanner + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class MultiTableSnapshotInputFormat extends TableSnapshotInputFormat + implements InputFormat { + + private final MultiTableSnapshotInputFormatImpl delegate; + + public MultiTableSnapshotInputFormat() { + this.delegate = new MultiTableSnapshotInputFormatImpl(); + } + + @Override + public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { + List splits = delegate.getSplits(job); + InputSplit[] results = new InputSplit[splits.size()]; + for (int i = 0; i < splits.size(); i++) { + results[i] = new TableSnapshotRegionSplit(splits.get(i)); + } + return results; + } + + @Override + public RecordReader getRecordReader(InputSplit split, JobConf job, + Reporter reporter) throws IOException { + return new TableSnapshotRecordReader((TableSnapshotRegionSplit) split, job); + } + + /** + * Configure conf to read from snapshotScans, with snapshots restored to a subdirectory of + * restoreDir. + * Sets: {@link org.apache.hadoop.hbase.mapreduce + * .MultiTableSnapshotInputFormatImpl#RESTORE_DIRS_KEY}, + * {@link org.apache.hadoop.hbase.mapreduce + * .MultiTableSnapshotInputFormatImpl#SNAPSHOT_TO_SCANS_KEY} + * + * @param conf + * @param snapshotScans + * @param restoreDir + * @throws IOException + */ + public static void setInput(Configuration conf, Map> snapshotScans, + Path restoreDir) throws IOException { + new MultiTableSnapshotInputFormatImpl().setInput(conf, snapshotScans, restoreDir); + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java index b5fefbb..2a040d9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableMapReduceUtil.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hbase.mapred; import java.io.IOException; +import java.util.Collection; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -30,11 +32,10 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.MutationSerialization; import org.apache.hadoop.hbase.mapreduce.ResultSerialization; -import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier; -import org.apache.hadoop.hbase.security.token.AuthenticationTokenSelector; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.security.token.TokenUtil; @@ -128,6 +129,40 @@ public class TableMapReduceUtil { } /** + * Sets up the job for reading from one or more multiple table snapshots, with one or more scans + * per snapshot. + * It bypasses hbase servers and read directly from snapshot files. + * + * @param snapshotScans map of snapshot name to scans on that snapshot. + * @param mapper The mapper class to use. + * @param outputKeyClass The class of the output key. + * @param outputValueClass The class of the output value. + * @param job The current job to adjust. Make sure the passed job is + * carrying all necessary HBase configuration. + * @param addDependencyJars upload HBase jars and jars for any of the configured + * job classes via the distributed cache (tmpjars). + */ + public static void initMultiTableSnapshotMapperJob(Map> snapshotScans, + Class mapper, Class outputKeyClass, Class outputValueClass, + JobConf job, boolean addDependencyJars, Path tmpRestoreDir) throws IOException { + MultiTableSnapshotInputFormat.setInput(job, snapshotScans, tmpRestoreDir); + + job.setInputFormat(MultiTableSnapshotInputFormat.class); + if (outputValueClass != null) { + job.setMapOutputValueClass(outputValueClass); + } + if (outputKeyClass != null) { + job.setMapOutputKeyClass(outputKeyClass); + } + job.setMapperClass(mapper); + if (addDependencyJars) { + addDependencyJars(job); + } + + org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.resetCacheConfig(job); + } + + /** * Sets up the job for reading from a table snapshot. It bypasses hbase servers * and read directly from snapshot files. * diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableSnapshotInputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableSnapshotInputFormat.java index 1c5e4bd..a5c62b2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableSnapshotInputFormat.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapred/TableSnapshotInputFormat.java @@ -18,12 +18,13 @@ package org.apache.hadoop.hbase.mapred; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormatImpl; import org.apache.hadoop.mapred.InputFormat; @@ -60,8 +61,9 @@ public class TableSnapshotInputFormat implements InputFormat locations) { - this.delegate = new TableSnapshotInputFormatImpl.InputSplit(htd, regionInfo, locations); + List locations, Scan scan, Path restoreDir) { + this.delegate = + new TableSnapshotInputFormatImpl.InputSplit(htd, regionInfo, locations, scan, restoreDir); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableSnapshotInputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableSnapshotInputFormat.java new file mode 100644 index 0000000..bd530c8 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableSnapshotInputFormat.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.mapreduce; + +import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * MultiTableSnapshotInputFormat generalizes + * {@link org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat} + * allowing a MapReduce job to run over one or more table snapshots, with one or more scans + * configured for each. + * Internally, the input format delegates to + * {@link org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat} + * and thus has the same performance advantages; + * see {@link org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat} for + * more details. + * Usage is similar to TableSnapshotInputFormat, with the following exception: + * initMultiTableSnapshotMapperJob takes in a map + * from snapshot name to a collection of scans. For each snapshot in the map, each corresponding + * scan will be applied; + * the overall dataset for the job is defined by the concatenation of the regions and tables + * included in each snapshot/scan + * pair. + * {@link org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil#initMultiTableSnapshotMapperJob + * (java.util.Map, Class, Class, Class, org.apache.hadoop.mapreduce.Job, boolean, org.apache + * .hadoop.fs.Path)} + * can be used to configure the job. + *
{@code
+ * Job job = new Job(conf);
+ * Map> snapshotScans = ImmutableMap.of(
+ *    "snapshot1", ImmutableList.of(new Scan(Bytes.toBytes("a"), Bytes.toBytes("b"))),
+ *    "snapshot2", ImmutableList.of(new Scan(Bytes.toBytes("1"), Bytes.toBytes("2")))
+ * );
+ * Path restoreDir = new Path("/tmp/snapshot_restore_dir")
+ * TableMapReduceUtil.initTableSnapshotMapperJob(
+ *     snapshotScans, MyTableMapper.class, MyMapKeyOutput.class,
+ *      MyMapOutputValueWritable.class, job, true, restoreDir);
+ * }
+ * 
+ * Internally, this input format restores each snapshot into a subdirectory of the given tmp + * directory. Input splits and + * record readers are created as described in {@link org.apache.hadoop.hbase.mapreduce + * .TableSnapshotInputFormat} + * (one per region). + * See {@link org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat} for more notes on + * permissioning; the + * same caveats apply here. + * + * @see org.apache.hadoop.hbase.mapreduce.TableSnapshotInputFormat + * @see org.apache.hadoop.hbase.client.TableSnapshotScanner + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class MultiTableSnapshotInputFormat extends TableSnapshotInputFormat { + + private final MultiTableSnapshotInputFormatImpl delegate; + + public MultiTableSnapshotInputFormat() { + this.delegate = new MultiTableSnapshotInputFormatImpl(); + } + + @Override + public List getSplits(JobContext jobContext) + throws IOException, InterruptedException { + List splits = + delegate.getSplits(jobContext.getConfiguration()); + List rtn = Lists.newArrayListWithCapacity(splits.size()); + + for (TableSnapshotInputFormatImpl.InputSplit split : splits) { + rtn.add(new TableSnapshotInputFormat.TableSnapshotRegionSplit(split)); + } + + return rtn; + } + + public static void setInput(Configuration configuration, + Map> snapshotScans, Path tmpRestoreDir) throws IOException { + new MultiTableSnapshotInputFormatImpl().setInput(configuration, snapshotScans, tmpRestoreDir); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableSnapshotInputFormatImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableSnapshotInputFormatImpl.java new file mode 100644 index 0000000..1b8b6c0 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableSnapshotInputFormatImpl.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.mapreduce; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; +import org.apache.hadoop.hbase.snapshot.SnapshotManifest; +import org.apache.hadoop.hbase.util.ConfigurationUtil; +import org.apache.hadoop.hbase.util.FSUtils; + +import java.io.IOException; +import java.util.AbstractMap; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * Shared implementation of mapreduce code over multiple table snapshots. + * Utilized by both mapreduce ({@link org.apache.hadoop.hbase.mapreduce + * .MultiTableSnapshotInputFormat} and mapred + * ({@link org.apache.hadoop.hbase.mapred.MultiTableSnapshotInputFormat} implementations. + */ +@InterfaceAudience.LimitedPrivate({ "HBase" }) +@InterfaceStability.Evolving +public class MultiTableSnapshotInputFormatImpl { + + private static final Log LOG = LogFactory.getLog(MultiTableSnapshotInputFormatImpl.class); + + public static final String RESTORE_DIRS_KEY = + "hbase.MultiTableSnapshotInputFormat.restore.snapshotDirMapping"; + public static final String SNAPSHOT_TO_SCANS_KEY = + "hbase.MultiTableSnapshotInputFormat.snapshotsToScans"; + + /** + * Configure conf to read from snapshotScans, with snapshots restored to a subdirectory of + * restoreDir. + * Sets: {@link #RESTORE_DIRS_KEY}, {@link #SNAPSHOT_TO_SCANS_KEY} + * + * @param conf + * @param snapshotScans + * @param restoreDir + * @throws IOException + */ + public void setInput(Configuration conf, Map> snapshotScans, + Path restoreDir) throws IOException { + Path rootDir = FSUtils.getRootDir(conf); + FileSystem fs = rootDir.getFileSystem(conf); + + setSnapshotToScans(conf, snapshotScans); + Map restoreDirs = + generateSnapshotToRestoreDirMapping(snapshotScans.keySet(), restoreDir); + setSnapshotDirs(conf, restoreDirs); + restoreSnapshots(conf, restoreDirs, fs); + } + + /** + * Return the list of splits extracted from the scans/snapshots pushed to conf by + * {@link + * #setInput(org.apache.hadoop.conf.Configuration, java.util.Map, org.apache.hadoop.fs.Path)} + * + * @param conf Configuration to determine splits from + * @return Return the list of splits extracted from the scans/snapshots pushed to conf + * @throws IOException + */ + public List getSplits(Configuration conf) + throws IOException { + Path rootDir = FSUtils.getRootDir(conf); + FileSystem fs = rootDir.getFileSystem(conf); + + List rtn = Lists.newArrayList(); + + Map> snapshotsToScans = getSnapshotsToScans(conf); + Map snapshotsToRestoreDirs = getSnapshotDirs(conf); + for (Map.Entry> entry : snapshotsToScans.entrySet()) { + String snapshotName = entry.getKey(); + + Path restoreDir = snapshotsToRestoreDirs.get(snapshotName); + + SnapshotManifest manifest = + TableSnapshotInputFormatImpl.getSnapshotManifest(conf, snapshotName, rootDir, fs); + List regionInfos = + TableSnapshotInputFormatImpl.getRegionInfosFromManifest(manifest); + + for (Scan scan : entry.getValue()) { + List splits = + TableSnapshotInputFormatImpl.getSplits(scan, manifest, regionInfos, restoreDir, conf); + rtn.addAll(splits); + } + } + return rtn; + } + + /** + * Retrieve the snapshot name -> list mapping pushed to configuration by + * {@link #setSnapshotToScans(org.apache.hadoop.conf.Configuration, java.util.Map)} + * + * @param conf Configuration to extract name -> list mappings from. + * @return the snapshot name -> list mapping pushed to configuration + * @throws IOException + */ + public Map> getSnapshotsToScans(Configuration conf) throws IOException { + + Map> rtn = Maps.newHashMap(); + + for (Map.Entry entry : ConfigurationUtil + .getKeyValues(conf, SNAPSHOT_TO_SCANS_KEY)) { + String snapshotName = entry.getKey(); + String scan = entry.getValue(); + + Collection snapshotScans = rtn.get(snapshotName); + if (snapshotScans == null) { + snapshotScans = Lists.newArrayList(); + rtn.put(snapshotName, snapshotScans); + } + + snapshotScans.add(TableMapReduceUtil.convertStringToScan(scan)); + } + + return rtn; + } + + /** + * Push snapshotScans to conf (under the key {@link #SNAPSHOT_TO_SCANS_KEY}) + * + * @param conf + * @param snapshotScans + * @throws IOException + */ + public void setSnapshotToScans(Configuration conf, Map> snapshotScans) + throws IOException { + // flatten out snapshotScans for serialization to the job conf + List> snapshotToSerializedScans = Lists.newArrayList(); + + for (Map.Entry> entry : snapshotScans.entrySet()) { + String snapshotName = entry.getKey(); + Collection scans = entry.getValue(); + + // serialize all scans and map them to the appropriate snapshot + for (Scan scan : scans) { + snapshotToSerializedScans.add(new AbstractMap.SimpleImmutableEntry(snapshotName, + TableMapReduceUtil.convertScanToString(scan))); + } + } + + ConfigurationUtil.setKeyValues(conf, SNAPSHOT_TO_SCANS_KEY, snapshotToSerializedScans); + } + + /** + * Retrieve the directories into which snapshots have been restored from + * ({@link #RESTORE_DIRS_KEY}) + * + * @param conf Configuration to extract restore directories from + * @return the directories into which snapshots have been restored from + * @throws IOException + */ + public Map getSnapshotDirs(Configuration conf) throws IOException { + List> kvps = ConfigurationUtil.getKeyValues(conf, RESTORE_DIRS_KEY); + Map rtn = Maps.newHashMapWithExpectedSize(kvps.size()); + + for (Map.Entry kvp : kvps) { + rtn.put(kvp.getKey(), new Path(kvp.getValue())); + } + + return rtn; + } + + public void setSnapshotDirs(Configuration conf, Map snapshotDirs) { + Map toSet = Maps.newHashMap(); + + for (Map.Entry entry : snapshotDirs.entrySet()) { + toSet.put(entry.getKey(), entry.getValue().toString()); + } + + ConfigurationUtil.setKeyValues(conf, RESTORE_DIRS_KEY, toSet.entrySet()); + } + + /** + * Generate a random path underneath baseRestoreDir for each snapshot in snapshots and + * return a map from the snapshot to the restore directory. + * + * @param snapshots collection of snapshot names to restore + * @param baseRestoreDir base directory under which all snapshots in snapshots will be restored + * @return a mapping from snapshot name to the directory in which that snapshot has been restored + */ + private Map generateSnapshotToRestoreDirMapping(Collection snapshots, + Path baseRestoreDir) { + Map rtn = Maps.newHashMap(); + + for (String snapshotName : snapshots) { + Path restoreSnapshotDir = + new Path(baseRestoreDir, snapshotName + "__" + UUID.randomUUID().toString()); + rtn.put(snapshotName, restoreSnapshotDir); + } + + return rtn; + } + + /** + * Restore each (snapshot name, restore directory) pair in snapshotToDir + * + * @param conf configuration to restore with + * @param snapshotToDir mapping from snapshot names to restore directories + * @param fs filesystem to do snapshot restoration on + * @throws IOException + */ + public void restoreSnapshots(Configuration conf, Map snapshotToDir, FileSystem fs) + throws IOException { + // TODO: restore from record readers to parallelize. + Path rootDir = FSUtils.getRootDir(conf); + + for (Map.Entry entry : snapshotToDir.entrySet()) { + String snapshotName = entry.getKey(); + Path restoreDir = entry.getValue(); + LOG.info("Restoring snapshot " + snapshotName + " into " + restoreDir + + " for MultiTableSnapshotInputFormat"); + restoreSnapshot(conf, snapshotName, rootDir, restoreDir, fs); + } + } + + void restoreSnapshot(Configuration conf, String snapshotName, Path rootDir, Path restoreDir, + FileSystem fs) throws IOException { + RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName); + } + + // TODO: these probably belong elsewhere/may already be implemented elsewhere. + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java index 7d793ab..8cea2f4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java @@ -22,13 +22,7 @@ import java.io.File; import java.io.IOException; import java.net.URL; import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -306,6 +300,43 @@ public class TableMapReduceUtil { } /** + * Sets up the job for reading from one or more table snapshots, with one or more scans + * per snapshot. + * It bypasses hbase servers and read directly from snapshot files. + * + * @param snapshotScans map of snapshot name to scans on that snapshot. + * @param mapper The mapper class to use. + * @param outputKeyClass The class of the output key. + * @param outputValueClass The class of the output value. + * @param job The current job to adjust. Make sure the passed job is + * carrying all necessary HBase configuration. + * @param addDependencyJars upload HBase jars and jars for any of the configured + * job classes via the distributed cache (tmpjars). + */ + public static void initMultiTableSnapshotMapperJob(Map> snapshotScans, + Class mapper, Class outputKeyClass, Class outputValueClass, + Job job, boolean addDependencyJars, Path tmpRestoreDir) throws IOException { + MultiTableSnapshotInputFormat.setInput(job.getConfiguration(), snapshotScans, tmpRestoreDir); + + job.setInputFormatClass(MultiTableSnapshotInputFormat.class); + if (outputValueClass != null) { + job.setMapOutputValueClass(outputValueClass); + } + if (outputKeyClass != null) { + job.setMapOutputKeyClass(outputKeyClass); + } + job.setMapperClass(mapper); + Configuration conf = job.getConfiguration(); + HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf)); + + if (addDependencyJars) { + addDependencyJars(job); + } + + resetCacheConfig(job.getConfiguration()); + } + + /** * Sets up the job for reading from a table snapshot. It bypasses hbase servers * and read directly from snapshot files. * diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java index 44d88c5..c40396f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormat.java @@ -18,19 +18,14 @@ package org.apache.hadoop.hbase.mapreduce; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.io.Writable; @@ -41,7 +36,12 @@ import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; -import com.google.common.annotations.VisibleForTesting; +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; /** * TableSnapshotInputFormat allows a MapReduce job to run over a table snapshot. The job @@ -98,8 +98,9 @@ public class TableSnapshotInputFormat extends InputFormat locations) { - this.delegate = new TableSnapshotInputFormatImpl.InputSplit(htd, regionInfo, locations); + List locations, Scan scan, Path restoreDir) { + this.delegate = + new TableSnapshotInputFormatImpl.InputSplit(htd, regionInfo, locations, scan, restoreDir); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java index 8496868..75c6fc5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatImpl.java @@ -18,8 +18,9 @@ package org.apache.hadoop.hbase.mapreduce; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; +import com.google.common.collect.Lists; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -28,6 +29,8 @@ import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.HDFSBlocksDistribution.HostAndWeight; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.client.ClientSideRegionScanner; import org.apache.hadoop.hbase.client.IsolationLevel; import org.apache.hadoop.hbase.client.Result; @@ -61,9 +64,11 @@ public class TableSnapshotInputFormatImpl { // TODO: Snapshots files are owned in fs by the hbase user. There is no // easy way to delegate access. + public static final Log LOG = LogFactory.getLog(TableSnapshotInputFormatImpl.class); + private static final String SNAPSHOT_NAME_KEY = "hbase.TableSnapshotInputFormat.snapshot.name"; // key for specifying the root dir of the restored snapshot - private static final String RESTORE_DIR_KEY = "hbase.TableSnapshotInputFormat.restore.dir"; + protected static final String RESTORE_DIR_KEY = "hbase.TableSnapshotInputFormat.restore.dir"; /** See {@link #getBestLocations(Configuration, HDFSBlocksDistribution)} */ private static final String LOCALITY_CUTOFF_MULTIPLIER = @@ -74,14 +79,18 @@ public class TableSnapshotInputFormatImpl { * Implementation class for InputSplit logic common between mapred and mapreduce. */ public static class InputSplit implements Writable { + private HTableDescriptor htd; private HRegionInfo regionInfo; private String[] locations; + private String scan; + private String restoreDir; // constructor for mapreduce framework / Writable public InputSplit() {} - public InputSplit(HTableDescriptor htd, HRegionInfo regionInfo, List locations) { + public InputSplit(HTableDescriptor htd, HRegionInfo regionInfo, List locations, + Scan scan, Path restoreDir) { this.htd = htd; this.regionInfo = regionInfo; if (locations == null || locations.isEmpty()) { @@ -89,6 +98,25 @@ public class TableSnapshotInputFormatImpl { } else { this.locations = locations.toArray(new String[locations.size()]); } + try { + this.scan = scan != null ? TableMapReduceUtil.convertScanToString(scan) : ""; + } catch (IOException e) { + LOG.warn("Failed to convert Scan to String", e); + } + + this.restoreDir = restoreDir.toString(); + } + + public HTableDescriptor getHtd() { + return htd; + } + + public String getScan() { + return scan; + } + + public String getRestoreDir() { + return restoreDir; } public long getLength() { @@ -128,6 +156,10 @@ public class TableSnapshotInputFormatImpl { byte[] buf = baos.toByteArray(); out.writeInt(buf.length); out.write(buf); + + Bytes.writeByteArray(out, Bytes.toBytes(scan)); + Bytes.writeByteArray(out, Bytes.toBytes(restoreDir)); + } @Override @@ -140,6 +172,9 @@ public class TableSnapshotInputFormatImpl { this.regionInfo = HRegionInfo.convert(split.getRegion()); List locationsList = split.getLocationsList(); this.locations = locationsList.toArray(new String[locationsList.size()]); + + this.scan = Bytes.toString(Bytes.readByteArray(in)); + this.restoreDir = Bytes.toString(Bytes.readByteArray(in)); } } @@ -158,28 +193,12 @@ public class TableSnapshotInputFormatImpl { } public void initialize(InputSplit split, Configuration conf) throws IOException { + this.scan = TableMapReduceUtil.convertStringToScan(split.getScan()); this.split = split; HTableDescriptor htd = split.htd; HRegionInfo hri = this.split.getRegionInfo(); FileSystem fs = FSUtils.getCurrentFileSystem(conf); - Path tmpRootDir = new Path(conf.get(RESTORE_DIR_KEY)); // This is the user specified root - // directory where snapshot was restored - - // create scan - // TODO: mapred does not support scan as input API. Work around for now. - if (conf.get(TableInputFormat.SCAN) != null) { - scan = TableMapReduceUtil.convertStringToScan(conf.get(TableInputFormat.SCAN)); - } else if (conf.get(org.apache.hadoop.hbase.mapred.TableInputFormat.COLUMN_LIST) != null) { - String[] columns = - conf.get(org.apache.hadoop.hbase.mapred.TableInputFormat.COLUMN_LIST).split(" "); - scan = new Scan(); - for (String col : columns) { - scan.addFamily(Bytes.toBytes(col)); - } - } else { - throw new IllegalArgumentException("A Scan is not configured for this job"); - } // region is immutable, this should be fine, // otherwise we have to set the thread read point @@ -187,7 +206,8 @@ public class TableSnapshotInputFormatImpl { // disable caching of data blocks scan.setCacheBlocks(false); - scanner = new ClientSideRegionScanner(conf, fs, tmpRootDir, htd, hri, scan, null); + scanner = + new ClientSideRegionScanner(conf, fs, new Path(split.restoreDir), htd, hri, scan, null); } public boolean nextKeyValue() throws IOException { @@ -233,18 +253,40 @@ public class TableSnapshotInputFormatImpl { Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); - Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); - SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); - SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); + SnapshotManifest manifest = getSnapshotManifest(conf, snapshotName, rootDir, fs); + + List regionInfos = getRegionInfosFromManifest(manifest); + + // TODO: mapred does not support scan as input API. Work around for now. + Scan scan = extractScanFromConf(conf); + // the temp dir where the snapshot is restored + Path restoreDir = new Path(conf.get(RESTORE_DIR_KEY)); + + return getSplits(scan, manifest, regionInfos, restoreDir, conf); + } + + public static List getRegionInfosFromManifest(SnapshotManifest manifest) { List regionManifests = manifest.getRegionManifests(); if (regionManifests == null) { throw new IllegalArgumentException("Snapshot seems empty"); } - // load table descriptor - HTableDescriptor htd = manifest.getTableDescriptor(); + List regionInfos = Lists.newArrayListWithCapacity(regionManifests.size()); - // TODO: mapred does not support scan as input API. Work around for now. + for (SnapshotRegionManifest regionManifest : regionManifests) { + regionInfos.add(HRegionInfo.convert(regionManifest.getRegionInfo())); + } + return regionInfos; + } + + public static SnapshotManifest getSnapshotManifest(Configuration conf, String snapshotName, + Path rootDir, FileSystem fs) throws IOException { + Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); + SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); + return SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); + } + + public static Scan extractScanFromConf(Configuration conf) throws IOException { Scan scan = null; if (conf.get(TableInputFormat.SCAN) != null) { scan = TableMapReduceUtil.convertStringToScan(conf.get(TableInputFormat.SCAN)); @@ -258,29 +300,35 @@ public class TableSnapshotInputFormatImpl { } else { throw new IllegalArgumentException("Unable to create scan"); } - // the temp dir where the snapshot is restored - Path restoreDir = new Path(conf.get(RESTORE_DIR_KEY)); + return scan; + } + + public static List getSplits(Scan scan, SnapshotManifest manifest, + List regionManifests, Path restoreDir, Configuration conf) throws IOException { + // load table descriptor + HTableDescriptor htd = manifest.getTableDescriptor(); + Path tableDir = FSUtils.getTableDir(restoreDir, htd.getTableName()); List splits = new ArrayList(); - for (SnapshotRegionManifest regionManifest : regionManifests) { + for (HRegionInfo hri : regionManifests) { // load region descriptor - HRegionInfo hri = HRegionInfo.convert(regionManifest.getRegionInfo()); - if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), - hri.getStartKey(), hri.getEndKey())) { + if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), hri.getStartKey(), + hri.getEndKey())) { // compute HDFS locations from snapshot files (which will get the locations for // referred hfiles) List hosts = getBestLocations(conf, - HRegion.computeHDFSBlocksDistribution(conf, htd, hri, tableDir)); + HRegion.computeHDFSBlocksDistribution(conf, htd, hri, tableDir)); int len = Math.min(3, hosts.size()); hosts = hosts.subList(0, len); - splits.add(new InputSplit(htd, hri, hosts)); + splits.add(new InputSplit(htd, hri, hosts, scan, restoreDir)); } } return splits; + } /** @@ -335,7 +383,7 @@ public class TableSnapshotInputFormatImpl { /** * Configures the job to use TableSnapshotInputFormat to read from a snapshot. - * @param conf the job to configuration + * @param conf the job to configure * @param snapshotName the name of the snapshot to read from * @param restoreDir a temporary directory to restore the snapshot into. Current user should * have write permissions to this directory, and this should not be a subdirectory of rootdir. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java index f94dac9..c91e7b0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java @@ -87,6 +87,7 @@ public class VerifyReplication extends Configured implements Tool { private ResultScanner replicatedScanner; private Result currentCompareRowInPeerTable; + private Table replicatedTable; /** * Map method that compares every scanned row with the equivalent from @@ -127,8 +128,7 @@ public class VerifyReplication extends Configured implements Tool { ZKUtil.applyClusterKeyToConf(peerConf, zkClusterKey); TableName tableName = TableName.valueOf(conf.get(NAME + ".tableName")); - // TODO: THis HTable doesn't get closed. Fix! - Table replicatedTable = new HTable(peerConf, tableName); + replicatedTable = new HTable(peerConf, tableName); scan.setStartRow(value.getRow()); scan.setStopRow(tableSplit.getEndRow()); replicatedScanner = replicatedTable.getScanner(scan); @@ -189,6 +189,14 @@ public class VerifyReplication extends Configured implements Tool { replicatedScanner = null; } } + if (replicatedTable != null) { + TableName tableName = replicatedTable.getName(); + try { + replicatedTable.close(); + } catch (IOException ioe) { + LOG.warn("Exception closing " + tableName, ioe); + } + } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index cd0e9d7..808a53f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -486,8 +486,7 @@ public class AssignmentManager extends ZooKeeperListener { Set deadServers = rebuildUserRegions(); // This method will assign all user regions if a clean server startup or - // it will reconstruct master state and cleanup any leftovers from - // previous master process. + // it will reconstruct master state and cleanup any leftovers from previous master process. boolean failover = processDeadServersAndRegionsInTransition(deadServers); if (!useZKForAssignment) { @@ -502,20 +501,18 @@ public class AssignmentManager extends ZooKeeperListener { /** * Process all regions that are in transition in zookeeper and also - * processes the list of dead servers by scanning the META. + * processes the list of dead servers. * Used by master joining an cluster. If we figure this is a clean cluster * startup, will assign all user regions. - * @param deadServers - * Map of dead servers and their regions. Can be null. + * @param deadServers Set of servers that are offline probably legitimately that were carrying + * regions according to a scan of hbase:meta. Can be null. * @throws KeeperException * @throws IOException * @throws InterruptedException */ - boolean processDeadServersAndRegionsInTransition( - final Set deadServers) throws KeeperException, - IOException, InterruptedException, CoordinatedStateException { - List nodes = ZKUtil.listChildrenNoWatch(watcher, - watcher.assignmentZNode); + boolean processDeadServersAndRegionsInTransition(final Set deadServers) + throws KeeperException, IOException, InterruptedException, CoordinatedStateException { + List nodes = ZKUtil.listChildrenNoWatch(watcher, watcher.assignmentZNode); if (useZKForAssignment && nodes == null) { String errorMessage = "Failed to get the children from ZK"; @@ -2755,15 +2752,13 @@ public class AssignmentManager extends ZooKeeperListener { } // Generate a round-robin bulk assignment plan - Map> bulkPlan - = balancer.roundRobinAssignment(regions, servers); + Map> bulkPlan = balancer.roundRobinAssignment(regions, servers); if (bulkPlan == null) { throw new IOException("Unable to determine a plan to assign region(s)"); } processFavoredNodes(regions); - assign(regions.size(), servers.size(), - "round-robin=true", bulkPlan); + assign(regions.size(), servers.size(), "round-robin=true", bulkPlan); } private void assign(int regions, int totalServers, @@ -2903,10 +2898,8 @@ public class AssignmentManager extends ZooKeeperListener { /** * Rebuild the list of user regions and assignment information. - *

- * Returns a set of servers that are not found to be online that hosted - * some regions. - * @return set of servers not online that hosted some regions per meta + * Updates regionstates with findings as we go through list of regions. + * @return set of servers not online that hosted some regions according to a scan of hbase:meta * @throws IOException */ Set rebuildUserRegions() throws @@ -3061,22 +3054,18 @@ public class AssignmentManager extends ZooKeeperListener { } /** - * Processes list of dead servers from result of hbase:meta scan and regions in RIT - *

+ * Processes list of dead servers from result of hbase:meta scan and regions in RIT. * This is used for failover to recover the lost regions that belonged to - * RegionServers which failed while there was no active master or regions - * that were in RIT. - *

- * + * RegionServers which failed while there was no active master or are offline for whatever + * reason and for regions that were in RIT. * * @param deadServers - * The list of dead servers which failed while there was no active - * master. Can be null. + * The list of dead servers which failed while there was no active master. Can be null. * @throws IOException * @throws KeeperException */ - private void processDeadServersAndRecoverLostRegions( - Set deadServers) throws IOException, KeeperException { + private void processDeadServersAndRecoverLostRegions(Set deadServers) + throws IOException, KeeperException { if (deadServers != null && !deadServers.isEmpty()) { for (ServerName serverName: deadServers) { if (!serverManager.isServerDead(serverName)) { @@ -3098,7 +3087,7 @@ public class AssignmentManager extends ZooKeeperListener { } void processRegionInTransitionZkLess() { - // We need to send RPC call again for PENDING_OPEN/PENDING_CLOSE regions + // We need to send RPC call again for PENDING_OPEN/PENDING_CLOSE regions // in case the RPC call is not sent out yet before the master was shut down // since we update the state before we send the RPC call. We can't update // the state after the RPC call. Otherwise, we don't know what's happened @@ -3403,15 +3392,15 @@ public class AssignmentManager extends ZooKeeperListener { } /** - * Process shutdown server removing any assignments. + * Clean out crashed server removing any assignments. * @param sn Server that went down. * @return list of regions in transition on this server */ - public List processServerShutdown(final ServerName sn) { + public List cleanOutCrashedServerReferences(final ServerName sn) { // Clean out any existing assignment plans for this server synchronized (this.regionPlans) { - for (Iterator > i = - this.regionPlans.entrySet().iterator(); i.hasNext();) { + for (Iterator > i = this.regionPlans.entrySet().iterator(); + i.hasNext();) { Map.Entry e = i.next(); ServerName otherSn = e.getValue().getDestination(); // The name will be null if the region is planned for a random assign. @@ -3429,8 +3418,7 @@ public class AssignmentManager extends ZooKeeperListener { // We need a lock on the region as we could update it Lock lock = locker.acquireLock(encodedName); try { - RegionState regionState = - regionStates.getRegionTransitionState(encodedName); + RegionState regionState = regionStates.getRegionTransitionState(encodedName); if (regionState == null || (regionState.getServerName() != null && !regionState.isOnServer(sn)) || !(regionState.isFailedClose() || regionState.isOffline() @@ -3635,8 +3623,7 @@ public class AssignmentManager extends ZooKeeperListener { } } - private void onRegionOpen( - final HRegionInfo hri, final ServerName sn, long openSeqNum) { + private void onRegionOpen(final HRegionInfo hri, final ServerName sn, long openSeqNum) { regionOnline(hri, sn, openSeqNum); if (useZKForAssignment) { try { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java index 83b12dd..8b16b00 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DeadServer.java @@ -38,6 +38,7 @@ import java.util.Set; /** * Class to hold dead servers list and utility querying dead server list. + * On znode expiration, servers are added here. */ @InterfaceAudience.Private public class DeadServer { @@ -115,7 +116,7 @@ public class DeadServer { } public synchronized void finish(ServerName sn) { - LOG.debug("Finished processing " + sn); + if (LOG.isDebugEnabled()) LOG.debug("Finished " + sn + "; numProcessing=" + this.numProcessing); this.numProcessing--; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 052cbcf..aaf288a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -269,7 +269,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { volatile boolean serviceStarted = false; // flag set after we complete assignMeta. - private volatile boolean serverShutdownHandlerEnabled = false; + private volatile boolean serverCrashProcessingEnabled = false; LoadBalancer balancer; private BalancerChore balancerChore; @@ -665,11 +665,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server { // get a list for previously failed RS which need log splitting work // we recover hbase:meta region servers inside master initialization and // handle other failed servers in SSH in order to start up master node ASAP - Set previouslyFailedServers = this.fileSystemManager - .getFailedServersFromLogFolders(); - - // remove stale recovering regions from previous run - this.fileSystemManager.removeStaleRecoveringRegionsFromZK(previouslyFailedServers); + Set previouslyFailedServers = + this.fileSystemManager.getFailedServersFromLogFolders(); // log splitting for hbase:meta server ServerName oldMetaServerLocation = metaTableLocator.getMetaRegionLocation(this.getZooKeeper()); @@ -703,14 +700,14 @@ public class HMaster extends HRegionServer implements MasterServices, Server { // Check if master is shutting down because of some issue // in initializing the regionserver or the balancer. - if(isStopped()) return; + if (isStopped()) return; // Make sure meta assigned before proceeding. status.setStatus("Assigning Meta Region"); assignMeta(status, previouslyFailedMetaRSs, HRegionInfo.DEFAULT_REPLICA_ID); // check if master is shutting down because above assignMeta could return even hbase:meta isn't // assigned when master is shutting down - if(isStopped()) return; + if (isStopped()) return; status.setStatus("Submitting log splitting work for previously failed region servers"); // Master has recovered hbase:meta region server and we put @@ -720,8 +717,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } // Update meta with new PB serialization if required. i.e migrate all HRI to PB serialization - // in meta. This must happen before we assign all user regions or else the assignment will - // fail. + // in meta. This must happen before we assign all user regions or else the assignment will fail. if (this.conf.getBoolean("hbase.MetaMigrationConvertingToPB", true)) { MetaMigrationConvertingToPB.updateMetaIfNecessary(this); } @@ -730,11 +726,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server { status.setStatus("Starting assignment manager"); this.assignmentManager.joinCluster(); - //set cluster status again after user regions are assigned + // set cluster status again after user regions are assigned this.balancer.setClusterStatus(getClusterStatus()); - // Start balancer and meta catalog janitor after meta and regions have - // been assigned. + // Start balancer and meta catalog janitor after meta and regions have been assigned. status.setStatus("Starting balancer and catalog janitor"); this.clusterStatusChore = new ClusterStatusChore(this, balancer); getChoreService().scheduleChore(clusterStatusChore); @@ -745,7 +740,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { status.setStatus("Starting namespace manager"); initNamespace(); - + if (this.cpHost != null) { try { this.cpHost.preMasterInitialization(); @@ -757,8 +752,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server { status.markComplete("Initialization successful"); LOG.info("Master has completed initialization"); configurationManager.registerObserver(this.balancer); + + // Set master as 'initialized'. initialized = true; - + status.setStatus("Starting quota manager"); initQuotaManager(); @@ -921,7 +918,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { // if the meta region server is died at this time, we need it to be re-assigned // by SSH so that system tables can be assigned. // No need to wait for meta is assigned = 0 when meta is just verified. - if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) enableServerShutdownHandler(assigned != 0); + if (replicaId == HRegionInfo.DEFAULT_REPLICA_ID) enableCrashedServerProcessing(assigned != 0); LOG.info("hbase:meta with replicaId " + replicaId + " assigned=" + assigned + ", rit=" + rit + ", location=" + metaTableLocator.getMetaRegionLocation(this.getZooKeeper(), replicaId)); status.setStatus("META assigned."); @@ -970,15 +967,14 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } } - private void enableServerShutdownHandler( - final boolean waitForMeta) throws IOException, InterruptedException { - // If ServerShutdownHandler is disabled, we enable it and expire those dead - // but not expired servers. This is required so that if meta is assigning to - // a server which dies after assignMeta starts assignment, - // SSH can re-assign it. Otherwise, we will be + private void enableCrashedServerProcessing(final boolean waitForMeta) + throws IOException, InterruptedException { + // If crashed server processing is disabled, we enable it and expire those dead but not expired + // servers. This is required so that if meta is assigning to a server which dies after + // assignMeta starts assignment, ServerCrashProcedure can re-assign it. Otherwise, we will be // stuck here waiting forever if waitForMeta is specified. - if (!serverShutdownHandlerEnabled) { - serverShutdownHandlerEnabled = true; + if (!serverCrashProcessingEnabled) { + serverCrashProcessingEnabled = true; this.serverManager.processQueuedDeadServers(); } @@ -1131,8 +1127,11 @@ public class HMaster extends HRegionServer implements MasterServices, Server { final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, Math.max(Runtime.getRuntime().availableProcessors(), MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS)); + final boolean abortOnCorruption = conf.getBoolean( + MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION, + MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION); procedureStore.start(numThreads); - procedureExecutor.start(numThreads); + procedureExecutor.start(numThreads, abortOnCorruption); } private void stopProcedureExecutor() { @@ -1499,6 +1498,15 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } // max versions already being checked + // HBASE-13776 Setting illegal versions for HColumnDescriptor + // does not throw IllegalArgumentException + // check minVersions <= maxVerions + if (hcd.getMinVersions() > hcd.getMaxVersions()) { + String message = "Min versions for column family " + hcd.getNameAsString() + + " must be less than the Max versions."; + warnOrThrowExceptionForFailure(logWarn, CONF_KEY, message, null); + } + // check replication scope if (hcd.getScope() < 0) { String message = "Replication scope for column family " @@ -1570,7 +1578,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server { LOG.fatal("Failed to become active master", t); // HBASE-5680: Likely hadoop23 vs hadoop 20.x/1.x incompatibility if (t instanceof NoClassDefFoundError && - t.getMessage().contains("org/apache/hadoop/hdfs/protocol/FSConstants$SafeModeAction")) { + t.getMessage() + .contains("org/apache/hadoop/hdfs/protocol/HdfsConstants$SafeModeAction")) { // improved error message for this special case abort("HBase is having a problem with its Hadoop jars. You may need to " + "recompile HBase against Hadoop version " @@ -2091,13 +2100,18 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } /** - * ServerShutdownHandlerEnabled is set false before completing - * assignMeta to prevent processing of ServerShutdownHandler. + * ServerCrashProcessingEnabled is set false before completing assignMeta to prevent processing + * of crashed servers. * @return true if assignMeta has completed; */ @Override - public boolean isServerShutdownHandlerEnabled() { - return this.serverShutdownHandlerEnabled; + public boolean isServerCrashProcessingEnabled() { + return this.serverCrashProcessingEnabled; + } + + @VisibleForTesting + public void setServerCrashProcessingEnabled(final boolean b) { + this.serverCrashProcessingEnabled = b; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java index 1b76cd4..bcf9ba0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java @@ -58,6 +58,8 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; +import com.google.common.annotations.VisibleForTesting; + /** * This class abstracts a bunch of operations the HMaster needs to interact with * the underlying file system, including splitting log files, checking file @@ -131,6 +133,11 @@ public class MasterFileSystem { this.distributedLogReplay = this.splitLogManager.isLogReplaying(); } + @VisibleForTesting + SplitLogManager getSplitLogManager() { + return this.splitLogManager; + } + /** * Create initial layout in filesystem. *

    diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 7a30bac..b2e8306 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.errorhandling.ForeignException; import org.apache.hadoop.hbase.exceptions.MergeRegionException; import org.apache.hadoop.hbase.exceptions.UnknownProtocolException; import org.apache.hadoop.hbase.ipc.QosPriority; @@ -667,6 +668,8 @@ public class MasterRpcServices extends RSRpcServices long waitTime = SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME; return ExecProcedureResponse.newBuilder().setExpectedTimeout( waitTime).build(); + } catch (ForeignException e) { + throw new ServiceException(e.getCause()); } catch (IOException e) { throw new ServiceException(e); } @@ -900,6 +903,8 @@ public class MasterRpcServices extends RSRpcServices boolean done = mpm.isProcedureDone(desc); builder.setDone(done); return builder.build(); + } catch (ForeignException e) { + throw new ServiceException(e.getCause()); } catch (IOException e) { throw new ServiceException(e); } @@ -925,6 +930,8 @@ public class MasterRpcServices extends RSRpcServices boolean done = master.snapshotManager.isRestoreDone(snapshot); builder.setDone(done); return builder.build(); + } catch (ForeignException e) { + throw new ServiceException(e.getCause()); } catch (IOException e) { throw new ServiceException(e); } @@ -948,6 +955,8 @@ public class MasterRpcServices extends RSRpcServices boolean done = master.snapshotManager.isSnapshotDone(request.getSnapshot()); builder.setDone(done); return builder.build(); + } catch (ForeignException e) { + throw new ServiceException(e.getCause()); } catch (IOException e) { throw new ServiceException(e); } @@ -1160,6 +1169,8 @@ public class MasterRpcServices extends RSRpcServices SnapshotDescription reqSnapshot = request.getSnapshot(); master.snapshotManager.restoreSnapshot(reqSnapshot); return RestoreSnapshotResponse.newBuilder().build(); + } catch (ForeignException e) { + throw new ServiceException(e.getCause()); } catch (IOException e) { throw new ServiceException(e); } @@ -1219,6 +1230,8 @@ public class MasterRpcServices extends RSRpcServices long waitTime = SnapshotDescriptionUtils.getMaxMasterTimeout(master.getConfiguration(), snapshot.getType(), SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME); return SnapshotResponse.newBuilder().setExpectedTimeout(waitTime).build(); + } catch (ForeignException e) { + throw new ServiceException(e.getCause()); } catch (IOException e) { throw new ServiceException(e); } @@ -1341,7 +1354,7 @@ public class MasterRpcServices extends RSRpcServices response.setEnabled(master.isBalancerOn()); return response.build(); } - + @Override public SetQuotaResponse setQuota(RpcController c, SetQuotaRequest req) throws ServiceException { try { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java index 59a078e..dd64bc8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java @@ -177,7 +177,7 @@ public interface MasterServices extends Server { /** * @return true if master enables ServerShutdownHandler; */ - boolean isServerShutdownHandlerEnabled(); + boolean isServerCrashProcessingEnabled(); /** * Registers a new protocol buffer {@link Service} subclass as a master coprocessor endpoint. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java index 78097ac..c658475 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java @@ -31,20 +31,19 @@ import java.util.TreeMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.RegionTransition; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.RegionTransition; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableStateManager; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.RegionReplicaUtil; -import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.util.Bytes; @@ -428,8 +427,7 @@ public class RegionStates { return updateRegionState(hri, state, serverName, HConstants.NO_SEQNUM); } - public void regionOnline( - final HRegionInfo hri, final ServerName serverName) { + public void regionOnline(final HRegionInfo hri, final ServerName serverName) { regionOnline(hri, serverName, HConstants.NO_SEQNUM); } @@ -438,16 +436,14 @@ public class RegionStates { * We can't confirm it is really online on specified region server * because it hasn't been put in region server's online region list yet. */ - public void regionOnline(final HRegionInfo hri, - final ServerName serverName, long openSeqNum) { + public void regionOnline(final HRegionInfo hri, final ServerName serverName, long openSeqNum) { String encodedName = hri.getEncodedName(); if (!serverManager.isServerOnline(serverName)) { // This is possible if the region server dies before master gets a // chance to handle ZK event in time. At this time, if the dead server // is already processed by SSH, we should ignore this event. // If not processed yet, ignore and let SSH deal with it. - LOG.warn("Ignored, " + encodedName - + " was opened on a dead server: " + serverName); + LOG.warn("Ignored, " + encodedName + " was opened on a dead server: " + serverName); return; } updateRegionState(hri, State.OPEN, serverName, openSeqNum); @@ -457,7 +453,7 @@ public class RegionStates { ServerName oldServerName = regionAssignments.put(hri, serverName); if (!serverName.equals(oldServerName)) { if (LOG.isDebugEnabled()) { - LOG.debug("Onlined " + hri.getShortNameToLog() + " on " + serverName + " " + hri); + LOG.debug("Onlined " + hri.getShortNameToLog() + " on " + serverName); } else { LOG.debug("Onlined " + hri.getShortNameToLog() + " on " + serverName); } @@ -529,7 +525,7 @@ public class RegionStates { } long now = System.currentTimeMillis(); if (LOG.isDebugEnabled()) { - LOG.debug("Adding to processed servers " + serverName); + LOG.debug("Adding to log splitting servers " + serverName); } processedServers.put(serverName, Long.valueOf(now)); Configuration conf = server.getConfiguration(); @@ -543,7 +539,7 @@ public class RegionStates { Map.Entry e = it.next(); if (e.getValue().longValue() < cutoff) { if (LOG.isDebugEnabled()) { - LOG.debug("Removed from processed servers " + e.getKey()); + LOG.debug("Removed from log splitting servers " + e.getKey()); } it.remove(); } @@ -648,7 +644,7 @@ public class RegionStates { // Region is open on this region server, but in transition. // This region must be moving away from this server, or splitting/merging. // SSH will handle it, either skip assigning, or re-assign. - LOG.info("Transitioning " + state + " will be handled by SSH for " + sn); + LOG.info("Transitioning " + state + " will be handled by ServerCrashProcedure for " + sn); } else if (sn.equals(state.getServerName())) { // Region is in transition on this region server, and this // region is not open on this server. So the region must be @@ -658,7 +654,8 @@ public class RegionStates { // transition. The region could be in failed_close state too if we have // tried several times to open it while this region server is not reachable) if (state.isPendingOpenOrOpening() || state.isFailedClose() || state.isOffline()) { - LOG.info("Found region in " + state + " to be reassigned by SSH for " + sn); + LOG.info("Found region in " + state + + " to be reassigned by ServerCrashProcedure for " + sn); rits.add(hri); } else if(state.isSplittingNew()) { regionsToCleanIfNoMetaEntry.add(state.getRegion()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index 5c8bd34..4e3796c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -53,8 +53,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; import org.apache.hadoop.hbase.client.RetriesExhaustedException; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; -import org.apache.hadoop.hbase.master.handler.MetaServerShutdownHandler; -import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler; +import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.RequestConverter; @@ -123,9 +122,15 @@ public class ServerManager { // Set if we are to shutdown the cluster. private volatile boolean clusterShutdown = false; + /** + * The last flushed sequence id for a region. + */ private final ConcurrentNavigableMap flushedSequenceIdByRegion = new ConcurrentSkipListMap(Bytes.BYTES_COMPARATOR); + /** + * The last flushed sequence id for a store in a region. + */ private final ConcurrentNavigableMap> storeFlushedSequenceIdsByRegion = new ConcurrentSkipListMap>(Bytes.BYTES_COMPARATOR); @@ -581,7 +586,7 @@ public class ServerManager { } return; } - if (!services.isServerShutdownHandlerEnabled()) { + if (!services.isServerCrashProcessingEnabled()) { LOG.info("Master doesn't enable ServerShutdownHandler during initialization, " + "delay expiring server " + serverName); this.queuedDeadServers.add(serverName); @@ -593,18 +598,8 @@ public class ServerManager { " but server shutdown already in progress"); return; } - synchronized (onlineServers) { - if (!this.onlineServers.containsKey(serverName)) { - LOG.warn("Expiration of " + serverName + " but server not online"); - } - // Remove the server from the known servers lists and update load info BUT - // add to deadservers first; do this so it'll show in dead servers list if - // not in online servers list. - this.deadservers.add(serverName); - this.onlineServers.remove(serverName); - onlineServers.notifyAll(); - } - this.rsAdmins.remove(serverName); + moveFromOnelineToDeadServers(serverName); + // If cluster is going down, yes, servers are going to be expiring; don't // process as a dead server if (this.clusterShutdown) { @@ -617,13 +612,8 @@ public class ServerManager { } boolean carryingMeta = services.getAssignmentManager().isCarryingMeta(serverName); - if (carryingMeta) { - this.services.getExecutorService().submit(new MetaServerShutdownHandler(this.master, - this.services, this.deadservers, serverName)); - } else { - this.services.getExecutorService().submit(new ServerShutdownHandler(this.master, - this.services, this.deadservers, serverName, true)); - } + this.services.getMasterProcedureExecutor(). + submitProcedure(new ServerCrashProcedure(serverName, true, carryingMeta)); LOG.debug("Added=" + serverName + " to dead servers, submitted shutdown handler to be executed meta=" + carryingMeta); @@ -635,8 +625,20 @@ public class ServerManager { } } - public synchronized void processDeadServer(final ServerName serverName) { - this.processDeadServer(serverName, false); + @VisibleForTesting + public void moveFromOnelineToDeadServers(final ServerName sn) { + synchronized (onlineServers) { + if (!this.onlineServers.containsKey(sn)) { + LOG.warn("Expiration of " + sn + " but server not online"); + } + // Remove the server from the known servers lists and update load info BUT + // add to deadservers first; do this so it'll show in dead servers list if + // not in online servers list. + this.deadservers.add(sn); + this.onlineServers.remove(sn); + onlineServers.notifyAll(); + } + this.rsAdmins.remove(sn); } public synchronized void processDeadServer(final ServerName serverName, boolean shouldSplitWal) { @@ -654,9 +656,8 @@ public class ServerManager { } this.deadservers.add(serverName); - this.services.getExecutorService().submit( - new ServerShutdownHandler(this.master, this.services, this.deadservers, serverName, - shouldSplitWal)); + this.services.getMasterProcedureExecutor(). + submitProcedure(new ServerCrashProcedure(serverName, shouldSplitWal, false)); } /** @@ -664,7 +665,7 @@ public class ServerManager { * called after HMaster#assignMeta and AssignmentManager#joinCluster. * */ synchronized void processQueuedDeadServers() { - if (!services.isServerShutdownHandlerEnabled()) { + if (!services.isServerCrashProcessingEnabled()) { LOG.info("Master hasn't enabled ServerShutdownHandler"); } Iterator serverIterator = queuedDeadServers.iterator(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java index a716369..3fc95cc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java @@ -405,16 +405,15 @@ public class SplitLogManager { // the function is only used in WALEdit direct replay mode return; } + if (serverNames == null || serverNames.isEmpty()) return; Set recoveredServerNameSet = new HashSet(); - if (serverNames != null) { - for (ServerName tmpServerName : serverNames) { - recoveredServerNameSet.add(tmpServerName.getServerName()); - } + for (ServerName tmpServerName : serverNames) { + recoveredServerNameSet.add(tmpServerName.getServerName()); } - + + this.recoveringRegionLock.lock(); try { - this.recoveringRegionLock.lock(); ((BaseCoordinatedStateManager) server.getCoordinatedStateManager()) .getSplitLogManagerCoordination().removeRecoveringRegions(recoveredServerNameSet, isMetaRecovery); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableLockManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableLockManager.java index cfaeb98..ef1e84f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableLockManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableLockManager.java @@ -25,17 +25,16 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.InterProcessLock; import org.apache.hadoop.hbase.InterProcessLock.MetadataHandler; import org.apache.hadoop.hbase.InterProcessReadWriteLock; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.exceptions.LockTimeoutException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; -import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/LogReplayHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/LogReplayHandler.java deleted file mode 100644 index 18e564a..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/LogReplayHandler.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Copyright The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package org.apache.hadoop.hbase.master.handler; - -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.Server; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.executor.EventHandler; -import org.apache.hadoop.hbase.executor.EventType; -import org.apache.hadoop.hbase.master.DeadServer; -import org.apache.hadoop.hbase.master.MasterServices; - -/** - * Handle logReplay work from SSH. Having a separate handler is not to block SSH in re-assigning - * regions from dead servers. Otherwise, available SSH handlers could be blocked by logReplay work - * (from {@link org.apache.hadoop.hbase.master.MasterFileSystem#splitLog(ServerName)}). - * During logReplay, if a receiving RS(say A) fails again, regions on A won't be able to be - * assigned to another live RS which causes the log replay unable to complete because WAL edits - * replay depends on receiving RS to be live - */ -@InterfaceAudience.Private -public class LogReplayHandler extends EventHandler { - private static final Log LOG = LogFactory.getLog(LogReplayHandler.class); - private final ServerName serverName; - protected final Server master; - protected final MasterServices services; - protected final DeadServer deadServers; - - public LogReplayHandler(final Server server, final MasterServices services, - final DeadServer deadServers, final ServerName serverName) { - super(server, EventType.M_LOG_REPLAY); - this.master = server; - this.services = services; - this.deadServers = deadServers; - this.serverName = serverName; - this.deadServers.add(serverName); - } - - @Override - public String toString() { - String name = serverName.toString(); - return getClass().getSimpleName() + "-" + name + "-" + getSeqid(); - } - - @Override - public void process() throws IOException { - try { - if (this.master != null && this.master.isStopped()) { - // we're exiting ... - return; - } - this.services.getMasterFileSystem().splitLog(serverName); - } catch (Exception ex) { - if (ex instanceof IOException) { - // resubmit log replay work when failed - this.services.getExecutorService().submit((LogReplayHandler) this); - this.deadServers.add(serverName); - throw new IOException("failed log replay for " + serverName + ", will retry", ex); - } else { - throw new IOException(ex); - } - } finally { - this.deadServers.finish(serverName); - } - // logReplay is the last step of SSH so log a line to indicate that - LOG.info("Finished processing shutdown of " + serverName); - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java deleted file mode 100644 index 409ac5e..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java +++ /dev/null @@ -1,222 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.handler; - -import java.io.IOException; -import java.io.InterruptedIOException; -import java.util.HashSet; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.Server; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.executor.EventType; -import org.apache.hadoop.hbase.master.AssignmentManager; -import org.apache.hadoop.hbase.master.DeadServer; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; -import org.apache.hadoop.hbase.util.Threads; -import org.apache.zookeeper.KeeperException; - -import com.google.common.annotations.VisibleForTesting; - -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Shutdown handler for the server hosting hbase:meta - */ -@InterfaceAudience.Private -public class MetaServerShutdownHandler extends ServerShutdownHandler { - private static final Log LOG = LogFactory.getLog(MetaServerShutdownHandler.class); - private AtomicInteger eventExceptionCount = new AtomicInteger(0); - @VisibleForTesting - static final int SHOW_STRACKTRACE_FREQUENCY = 100; - - public MetaServerShutdownHandler(final Server server, - final MasterServices services, - final DeadServer deadServers, final ServerName serverName) { - super(server, services, deadServers, serverName, - EventType.M_META_SERVER_SHUTDOWN, true); - } - - @Override - public void process() throws IOException { - boolean gotException = true; - try { - AssignmentManager am = this.services.getAssignmentManager(); - this.services.getMasterFileSystem().setLogRecoveryMode(); - boolean distributedLogReplay = - (this.services.getMasterFileSystem().getLogRecoveryMode() == RecoveryMode.LOG_REPLAY); - try { - if (this.shouldSplitWal) { - LOG.info("Splitting hbase:meta logs for " + serverName); - if (distributedLogReplay) { - Set regions = new HashSet(); - regions.add(HRegionInfo.FIRST_META_REGIONINFO); - this.services.getMasterFileSystem().prepareLogReplay(serverName, regions); - } else { - this.services.getMasterFileSystem().splitMetaLog(serverName); - } - am.getRegionStates().logSplit(HRegionInfo.FIRST_META_REGIONINFO); - } - } catch (IOException ioe) { - this.services.getExecutorService().submit(this); - this.deadServers.add(serverName); - throw new IOException("failed log splitting for " + serverName + ", will retry", ioe); - } - - // Assign meta if we were carrying it. - // Check again: region may be assigned to other where because of RIT - // timeout - if (am.isCarryingMeta(serverName)) { - LOG.info("Server " + serverName + " was carrying META. Trying to assign."); - am.regionOffline(HRegionInfo.FIRST_META_REGIONINFO); - verifyAndAssignMetaWithRetries(); - } else if (!server.getMetaTableLocator().isLocationAvailable(this.server.getZooKeeper())) { - // the meta location as per master is null. This could happen in case when meta assignment - // in previous run failed, while meta znode has been updated to null. We should try to - // assign the meta again. - verifyAndAssignMetaWithRetries(); - } else { - LOG.info("META has been assigned to otherwhere, skip assigning."); - } - - try { - if (this.shouldSplitWal && distributedLogReplay) { - if (!am.waitOnRegionToClearRegionsInTransition(HRegionInfo.FIRST_META_REGIONINFO, - regionAssignmentWaitTimeout)) { - // Wait here is to avoid log replay hits current dead server and incur a RPC timeout - // when replay happens before region assignment completes. - LOG.warn("Region " + HRegionInfo.FIRST_META_REGIONINFO.getEncodedName() - + " didn't complete assignment in time"); - } - this.services.getMasterFileSystem().splitMetaLog(serverName); - } - } catch (Exception ex) { - if (ex instanceof IOException) { - this.services.getExecutorService().submit(this); - this.deadServers.add(serverName); - throw new IOException("failed log splitting for " + serverName + ", will retry", ex); - } else { - throw new IOException(ex); - } - } - - gotException = false; - } finally { - if (gotException){ - // If we had an exception, this.deadServers.finish will be skipped in super.process() - this.deadServers.finish(serverName); - } - } - - super.process(); - // Clear this counter on successful handling. - this.eventExceptionCount.set(0); - } - - @Override - boolean isCarryingMeta() { - return true; - } - - /** - * Before assign the hbase:meta region, ensure it haven't - * been assigned by other place - *

    - * Under some scenarios, the hbase:meta region can be opened twice, so it seemed online - * in two regionserver at the same time. - * If the hbase:meta region has been assigned, so the operation can be canceled. - * @throws InterruptedException - * @throws IOException - * @throws KeeperException - */ - private void verifyAndAssignMeta() - throws InterruptedException, IOException, KeeperException { - long timeout = this.server.getConfiguration(). - getLong("hbase.catalog.verification.timeout", 1000); - if (!server.getMetaTableLocator().verifyMetaRegionLocation(server.getConnection(), - this.server.getZooKeeper(), timeout)) { - this.services.getAssignmentManager().assignMeta(HRegionInfo.FIRST_META_REGIONINFO); - } else if (serverName.equals(server.getMetaTableLocator().getMetaRegionLocation( - this.server.getZooKeeper()))) { - throw new IOException("hbase:meta is onlined on the dead server " - + serverName); - } else { - LOG.info("Skip assigning hbase:meta, because it is online on the " - + server.getMetaTableLocator().getMetaRegionLocation(this.server.getZooKeeper())); - } - } - - /** - * Failed many times, shutdown processing - * @throws IOException - */ - private void verifyAndAssignMetaWithRetries() throws IOException { - int iTimes = this.server.getConfiguration().getInt( - "hbase.catalog.verification.retries", 10); - - long waitTime = this.server.getConfiguration().getLong( - "hbase.catalog.verification.timeout", 1000); - - int iFlag = 0; - while (true) { - try { - verifyAndAssignMeta(); - break; - } catch (KeeperException e) { - this.server.abort("In server shutdown processing, assigning meta", e); - throw new IOException("Aborting", e); - } catch (Exception e) { - if (iFlag >= iTimes) { - this.server.abort("verifyAndAssignMeta failed after" + iTimes - + " times retries, aborting", e); - throw new IOException("Aborting", e); - } - try { - Thread.sleep(waitTime); - } catch (InterruptedException e1) { - LOG.warn("Interrupted when is the thread sleep", e1); - Thread.currentThread().interrupt(); - throw (InterruptedIOException)new InterruptedIOException().initCause(e1); - } - iFlag++; - } - } - } - - @Override - protected void handleException(Throwable t) { - int count = eventExceptionCount.getAndIncrement(); - if (count < 0) count = eventExceptionCount.getAndSet(0); - if (count > SHOW_STRACKTRACE_FREQUENCY) { // Too frequent, let's slow reporting - Threads.sleep(1000); - } - if (count % SHOW_STRACKTRACE_FREQUENCY == 0) { - LOG.error("Caught " + eventType + ", count=" + this.eventExceptionCount, t); - } else { - LOG.error("Caught " + eventType + ", count=" + this.eventExceptionCount + - "; " + t.getMessage() + "; stack trace shows every " + SHOW_STRACKTRACE_FREQUENCY + - "th time."); - } - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java deleted file mode 100644 index 7789ee1..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java +++ /dev/null @@ -1,399 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master.handler; - -import java.io.IOException; -import java.io.InterruptedIOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.concurrent.locks.Lock; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.Server; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.client.RegionReplicaUtil; -import org.apache.hadoop.hbase.executor.EventHandler; -import org.apache.hadoop.hbase.executor.EventType; -import org.apache.hadoop.hbase.master.AssignmentManager; -import org.apache.hadoop.hbase.master.DeadServer; -import org.apache.hadoop.hbase.master.MasterFileSystem; -import org.apache.hadoop.hbase.master.MasterServices; -import org.apache.hadoop.hbase.master.RegionState; -import org.apache.hadoop.hbase.master.RegionState.State; -import org.apache.hadoop.hbase.master.RegionStates; -import org.apache.hadoop.hbase.master.ServerManager; -import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer; -import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; -import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; -import org.apache.hadoop.hbase.util.ConfigUtil; -import org.apache.hadoop.hbase.zookeeper.ZKAssign; -import org.apache.zookeeper.KeeperException; - -/** - * Process server shutdown. - * Server-to-handle must be already in the deadservers lists. See - * {@link ServerManager#expireServer(ServerName)} - */ -@InterfaceAudience.Private -public class ServerShutdownHandler extends EventHandler { - private static final Log LOG = LogFactory.getLog(ServerShutdownHandler.class); - protected final ServerName serverName; - protected final MasterServices services; - protected final DeadServer deadServers; - protected final boolean shouldSplitWal; // whether to split WAL or not - protected final int regionAssignmentWaitTimeout; - - public ServerShutdownHandler(final Server server, final MasterServices services, - final DeadServer deadServers, final ServerName serverName, - final boolean shouldSplitWal) { - this(server, services, deadServers, serverName, EventType.M_SERVER_SHUTDOWN, - shouldSplitWal); - } - - ServerShutdownHandler(final Server server, final MasterServices services, - final DeadServer deadServers, final ServerName serverName, EventType type, - final boolean shouldSplitWal) { - super(server, type); - this.serverName = serverName; - this.server = server; - this.services = services; - this.deadServers = deadServers; - if (!this.deadServers.isDeadServer(this.serverName)) { - LOG.warn(this.serverName + " is NOT in deadservers; it should be!"); - } - this.shouldSplitWal = shouldSplitWal; - this.regionAssignmentWaitTimeout = server.getConfiguration().getInt( - HConstants.LOG_REPLAY_WAIT_REGION_TIMEOUT, 15000); - } - - @Override - public String getInformativeName() { - if (serverName != null) { - return this.getClass().getSimpleName() + " for " + serverName; - } else { - return super.getInformativeName(); - } - } - - /** - * @return True if the server we are processing was carrying hbase:meta - */ - boolean isCarryingMeta() { - return false; - } - - @Override - public String toString() { - return getClass().getSimpleName() + "-" + serverName + "-" + getSeqid(); - } - - @Override - public void process() throws IOException { - boolean hasLogReplayWork = false; - final ServerName serverName = this.serverName; - try { - - // We don't want worker thread in the MetaServerShutdownHandler - // executor pool to block by waiting availability of hbase:meta - // Otherwise, it could run into the following issue: - // 1. The current MetaServerShutdownHandler instance For RS1 waits for the hbase:meta - // to come online. - // 2. The newly assigned hbase:meta region server RS2 was shutdown right after - // it opens the hbase:meta region. So the MetaServerShutdownHandler - // instance For RS1 will still be blocked. - // 3. The new instance of MetaServerShutdownHandler for RS2 is queued. - // 4. The newly assigned hbase:meta region server RS3 was shutdown right after - // it opens the hbase:meta region. So the MetaServerShutdownHandler - // instance For RS1 and RS2 will still be blocked. - // 5. The new instance of MetaServerShutdownHandler for RS3 is queued. - // 6. Repeat until we run out of MetaServerShutdownHandler worker threads - // The solution here is to resubmit a ServerShutdownHandler request to process - // user regions on that server so that MetaServerShutdownHandler - // executor pool is always available. - // - // If AssignmentManager hasn't finished rebuilding user regions, - // we are not ready to assign dead regions either. So we re-queue up - // the dead server for further processing too. - AssignmentManager am = services.getAssignmentManager(); - ServerManager serverManager = services.getServerManager(); - if (isCarryingMeta() /* hbase:meta */ || !am.isFailoverCleanupDone()) { - serverManager.processDeadServer(serverName, this.shouldSplitWal); - return; - } - - // Wait on meta to come online; we need it to progress. - // TODO: Best way to hold strictly here? We should build this retry logic - // into the MetaTableAccessor operations themselves. - // TODO: Is the reading of hbase:meta necessary when the Master has state of - // cluster in its head? It should be possible to do without reading hbase:meta - // in all but one case. On split, the RS updates the hbase:meta - // table and THEN informs the master of the split via zk nodes in - // 'unassigned' dir. Currently the RS puts ephemeral nodes into zk so if - // the regionserver dies, these nodes do not stick around and this server - // shutdown processing does fixup (see the fixupDaughters method below). - // If we wanted to skip the hbase:meta scan, we'd have to change at least the - // final SPLIT message to be permanent in zk so in here we'd know a SPLIT - // completed (zk is updated after edits to hbase:meta have gone in). See - // {@link SplitTransaction}. We'd also have to be figure another way for - // doing the below hbase:meta daughters fixup. - Set hris = null; - while (!this.server.isStopped()) { - try { - server.getMetaTableLocator().waitMetaRegionLocation(server.getZooKeeper()); - if (BaseLoadBalancer.tablesOnMaster(server.getConfiguration())) { - while (!this.server.isStopped() && serverManager.countOfRegionServers() < 2) { - // Wait till at least another regionserver is up besides the active master - // so that we don't assign all regions to the active master. - // This is best of efforts, because newly joined regionserver - // could crash right after that. - Thread.sleep(100); - } - } - // Skip getting user regions if the server is stopped. - if (!this.server.isStopped()) { - if (ConfigUtil.useZKForAssignment(server.getConfiguration())) { - hris = MetaTableAccessor.getServerUserRegions(this.server.getConnection(), - this.serverName).keySet(); - } else { - // Not using ZK for assignment, regionStates has everything we want - hris = am.getRegionStates().getServerRegions(serverName); - } - } - break; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw (InterruptedIOException)new InterruptedIOException().initCause(e); - } catch (IOException ioe) { - LOG.info("Received exception accessing hbase:meta during server shutdown of " + - serverName + ", retrying hbase:meta read", ioe); - } - } - if (this.server.isStopped()) { - throw new IOException("Server is stopped"); - } - - // delayed to set recovery mode based on configuration only after all outstanding splitlogtask - // drained - this.services.getMasterFileSystem().setLogRecoveryMode(); - boolean distributedLogReplay = - (this.services.getMasterFileSystem().getLogRecoveryMode() == RecoveryMode.LOG_REPLAY); - - try { - if (this.shouldSplitWal) { - if (distributedLogReplay) { - LOG.info("Mark regions in recovery for crashed server " + serverName + - " before assignment; regions=" + hris); - MasterFileSystem mfs = this.services.getMasterFileSystem(); - mfs.prepareLogReplay(serverName, hris); - } else { - LOG.info("Splitting logs for " + serverName + - " before assignment; region count=" + (hris == null ? 0 : hris.size())); - this.services.getMasterFileSystem().splitLog(serverName); - } - am.getRegionStates().logSplit(serverName); - } else { - LOG.info("Skipping log splitting for " + serverName); - } - } catch (IOException ioe) { - resubmit(serverName, ioe); - } - List toAssignRegions = new ArrayList(); - int replicaCount = services.getConfiguration().getInt(HConstants.META_REPLICAS_NUM, - HConstants.DEFAULT_META_REPLICA_NUM); - for (int i = 1; i < replicaCount; i++) { - HRegionInfo metaHri = - RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO, i); - if (am.isCarryingMetaReplica(serverName, metaHri)) { - LOG.info("Reassigning meta replica" + metaHri + " that was on " + serverName); - toAssignRegions.add(metaHri); - } - } - // Clean out anything in regions in transition. Being conservative and - // doing after log splitting. Could do some states before -- OPENING? - // OFFLINE? -- and then others after like CLOSING that depend on log - // splitting. - List regionsInTransition = am.processServerShutdown(serverName); - LOG.info("Reassigning " + ((hris == null)? 0: hris.size()) + - " region(s) that " + (serverName == null? "null": serverName) + - " was carrying (and " + regionsInTransition.size() + - " regions(s) that were opening on this server)"); - - toAssignRegions.addAll(regionsInTransition); - - // Iterate regions that were on this server and assign them - if (hris != null && !hris.isEmpty()) { - RegionStates regionStates = am.getRegionStates(); - for (HRegionInfo hri: hris) { - if (regionsInTransition.contains(hri)) { - continue; - } - String encodedName = hri.getEncodedName(); - Lock lock = am.acquireRegionLock(encodedName); - try { - RegionState rit = regionStates.getRegionTransitionState(hri); - if (processDeadRegion(hri, am)) { - ServerName addressFromAM = regionStates.getRegionServerOfRegion(hri); - if (addressFromAM != null && !addressFromAM.equals(this.serverName)) { - // If this region is in transition on the dead server, it must be - // opening or pending_open, which should have been covered by AM#processServerShutdown - LOG.info("Skip assigning region " + hri.getRegionNameAsString() - + " because it has been opened in " + addressFromAM.getServerName()); - continue; - } - if (rit != null) { - if (rit.getServerName() != null && !rit.isOnServer(serverName)) { - // Skip regions that are in transition on other server - LOG.info("Skip assigning region in transition on other server" + rit); - continue; - } - try{ - //clean zk node - LOG.info("Reassigning region with rs = " + rit + " and deleting zk node if exists"); - ZKAssign.deleteNodeFailSilent(services.getZooKeeper(), hri); - regionStates.updateRegionState(hri, State.OFFLINE); - } catch (KeeperException ke) { - this.server.abort("Unexpected ZK exception deleting unassigned node " + hri, ke); - return; - } - } else if (regionStates.isRegionInState( - hri, State.SPLITTING_NEW, State.MERGING_NEW)) { - regionStates.updateRegionState(hri, State.OFFLINE); - } - toAssignRegions.add(hri); - } else if (rit != null) { - if ((rit.isPendingCloseOrClosing() || rit.isOffline()) - && am.getTableStateManager().isTableState(hri.getTable(), - ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING) || - am.getReplicasToClose().contains(hri)) { - // If the table was partially disabled and the RS went down, we should clear the RIT - // and remove the node for the region. - // The rit that we use may be stale in case the table was in DISABLING state - // but though we did assign we will not be clearing the znode in CLOSING state. - // Doing this will have no harm. See HBASE-5927 - regionStates.updateRegionState(hri, State.OFFLINE); - am.deleteClosingOrClosedNode(hri, rit.getServerName()); - am.offlineDisabledRegion(hri); - } else { - LOG.warn("THIS SHOULD NOT HAPPEN: unexpected region in transition " - + rit + " not to be assigned by SSH of server " + serverName); - } - } - } finally { - lock.unlock(); - } - } - } - - try { - am.assign(toAssignRegions); - } catch (InterruptedException ie) { - LOG.error("Caught " + ie + " during round-robin assignment"); - throw (InterruptedIOException)new InterruptedIOException().initCause(ie); - } catch (IOException ioe) { - LOG.info("Caught " + ioe + " during region assignment, will retry"); - // Only do wal splitting if shouldSplitWal and in DLR mode - serverManager.processDeadServer(serverName, - this.shouldSplitWal && distributedLogReplay); - return; - } - - if (this.shouldSplitWal && distributedLogReplay) { - // wait for region assignment completes - for (HRegionInfo hri : toAssignRegions) { - try { - if (!am.waitOnRegionToClearRegionsInTransition(hri, regionAssignmentWaitTimeout)) { - // Wait here is to avoid log replay hits current dead server and incur a RPC timeout - // when replay happens before region assignment completes. - LOG.warn("Region " + hri.getEncodedName() - + " didn't complete assignment in time"); - } - } catch (InterruptedException ie) { - throw new InterruptedIOException("Caught " + ie - + " during waitOnRegionToClearRegionsInTransition"); - } - } - // submit logReplay work - this.services.getExecutorService().submit( - new LogReplayHandler(this.server, this.services, this.deadServers, this.serverName)); - hasLogReplayWork = true; - } - } finally { - this.deadServers.finish(serverName); - } - - if (!hasLogReplayWork) { - LOG.info("Finished processing of shutdown of " + serverName); - } - } - - private void resubmit(final ServerName serverName, IOException ex) throws IOException { - // typecast to SSH so that we make sure that it is the SSH instance that - // gets submitted as opposed to MSSH or some other derived instance of SSH - this.services.getExecutorService().submit((ServerShutdownHandler) this); - this.deadServers.add(serverName); - throw new IOException("failed log splitting for " + serverName + ", will retry", ex); - } - - /** - * Process a dead region from a dead RS. Checks if the region is disabled or - * disabling or if the region has a partially completed split. - * @param hri - * @param assignmentManager - * @return Returns true if specified region should be assigned, false if not. - * @throws IOException - */ - public static boolean processDeadRegion(HRegionInfo hri, - AssignmentManager assignmentManager) - throws IOException { - boolean tablePresent = assignmentManager.getTableStateManager().isTablePresent(hri.getTable()); - if (!tablePresent) { - LOG.info("The table " + hri.getTable() - + " was deleted. Hence not proceeding."); - return false; - } - // If table is not disabled but the region is offlined, - boolean disabled = assignmentManager.getTableStateManager().isTableState(hri.getTable(), - ZooKeeperProtos.Table.State.DISABLED); - if (disabled){ - LOG.info("The table " + hri.getTable() - + " was disabled. Hence not proceeding."); - return false; - } - if (hri.isOffline() && hri.isSplit()) { - //HBASE-7721: Split parent and daughters are inserted into hbase:meta as an atomic operation. - //If the meta scanner saw the parent split, then it should see the daughters as assigned - //to the dead server. We don't have to do anything. - return false; - } - boolean disabling = assignmentManager.getTableStateManager().isTableState(hri.getTable(), - ZooKeeperProtos.Table.State.DISABLING); - if (disabling) { - LOG.info("The table " + hri.getTable() - + " is disabled. Hence not assigning region" + hri.getEncodedName()); - return false; - } - return true; - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java index 6c80dd2..8dc0ca1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AddColumnFamilyProcedure.java @@ -80,7 +80,8 @@ public class AddColumnFamilyProcedure } @Override - protected Flow executeFromState(final MasterProcedureEnv env, final AddColumnFamilyState state) { + protected Flow executeFromState(final MasterProcedureEnv env, final AddColumnFamilyState state) + throws InterruptedException { if (isTraceEnabled()) { LOG.trace(this + " execute state=" + state); } @@ -109,7 +110,7 @@ public class AddColumnFamilyProcedure default: throw new UnsupportedOperationException(this + " unhandled state=" + state); } - } catch (InterruptedException|IOException e) { + } catch (IOException e) { LOG.warn("Error trying to add the column family" + getColumnFamilyName() + " to the table " + tableName + " (in state=" + state + ")", e); @@ -184,14 +185,14 @@ public class AddColumnFamilyProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { if (!env.isInitialized()) return false; - return env.getProcedureQueue().tryAcquireTableWrite( + return env.getProcedureQueue().tryAcquireTableExclusiveLock( tableName, EventType.C_M_ADD_FAMILY.toString()); } @Override protected void releaseLock(final MasterProcedureEnv env) { - env.getProcedureQueue().releaseTableWrite(tableName); + env.getProcedureQueue().releaseTableExclusiveLock(tableName); } @Override @@ -404,4 +405,4 @@ public class AddColumnFamilyProcedure } return regionInfoList; } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index 360637f..a791b70 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -95,7 +95,8 @@ public class CreateTableProcedure } @Override - protected Flow executeFromState(final MasterProcedureEnv env, final CreateTableState state) { + protected Flow executeFromState(final MasterProcedureEnv env, final CreateTableState state) + throws InterruptedException { if (LOG.isTraceEnabled()) { LOG.trace(this + " execute state=" + state); } @@ -136,7 +137,7 @@ public class CreateTableProcedure default: throw new UnsupportedOperationException("unhandled state=" + state); } - } catch (InterruptedException|HBaseException|IOException e) { + } catch (HBaseException|IOException e) { LOG.error("Error trying to create table=" + getTableName() + " state=" + state, e); setFailure("master-create-table", e); } @@ -269,12 +270,15 @@ public class CreateTableProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { - return env.getProcedureQueue().tryAcquireTableWrite(getTableName(), "create table"); + if (!env.isInitialized() && !getTableName().isSystemTable()) { + return false; + } + return env.getProcedureQueue().tryAcquireTableExclusiveLock(getTableName(), "create table"); } @Override protected void releaseLock(final MasterProcedureEnv env) { - env.getProcedureQueue().releaseTableWrite(getTableName()); + env.getProcedureQueue().releaseTableExclusiveLock(getTableName()); } private boolean prepareCreate(final MasterProcedureEnv env) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java index 316f225..76bda07 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteColumnFamilyProcedure.java @@ -81,7 +81,8 @@ public class DeleteColumnFamilyProcedure } @Override - protected Flow executeFromState(final MasterProcedureEnv env, DeleteColumnFamilyState state) { + protected Flow executeFromState(final MasterProcedureEnv env, DeleteColumnFamilyState state) + throws InterruptedException { if (isTraceEnabled()) { LOG.trace(this + " execute state=" + state); } @@ -114,7 +115,7 @@ public class DeleteColumnFamilyProcedure default: throw new UnsupportedOperationException(this + " unhandled state=" + state); } - } catch (InterruptedException|IOException e) { + } catch (IOException e) { if (!isRollbackSupported(state)) { // We reach a state that cannot be rolled back. We just need to keep retry. LOG.warn("Error trying to delete the column family " + getColumnFamilyName() @@ -200,14 +201,14 @@ public class DeleteColumnFamilyProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { if (!env.isInitialized()) return false; - return env.getProcedureQueue().tryAcquireTableWrite( + return env.getProcedureQueue().tryAcquireTableExclusiveLock( tableName, EventType.C_M_DELETE_FAMILY.toString()); } @Override protected void releaseLock(final MasterProcedureEnv env) { - env.getProcedureQueue().releaseTableWrite(tableName); + env.getProcedureQueue().releaseTableExclusiveLock(tableName); } @Override @@ -283,6 +284,11 @@ public class DeleteColumnFamilyProcedure throw new InvalidFamilyOperationException("Family '" + getColumnFamilyName() + "' does not exist, so it cannot be deleted"); } + + if (unmodifiedHTableDescriptor.getColumnFamilies().length == 1) { + throw new InvalidFamilyOperationException("Family '" + getColumnFamilyName() + + "' is the only column family in the table, so it cannot be deleted"); + } } /** @@ -436,4 +442,4 @@ public class DeleteColumnFamilyProcedure } return regionInfoList; } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java index 2ba7b42..3305c73 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteTableProcedure.java @@ -90,7 +90,8 @@ public class DeleteTableProcedure } @Override - protected Flow executeFromState(final MasterProcedureEnv env, DeleteTableState state) { + protected Flow executeFromState(final MasterProcedureEnv env, DeleteTableState state) + throws InterruptedException { if (LOG.isTraceEnabled()) { LOG.trace(this + " execute state=" + state); } @@ -146,9 +147,6 @@ public class DeleteTableProcedure } } catch (HBaseException|IOException e) { LOG.warn("Retriable error trying to delete table=" + getTableName() + " state=" + state, e); - } catch (InterruptedException e) { - // if the interrupt is real, the executor will be stopped. - LOG.warn("Interrupted trying to delete table=" + getTableName() + " state=" + state, e); } return Flow.HAS_MORE_STATE; } @@ -200,12 +198,12 @@ public class DeleteTableProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { if (!env.isInitialized()) return false; - return env.getProcedureQueue().tryAcquireTableWrite(getTableName(), "delete table"); + return env.getProcedureQueue().tryAcquireTableExclusiveLock(getTableName(), "delete table"); } @Override protected void releaseLock(final MasterProcedureEnv env) { - env.getProcedureQueue().releaseTableWrite(getTableName()); + env.getProcedureQueue().releaseTableExclusiveLock(getTableName()); } @Override @@ -405,6 +403,8 @@ public class DeleteTableProcedure protected static void deleteTableStates(final MasterProcedureEnv env, final TableName tableName) throws IOException { - ProcedureSyncWait.getMasterQuotaManager(env).removeTableFromNamespaceQuota(tableName); + if (!tableName.isSystemTable()) { + ProcedureSyncWait.getMasterQuotaManager(env).removeTableFromNamespaceQuota(tableName); + } } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java index bd8f29e..efc9846 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DisableTableProcedure.java @@ -121,7 +121,8 @@ public class DisableTableProcedure } @Override - protected Flow executeFromState(final MasterProcedureEnv env, final DisableTableState state) { + protected Flow executeFromState(final MasterProcedureEnv env, final DisableTableState state) + throws InterruptedException { if (isTraceEnabled()) { LOG.trace(this + " execute state=" + state); } @@ -162,7 +163,7 @@ public class DisableTableProcedure default: throw new UnsupportedOperationException("unhandled state=" + state); } - } catch (InterruptedException|HBaseException|IOException e) { + } catch (HBaseException|IOException e) { LOG.warn("Retriable error trying to disable table=" + tableName + " state=" + state, e); } return Flow.HAS_MORE_STATE; @@ -214,14 +215,14 @@ public class DisableTableProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { if (!env.isInitialized()) return false; - return env.getProcedureQueue().tryAcquireTableWrite( + return env.getProcedureQueue().tryAcquireTableExclusiveLock( tableName, EventType.C_M_DISABLE_TABLE.toString()); } @Override protected void releaseLock(final MasterProcedureEnv env) { - env.getProcedureQueue().releaseTableWrite(tableName); + env.getProcedureQueue().releaseTableExclusiveLock(tableName); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java index 989e81a..0063fb9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java @@ -94,9 +94,9 @@ public class EnableTableProcedure /** * Constructor * @param env MasterProcedureEnv - * @throws IOException * @param tableName the table to operate on * @param skipTableStateCheck whether to check table state + * @throws IOException */ public EnableTableProcedure( final MasterProcedureEnv env, @@ -119,7 +119,8 @@ public class EnableTableProcedure } @Override - protected Flow executeFromState(final MasterProcedureEnv env, final EnableTableState state) { + protected Flow executeFromState(final MasterProcedureEnv env, final EnableTableState state) + throws InterruptedException { if (isTraceEnabled()) { LOG.trace(this + " execute state=" + state); } @@ -156,7 +157,7 @@ public class EnableTableProcedure default: throw new UnsupportedOperationException("unhandled state=" + state); } - } catch (InterruptedException|HBaseException|IOException e) { + } catch (HBaseException|IOException e) { LOG.error("Error trying to enable table=" + tableName + " state=" + state, e); setFailure("master-enable-table", e); } @@ -238,14 +239,14 @@ public class EnableTableProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { if (!env.isInitialized()) return false; - return env.getProcedureQueue().tryAcquireTableWrite( + return env.getProcedureQueue().tryAcquireTableExclusiveLock( tableName, EventType.C_M_ENABLE_TABLE.toString()); } @Override protected void releaseLock(final MasterProcedureEnv env) { - env.getProcedureQueue().releaseTableWrite(tableName); + env.getProcedureQueue().releaseTableExclusiveLock(tableName); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureConstants.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureConstants.java index 90ed4ee..c21137d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureConstants.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureConstants.java @@ -24,8 +24,21 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; public final class MasterProcedureConstants { private MasterProcedureConstants() {} + /** Used to construct the name of the log directory for master procedures */ public static final String MASTER_PROCEDURE_LOGDIR = "MasterProcWALs"; + /** Number of threads used by the procedure executor */ public static final String MASTER_PROCEDURE_THREADS = "hbase.master.procedure.threads"; public static final int DEFAULT_MIN_MASTER_PROCEDURE_THREADS = 4; + + /** + * Procedure replay sanity check. In case a WAL is missing or unreadable we + * may lose information about pending/running procedures. + * Set this to true in case you want the Master failing on load if a corrupted + * procedure is encountred. + * (Default is off, because we prefer having the Master up and running and + * fix the "in transition" state "by hand") + */ + public static final String EXECUTOR_ABORT_ON_CORRUPTION = "hbase.procedure.abort.on.corruption"; + public static final boolean DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION = false; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java index 0a33cd4..f2f4bf3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java @@ -120,4 +120,4 @@ public class MasterProcedureEnv { public boolean isInitialized() { return master.isInitialized(); } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureQueue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureQueue.java index 0dd0c3d..af9eecf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureQueue.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureQueue.java @@ -27,6 +27,7 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.TableNotFoundException; @@ -43,11 +44,12 @@ import org.apache.hadoop.hbase.master.procedure.TableProcedureInterface.TableOpe * ProcedureRunnableSet for the Master Procedures. * This RunnableSet tries to provide to the ProcedureExecutor procedures * that can be executed without having to wait on a lock. - * Most of the master operations can be executed concurrently, if the they + * Most of the master operations can be executed concurrently, if they * are operating on different tables (e.g. two create table can be performed - * at the same, time assuming table A and table B). + * at the same, time assuming table A and table B) or against two different servers; say + * two servers that crashed at about the same time. * - * Each procedure should implement an interface providing information for this queue. + *

    Each procedure should implement an interface providing information for this queue. * for example table related procedures should implement TableProcedureInterface. * each procedure will be pushed in its own queue, and based on the operation type * we may take smarter decision. e.g. we can abort all the operations preceding @@ -58,7 +60,18 @@ import org.apache.hadoop.hbase.master.procedure.TableProcedureInterface.TableOpe public class MasterProcedureQueue implements ProcedureRunnableSet { private static final Log LOG = LogFactory.getLog(MasterProcedureQueue.class); - private final ProcedureFairRunQueues fairq; + // Two queues to ensure that server procedures run ahead of table precedures always. + private final ProcedureFairRunQueues tableFairQ; + /** + * Rely on basic fair q. ServerCrashProcedure will yield if meta is not assigned. This way, the + * server that was carrying meta should rise to the top of the queue (this is how it used to + * work when we had handlers and ServerShutdownHandler ran). TODO: special handling of servers + * that were carrying system tables on crash; do I need to have these servers have priority? + * + *

    Apart from the special-casing of meta and system tables, fairq is what we want + */ + private final ProcedureFairRunQueues serverFairQ; + private final ReentrantLock lock = new ReentrantLock(); private final Condition waitCond = lock.newCondition(); private final TableLockManager lockManager; @@ -66,11 +79,16 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { private final int metaTablePriority; private final int userTablePriority; private final int sysTablePriority; + private static final int DEFAULT_SERVER_PRIORITY = 1; + /** + * Keeps count across server and table queues. + */ private int queueSize; public MasterProcedureQueue(final Configuration conf, final TableLockManager lockManager) { - this.fairq = new ProcedureFairRunQueues(1); + this.tableFairQ = new ProcedureFairRunQueues(1); + this.serverFairQ = new ProcedureFairRunQueues(1); this.lockManager = lockManager; // TODO: should this be part of the HTD? @@ -105,12 +123,13 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { @Override public void yield(final Procedure proc) { - addFront(proc); + addBack(proc); } @Override @edu.umd.cs.findbugs.annotations.SuppressWarnings("WA_AWAIT_NOT_IN_LOOP") public Long poll() { + Long pollResult = null; lock.lock(); try { if (queueSize == 0) { @@ -119,19 +138,25 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { return null; } } - - RunQueue queue = fairq.poll(); - if (queue != null && queue.isAvailable()) { - queueSize--; - return queue.poll(); + // For now, let server handling have precedence over table handling; presumption is that it + // is more important handling crashed servers than it is running the + // enabling/disabling tables, etc. + pollResult = doPoll(serverFairQ.poll()); + if (pollResult == null) { + pollResult = doPoll(tableFairQ.poll()); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); - return null; } finally { lock.unlock(); } - return null; + return pollResult; + } + + private Long doPoll(final RunQueue rq) { + if (rq == null || !rq.isAvailable()) return null; + this.queueSize--; + return rq.poll(); } @Override @@ -148,7 +173,8 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { public void clear() { lock.lock(); try { - fairq.clear(); + serverFairQ.clear(); + tableFairQ.clear(); queueSize = 0; } finally { lock.unlock(); @@ -169,7 +195,8 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { public String toString() { lock.lock(); try { - return "MasterProcedureQueue size=" + queueSize + ": " + fairq; + return "MasterProcedureQueue size=" + queueSize + ": tableFairQ: " + tableFairQ + + ", serverFairQ: " + serverFairQ; } finally { lock.unlock(); } @@ -197,6 +224,7 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { markTableAsDeleted(iProcTable.getTableName()); } } + // No cleanup for ServerProcedureInterface types, yet. } private RunQueue getRunQueueOrCreate(final Procedure proc) { @@ -204,17 +232,26 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { final TableName table = ((TableProcedureInterface)proc).getTableName(); return getRunQueueOrCreate(table); } - // TODO: at the moment we only have Table procedures - // if you are implementing a non-table procedure, you have two option create - // a group for all the non-table procedures or try to find a key for your - // non-table procedure and implement something similar to the TableRunQueue. + if (proc instanceof ServerProcedureInterface) { + return getRunQueueOrCreate((ServerProcedureInterface)proc); + } + // TODO: at the moment we only have Table and Server procedures + // if you are implementing a non-table/non-server procedure, you have two options: create + // a group for all the non-table/non-server procedures or try to find a key for your + // non-table/non-server procedures and implement something similar to the TableRunQueue. throw new UnsupportedOperationException("RQs for non-table procedures are not implemented yet"); } private TableRunQueue getRunQueueOrCreate(final TableName table) { final TableRunQueue queue = getRunQueue(table); if (queue != null) return queue; - return (TableRunQueue)fairq.add(table, createTableRunQueue(table)); + return (TableRunQueue)tableFairQ.add(table, createTableRunQueue(table)); + } + + private ServerRunQueue getRunQueueOrCreate(final ServerProcedureInterface spi) { + final ServerRunQueue queue = getRunQueue(spi.getServerName()); + if (queue != null) return queue; + return (ServerRunQueue)serverFairQ.add(spi.getServerName(), createServerRunQueue(spi)); } private TableRunQueue createTableRunQueue(final TableName table) { @@ -227,8 +264,35 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { return new TableRunQueue(priority); } + private ServerRunQueue createServerRunQueue(final ServerProcedureInterface spi) { + return new ServerRunQueue(DEFAULT_SERVER_PRIORITY); + } + private TableRunQueue getRunQueue(final TableName table) { - return (TableRunQueue)fairq.get(table); + return (TableRunQueue)tableFairQ.get(table); + } + + private ServerRunQueue getRunQueue(final ServerName sn) { + return (ServerRunQueue)serverFairQ.get(sn); + } + + /** + * Try to acquire the write lock on the specified table. + * other operations in the table-queue will be executed after the lock is released. + * @param table Table to lock + * @param purpose Human readable reason for locking the table + * @return true if we were able to acquire the lock on the table, otherwise false. + */ + public boolean tryAcquireTableExclusiveLock(final TableName table, final String purpose) { + return getRunQueueOrCreate(table).tryExclusiveLock(lockManager, table, purpose); + } + + /** + * Release the write lock taken with tryAcquireTableWrite() + * @param table the name of the table that has the write lock + */ + public void releaseTableExclusiveLock(final TableName table) { + getRunQueue(table).releaseExclusiveLock(lockManager, table); } /** @@ -239,35 +303,54 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { * @param purpose Human readable reason for locking the table * @return true if we were able to acquire the lock on the table, otherwise false. */ - public boolean tryAcquireTableRead(final TableName table, final String purpose) { - return getRunQueueOrCreate(table).tryRead(lockManager, table, purpose); + public boolean tryAcquireTableSharedLock(final TableName table, final String purpose) { + return getRunQueueOrCreate(table).trySharedLock(lockManager, table, purpose); } /** * Release the read lock taken with tryAcquireTableRead() * @param table the name of the table that has the read lock */ - public void releaseTableRead(final TableName table) { - getRunQueue(table).releaseRead(lockManager, table); + public void releaseTableSharedLock(final TableName table) { + getRunQueue(table).releaseSharedLock(lockManager, table); } /** - * Try to acquire the write lock on the specified table. - * other operations in the table-queue will be executed after the lock is released. - * @param table Table to lock - * @param purpose Human readable reason for locking the table - * @return true if we were able to acquire the lock on the table, otherwise false. + * Try to acquire the write lock on the specified server. + * @see #releaseServerExclusiveLock(ServerProcedureInterface) + * @param spi Server to lock + * @return true if we were able to acquire the lock on the server, otherwise false. */ - public boolean tryAcquireTableWrite(final TableName table, final String purpose) { - return getRunQueueOrCreate(table).tryWrite(lockManager, table, purpose); + public boolean tryAcquireServerExclusiveLock(final ServerProcedureInterface spi) { + return getRunQueueOrCreate(spi).tryExclusiveLock(); } /** - * Release the write lock taken with tryAcquireTableWrite() - * @param table the name of the table that has the write lock + * Release the write lock + * @see #tryAcquireServerExclusiveLock(ServerProcedureInterface) + * @param spi the server that has the write lock */ - public void releaseTableWrite(final TableName table) { - getRunQueue(table).releaseWrite(lockManager, table); + public void releaseServerExclusiveLock(final ServerProcedureInterface spi) { + getRunQueue(spi.getServerName()).releaseExclusiveLock(); + } + + /** + * Try to acquire the read lock on the specified server. + * @see #releaseServerSharedLock(ServerProcedureInterface) + * @param spi Server to lock + * @return true if we were able to acquire the lock on the server, otherwise false. + */ + public boolean tryAcquireServerSharedLock(final ServerProcedureInterface spi) { + return getRunQueueOrCreate(spi).trySharedLock(); + } + + /** + * Release the read lock taken + * @see #tryAcquireServerSharedLock(ServerProcedureInterface) + * @param spi the server that has the read lock + */ + public void releaseServerSharedLock(final ServerProcedureInterface spi) { + getRunQueue(spi.getServerName()).releaseSharedLock(); } /** @@ -284,7 +367,7 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { lock.lock(); try { if (queue.isEmpty() && !queue.isLocked()) { - fairq.remove(table); + tableFairQ.remove(table); // Remove the table lock try { @@ -311,114 +394,167 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { } /** - * Run Queue for a Table. It contains a read-write lock that is used by the - * MasterProcedureQueue to decide if we should fetch an item from this queue - * or skip to another one which will be able to run without waiting for locks. + * Base abstract class for RunQueue implementations. + * Be careful honoring synchronizations in subclasses. In here we protect access but if you are + * acting on a state found in here, be sure dependent code keeps synchronization. + * Implements basic in-memory read/write locking mechanism to prevent procedure steps being run + * in parallel. */ - private static class TableRunQueue implements RunQueue { + private static abstract class AbstractRunQueue implements RunQueue { + // All modification of runnables happens with #lock held. private final Deque runnables = new ArrayDeque(); private final int priority; + private boolean exclusiveLock = false; + private int sharedLock = 0; - private TableLock tableLock = null; - private boolean wlock = false; - private int rlock = 0; - - public TableRunQueue(int priority) { + public AbstractRunQueue(int priority) { this.priority = priority; } + boolean isEmpty() { + return this.runnables.isEmpty(); + } + @Override - public void addFront(final Procedure proc) { - runnables.addFirst(proc.getProcId()); + public boolean isAvailable() { + synchronized (this) { + return !exclusiveLock && !runnables.isEmpty(); + } } - // TODO: Improve run-queue push with TableProcedureInterface.getType() - // we can take smart decisions based on the type of the operation (e.g. create/delete) @Override - public void addBack(final Procedure proc) { - runnables.addLast(proc.getProcId()); + public int getPriority() { + return this.priority; + } + + @Override + public void addFront(Procedure proc) { + this.runnables.addFirst(proc.getProcId()); + } + + @Override + public void addBack(Procedure proc) { + this.runnables.addLast(proc.getProcId()); } @Override public Long poll() { - return runnables.poll(); + return this.runnables.poll(); } @Override - public boolean isAvailable() { - synchronized (this) { - return !wlock && !runnables.isEmpty(); - } + public synchronized boolean isLocked() { + return isExclusiveLock() || sharedLock > 0; + } + + public synchronized boolean isExclusiveLock() { + return this.exclusiveLock; + } + + public synchronized boolean trySharedLock() { + if (isExclusiveLock()) return false; + sharedLock++; + return true; + } + + public synchronized void releaseSharedLock() { + sharedLock--; } - public boolean isEmpty() { - return runnables.isEmpty(); + /** + * @return True if only one instance of a shared lock outstanding. + */ + synchronized boolean isSingleSharedLock() { + return sharedLock == 1; } + public synchronized boolean tryExclusiveLock() { + if (isLocked()) return false; + exclusiveLock = true; + return true; + } + + public synchronized void releaseExclusiveLock() { + exclusiveLock = false; + } + @Override - public boolean isLocked() { - synchronized (this) { - return wlock || rlock > 0; - } + public String toString() { + return this.runnables.toString(); } + } - public boolean tryRead(final TableLockManager lockManager, - final TableName tableName, final String purpose) { - synchronized (this) { - if (wlock) { - return false; - } + /** + * Run Queue for Server procedures. + */ + private static class ServerRunQueue extends AbstractRunQueue { + public ServerRunQueue(int priority) { + super(priority); + } + } - // Take zk-read-lock - tableLock = lockManager.readLock(tableName, purpose); - try { - tableLock.acquire(); - } catch (IOException e) { - LOG.error("failed acquire read lock on " + tableName, e); - tableLock = null; - return false; - } + /** + * Run Queue for a Table. It contains a read-write lock that is used by the + * MasterProcedureQueue to decide if we should fetch an item from this queue + * or skip to another one which will be able to run without waiting for locks. + */ + private static class TableRunQueue extends AbstractRunQueue { + private TableLock tableLock = null; - rlock++; + public TableRunQueue(int priority) { + super(priority); + } + + // TODO: Improve run-queue push with TableProcedureInterface.getType() + // we can take smart decisions based on the type of the operation (e.g. create/delete) + @Override + public void addBack(final Procedure proc) { + super.addBack(proc); + } + + public synchronized boolean trySharedLock(final TableLockManager lockManager, + final TableName tableName, final String purpose) { + if (isExclusiveLock()) return false; + + // Take zk-read-lock + tableLock = lockManager.readLock(tableName, purpose); + try { + tableLock.acquire(); + } catch (IOException e) { + LOG.error("failed acquire read lock on " + tableName, e); + tableLock = null; + return false; } + trySharedLock(); return true; } - public void releaseRead(final TableLockManager lockManager, + public synchronized void releaseSharedLock(final TableLockManager lockManager, final TableName tableName) { - synchronized (this) { - releaseTableLock(lockManager, rlock == 1); - rlock--; - } + releaseTableLock(lockManager, isSingleSharedLock()); + releaseSharedLock(); } - public boolean tryWrite(final TableLockManager lockManager, + public synchronized boolean tryExclusiveLock(final TableLockManager lockManager, final TableName tableName, final String purpose) { - synchronized (this) { - if (wlock || rlock > 0) { - return false; - } - - // Take zk-write-lock - tableLock = lockManager.writeLock(tableName, purpose); - try { - tableLock.acquire(); - } catch (IOException e) { - LOG.error("failed acquire write lock on " + tableName, e); - tableLock = null; - return false; - } - wlock = true; + if (isLocked()) return false; + // Take zk-write-lock + tableLock = lockManager.writeLock(tableName, purpose); + try { + tableLock.acquire(); + } catch (IOException e) { + LOG.error("failed acquire write lock on " + tableName, e); + tableLock = null; + return false; } + tryExclusiveLock(); return true; } - public void releaseWrite(final TableLockManager lockManager, + public synchronized void releaseExclusiveLock(final TableLockManager lockManager, final TableName tableName) { - synchronized (this) { - releaseTableLock(lockManager, true); - wlock = false; - } + releaseTableLock(lockManager, true); + releaseExclusiveLock(); } private void releaseTableLock(final TableLockManager lockManager, boolean reset) { @@ -434,15 +570,5 @@ public class MasterProcedureQueue implements ProcedureRunnableSet { } } } - - @Override - public int getPriority() { - return priority; - } - - @Override - public String toString() { - return runnables.toString(); - } } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyColumnFamilyProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyColumnFamilyProcedure.java index 3de5202..24b17be 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyColumnFamilyProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyColumnFamilyProcedure.java @@ -78,7 +78,7 @@ public class ModifyColumnFamilyProcedure @Override protected Flow executeFromState(final MasterProcedureEnv env, - final ModifyColumnFamilyState state) { + final ModifyColumnFamilyState state) throws InterruptedException { if (isTraceEnabled()) { LOG.trace(this + " execute state=" + state); } @@ -107,7 +107,7 @@ public class ModifyColumnFamilyProcedure default: throw new UnsupportedOperationException(this + " unhandled state=" + state); } - } catch (InterruptedException|IOException e) { + } catch (IOException e) { LOG.warn("Error trying to modify the column family " + getColumnFamilyName() + " of the table " + tableName + "(in state=" + state + ")", e); @@ -182,14 +182,14 @@ public class ModifyColumnFamilyProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { if (!env.isInitialized()) return false; - return env.getProcedureQueue().tryAcquireTableWrite( + return env.getProcedureQueue().tryAcquireTableExclusiveLock( tableName, EventType.C_M_MODIFY_FAMILY.toString()); } @Override protected void releaseLock(final MasterProcedureEnv env) { - env.getProcedureQueue().releaseTableWrite(tableName); + env.getProcedureQueue().releaseTableExclusiveLock(tableName); } @Override @@ -379,4 +379,4 @@ public class ModifyColumnFamilyProcedure }); } } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java index e9636e6..f764022 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java @@ -87,7 +87,8 @@ public class ModifyTableProcedure } @Override - protected Flow executeFromState(final MasterProcedureEnv env, final ModifyTableState state) { + protected Flow executeFromState(final MasterProcedureEnv env, final ModifyTableState state) + throws InterruptedException { if (isTraceEnabled()) { LOG.trace(this + " execute state=" + state); } @@ -128,7 +129,7 @@ public class ModifyTableProcedure default: throw new UnsupportedOperationException("unhandled state=" + state); } - } catch (InterruptedException|IOException e) { + } catch (IOException e) { if (!isRollbackSupported(state)) { // We reach a state that cannot be rolled back. We just need to keep retry. LOG.warn("Error trying to modify table=" + getTableName() + " state=" + state, e); @@ -214,14 +215,14 @@ public class ModifyTableProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { if (!env.isInitialized()) return false; - return env.getProcedureQueue().tryAcquireTableWrite( + return env.getProcedureQueue().tryAcquireTableExclusiveLock( getTableName(), EventType.C_M_MODIFY_TABLE.toString()); } @Override protected void releaseLock(final MasterProcedureEnv env) { - env.getProcedureQueue().releaseTableWrite(getTableName()); + env.getProcedureQueue().releaseTableExclusiveLock(getTableName()); } @Override @@ -507,4 +508,4 @@ public class ModifyTableProcedure } return regionInfoList; } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java new file mode 100644 index 0000000..63d99e7 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerCrashProcedure.java @@ -0,0 +1,762 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.locks.Lock; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.client.RegionReplicaUtil; +import org.apache.hadoop.hbase.master.AssignmentManager; +import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.master.RegionStates; +import org.apache.hadoop.hbase.procedure2.ProcedureYieldException; +import org.apache.hadoop.hbase.procedure2.StateMachineProcedure; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionInfo; +import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos; +import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ServerCrashState; +import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; +import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; +import org.apache.hadoop.hbase.zookeeper.ZKAssign; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.util.StringUtils; +import org.apache.zookeeper.KeeperException; + +/** + * Handle crashed server. This is a port to ProcedureV2 of what used to be euphemistically called + * ServerShutdownHandler. + * + *

    The procedure flow varies dependent on whether meta is assigned, if we are + * doing distributed log replay versus distributed log splitting, and if we are to split logs at + * all. + * + *

    This procedure asks that all crashed servers get processed equally; we yield after the + * completion of each successful flow step. We do this so that we do not 'deadlock' waiting on + * a region assignment so we can replay edits which could happen if a region moved there are edits + * on two servers for replay. + * + *

    TODO: ASSIGN and WAIT_ON_ASSIGN (at least) are not idempotent. Revisit when assign is pv2. + * TODO: We do not have special handling for system tables. + */ +public class ServerCrashProcedure +extends StateMachineProcedure +implements ServerProcedureInterface { + private static final Log LOG = LogFactory.getLog(ServerCrashProcedure.class); + + /** + * Configuration key to set how long to wait in ms doing a quick check on meta state. + */ + public static final String KEY_SHORT_WAIT_ON_META = + "hbase.master.servercrash.short.wait.on.meta.ms"; + + public static final int DEFAULT_SHORT_WAIT_ON_META = 1000; + + /** + * Configuration key to set how many retries to cycle before we give up on meta. + * Each attempt will wait at least {@link #KEY_SHORT_WAIT_ON_META} milliseconds. + */ + public static final String KEY_RETRIES_ON_META = + "hbase.master.servercrash.meta.retries"; + + public static final int DEFAULT_RETRIES_ON_META = 10; + + /** + * Configuration key to set how long to wait in ms on regions in transition. + */ + public static final String KEY_WAIT_ON_RIT = + "hbase.master.servercrash.wait.on.rit.ms"; + + public static final int DEFAULT_WAIT_ON_RIT = 30000; + + private static final Set META_REGION_SET = new HashSet(); + static { + META_REGION_SET.add(HRegionInfo.FIRST_META_REGIONINFO); + } + + /** + * Name of the crashed server to process. + */ + private ServerName serverName; + + /** + * Regions that were on the crashed server. + */ + private Set regionsOnCrashedServer; + + /** + * Regions to assign. Usually some subset of {@link #regionsOnCrashedServer} + */ + private List regionsToAssign; + + private boolean distributedLogReplay = false; + private boolean carryingMeta = false; + private boolean shouldSplitWal; + + /** + * Cycles on same state. Good for figuring if we are stuck. + */ + private int cycles = 0; + + /** + * Ordinal of the previous state. So we can tell if we are progressing or not. TODO: if useful, + * move this back up into StateMachineProcedure + */ + private int previousState; + + /** + * Call this constructor queuing up a Procedure. + * @param serverName Name of the crashed server. + * @param shouldSplitWal True if we should split WALs as part of crashed server processing. + * @param carryingMeta True if carrying hbase:meta table region. + */ + public ServerCrashProcedure(final ServerName serverName, + final boolean shouldSplitWal, final boolean carryingMeta) { + this.serverName = serverName; + this.shouldSplitWal = shouldSplitWal; + this.carryingMeta = carryingMeta; + // Currently not used. + } + + /** + * Used when deserializing from a procedure store; we'll construct one of these then call + * {@link #deserializeStateData(InputStream)}. Do not use directly. + */ + public ServerCrashProcedure() { + super(); + } + + private void throwProcedureYieldException(final String msg) throws ProcedureYieldException { + String logMsg = msg + "; cycle=" + this.cycles + ", running for " + + StringUtils.formatTimeDiff(System.currentTimeMillis(), getStartTime()); + // The procedure executor logs ProcedureYieldException at trace level. For now, log these + // yields for server crash processing at DEBUG. Revisit when stable. + if (LOG.isDebugEnabled()) LOG.debug(logMsg); + throw new ProcedureYieldException(logMsg); + } + + @Override + protected Flow executeFromState(MasterProcedureEnv env, ServerCrashState state) + throws ProcedureYieldException { + if (LOG.isTraceEnabled()) { + LOG.trace(state); + } + // Keep running count of cycles + if (state.ordinal() != this.previousState) { + this.previousState = state.ordinal(); + this.cycles = 0; + } else { + this.cycles++; + } + MasterServices services = env.getMasterServices(); + try { + switch (state) { + case SERVER_CRASH_START: + // Is master fully online? If not, yield. No processing of servers unless master is up + if (!services.getAssignmentManager().isFailoverCleanupDone()) { + throwProcedureYieldException("Waiting on master failover to complete"); + } + LOG.info("Start processing crashed " + this.serverName); + start(env); + // If carrying meta, process it first. Else, get list of regions on crashed server. + if (this.carryingMeta) setNextState(ServerCrashState.SERVER_CRASH_PROCESS_META); + else setNextState(ServerCrashState.SERVER_CRASH_GET_REGIONS); + break; + + case SERVER_CRASH_GET_REGIONS: + // If hbase:meta is not assigned, yield. + if (!isMetaAssignedQuickTest(env)) { + throwProcedureYieldException("Waiting on hbase:meta assignment"); + } + this.regionsOnCrashedServer = + services.getAssignmentManager().getRegionStates().getServerRegions(this.serverName); + // Where to go next? Depends on whether we should split logs at all or if we should do + // distributed log splitting (DLS) vs distributed log replay (DLR). + if (!this.shouldSplitWal) { + setNextState(ServerCrashState.SERVER_CRASH_CALC_REGIONS_TO_ASSIGN); + } else if (this.distributedLogReplay) { + setNextState(ServerCrashState.SERVER_CRASH_PREPARE_LOG_REPLAY); + } else { + setNextState(ServerCrashState.SERVER_CRASH_SPLIT_LOGS); + } + break; + + case SERVER_CRASH_PROCESS_META: + // If we fail processing hbase:meta, yield. + if (!processMeta(env)) { + throwProcedureYieldException("Waiting on regions-in-transition to clear"); + } + setNextState(ServerCrashState.SERVER_CRASH_GET_REGIONS); + break; + + case SERVER_CRASH_PREPARE_LOG_REPLAY: + prepareLogReplay(env, this.regionsOnCrashedServer); + setNextState(ServerCrashState.SERVER_CRASH_CALC_REGIONS_TO_ASSIGN); + break; + + case SERVER_CRASH_SPLIT_LOGS: + splitLogs(env); + // If DLR, go to FINISH. Otherwise, if DLS, go to SERVER_CRASH_CALC_REGIONS_TO_ASSIGN + if (this.distributedLogReplay) setNextState(ServerCrashState.SERVER_CRASH_FINISH); + else setNextState(ServerCrashState.SERVER_CRASH_CALC_REGIONS_TO_ASSIGN); + break; + + case SERVER_CRASH_CALC_REGIONS_TO_ASSIGN: + this.regionsToAssign = calcRegionsToAssign(env); + setNextState(ServerCrashState.SERVER_CRASH_ASSIGN); + break; + + case SERVER_CRASH_ASSIGN: + // Assign may not be idempotent. SSH used to requeue the SSH if we got an IOE assigning + // which is what we are mimicing here but it looks prone to double assignment if assign + // fails midway. TODO: Test. + + // If no regions to assign, skip assign and skip to the finish. + boolean regions = this.regionsToAssign != null && !this.regionsToAssign.isEmpty(); + if (regions) { + if (!assign(env, this.regionsToAssign)) { + throwProcedureYieldException("Failed assign; will retry"); + } + } + if (regions && this.shouldSplitWal && distributedLogReplay) { + setNextState(ServerCrashState.SERVER_CRASH_WAIT_ON_ASSIGN); + } else { + setNextState(ServerCrashState.SERVER_CRASH_FINISH); + } + break; + + case SERVER_CRASH_WAIT_ON_ASSIGN: + // TODO: The list of regionsToAssign may be more than we actually assigned. See down in + // AM #1629 around 'if (regionStates.wasRegionOnDeadServer(encodedName)) {' where where we + // will skip assigning a region because it is/was on a dead server. Should never happen! + // It was on this server. Worst comes to worst, we'll still wait here till other server is + // processed. + + // If the wait on assign failed, yield -- if we have regions to assign. + if (this.regionsToAssign != null && !this.regionsToAssign.isEmpty()) { + if (!waitOnAssign(env, this.regionsToAssign)) { + throwProcedureYieldException("Waiting on region assign"); + } + } + setNextState(ServerCrashState.SERVER_CRASH_SPLIT_LOGS); + break; + + case SERVER_CRASH_FINISH: + LOG.info("Finished processing of crashed " + serverName); + services.getServerManager().getDeadServers().finish(serverName); + return Flow.NO_MORE_STATE; + + default: + throw new UnsupportedOperationException("unhandled state=" + state); + } + } catch (IOException e) { + LOG.warn("Failed serverName=" + this.serverName + ", state=" + state + "; retry", e); + } catch (InterruptedException e) { + // TODO: Make executor allow IEs coming up out of execute. + LOG.warn("Interrupted serverName=" + this.serverName + ", state=" + state + "; retry", e); + Thread.currentThread().interrupt(); + } + return Flow.HAS_MORE_STATE; + } + + /** + * Start processing of crashed server. In here we'll just set configs. and return. + * @param env + * @throws IOException + */ + private void start(final MasterProcedureEnv env) throws IOException { + MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); + // Set recovery mode late. This is what the old ServerShutdownHandler used do. + mfs.setLogRecoveryMode(); + this.distributedLogReplay = mfs.getLogRecoveryMode() == RecoveryMode.LOG_REPLAY; + } + + /** + * @param env + * @return False if we fail to assign and split logs on meta ('process'). + * @throws IOException + * @throws InterruptedException + */ + private boolean processMeta(final MasterProcedureEnv env) + throws IOException { + if (LOG.isDebugEnabled()) LOG.debug("Processing hbase:meta that was on " + this.serverName); + MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); + AssignmentManager am = env.getMasterServices().getAssignmentManager(); + HRegionInfo metaHRI = HRegionInfo.FIRST_META_REGIONINFO; + if (this.shouldSplitWal) { + if (this.distributedLogReplay) { + prepareLogReplay(env, META_REGION_SET); + } else { + // TODO: Matteo. We BLOCK here but most important thing to be doing at this moment. + mfs.splitMetaLog(serverName); + am.getRegionStates().logSplit(metaHRI); + } + } + + // Assign meta if still carrying it. Check again: region may be assigned because of RIT timeout + boolean processed = true; + if (am.isCarryingMeta(serverName)) { + // TODO: May block here if hard time figuring state of meta. + am.regionOffline(HRegionInfo.FIRST_META_REGIONINFO); + verifyAndAssignMetaWithRetries(env); + if (this.shouldSplitWal && distributedLogReplay) { + int timeout = env.getMasterConfiguration().getInt(KEY_WAIT_ON_RIT, DEFAULT_WAIT_ON_RIT); + if (!waitOnRegionToClearRegionsInTransition(am, metaHRI, timeout)) { + processed = false; + } else { + // TODO: Matteo. We BLOCK here but most important thing to be doing at this moment. + mfs.splitMetaLog(serverName); + } + } + } + return processed; + } + + /** + * @return True if region cleared RIT, else false if we timed out waiting. + * @throws InterruptedIOException + */ + private boolean waitOnRegionToClearRegionsInTransition(AssignmentManager am, + final HRegionInfo hri, final int timeout) + throws InterruptedIOException { + try { + if (!am.waitOnRegionToClearRegionsInTransition(hri, timeout)) { + // Wait here is to avoid log replay hits current dead server and incur a RPC timeout + // when replay happens before region assignment completes. + LOG.warn("Region " + hri.getEncodedName() + " didn't complete assignment in time"); + return false; + } + } catch (InterruptedException ie) { + throw new InterruptedIOException("Caught " + ie + + " during waitOnRegionToClearRegionsInTransition for " + hri); + } + return true; + } + + private void prepareLogReplay(final MasterProcedureEnv env, final Set regions) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Mark " + size(this.regionsOnCrashedServer) + + " regions-in-recovery from " + this.serverName); + } + MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); + AssignmentManager am = env.getMasterServices().getAssignmentManager(); + mfs.prepareLogReplay(this.serverName, regions); + am.getRegionStates().logSplit(this.serverName); + } + + private void splitLogs(final MasterProcedureEnv env) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Splitting logs from " + serverName + "; region count=" + + size(this.regionsOnCrashedServer)); + } + MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); + AssignmentManager am = env.getMasterServices().getAssignmentManager(); + // TODO: For Matteo. Below BLOCKs!!!! Redo so can relinquish executor while it is running. + mfs.splitLog(this.serverName); + am.getRegionStates().logSplit(this.serverName); + } + + static int size(final Collection hris) { + return hris == null? 0: hris.size(); + } + + /** + * Figure out what we need to assign. Should be idempotent. + * @param env + * @return List of calculated regions to assign; may be empty or null. + * @throws IOException + */ + private List calcRegionsToAssign(final MasterProcedureEnv env) + throws IOException { + AssignmentManager am = env.getMasterServices().getAssignmentManager(); + List regionsToAssignAggregator = new ArrayList(); + int replicaCount = env.getMasterConfiguration().getInt(HConstants.META_REPLICAS_NUM, + HConstants.DEFAULT_META_REPLICA_NUM); + for (int i = 1; i < replicaCount; i++) { + HRegionInfo metaHri = + RegionReplicaUtil.getRegionInfoForReplica(HRegionInfo.FIRST_META_REGIONINFO, i); + if (am.isCarryingMetaReplica(this.serverName, metaHri)) { + if (LOG.isDebugEnabled()) { + LOG.debug("Reassigning meta replica" + metaHri + " that was on " + this.serverName); + } + regionsToAssignAggregator.add(metaHri); + } + } + // Clean out anything in regions in transition. + List regionsInTransition = am.cleanOutCrashedServerReferences(serverName); + if (LOG.isDebugEnabled()) { + LOG.debug("Reassigning " + size(this.regionsOnCrashedServer) + + " region(s) that " + (serverName == null? "null": serverName) + + " was carrying (and " + regionsInTransition.size() + + " regions(s) that were opening on this server)"); + } + regionsToAssignAggregator.addAll(regionsInTransition); + + // Iterate regions that were on this server and figure which of these we need to reassign + if (this.regionsOnCrashedServer != null && !this.regionsOnCrashedServer.isEmpty()) { + RegionStates regionStates = am.getRegionStates(); + for (HRegionInfo hri: this.regionsOnCrashedServer) { + if (regionsInTransition.contains(hri)) continue; + String encodedName = hri.getEncodedName(); + Lock lock = am.acquireRegionLock(encodedName); + try { + RegionState rit = regionStates.getRegionTransitionState(hri); + if (processDeadRegion(hri, am)) { + ServerName addressFromAM = regionStates.getRegionServerOfRegion(hri); + if (addressFromAM != null && !addressFromAM.equals(this.serverName)) { + // If this region is in transition on the dead server, it must be + // opening or pending_open, which should have been covered by + // AM#cleanOutCrashedServerReferences + LOG.info("Skip assigning region " + hri.getRegionNameAsString() + + " because it has been opened in " + addressFromAM.getServerName()); + continue; + } + if (rit != null) { + if (rit.getServerName() != null && !rit.isOnServer(this.serverName)) { + // Skip regions that are in transition on other server + LOG.info("Skip assigning region in transition on other server" + rit); + continue; + } + LOG.info("Reassigning region " + rit + " and clearing zknode if exists"); + try { + // This clears out any RIT that might be sticking around. + ZKAssign.deleteNodeFailSilent(env.getMasterServices().getZooKeeper(), hri); + } catch (KeeperException e) { + // TODO: FIX!!!! ABORTING SERVER BECAUSE COULDN"T PURGE ZNODE. This is what we + // used to do but that doesn't make it right!!! + env.getMasterServices().abort("Unexpected error deleting RIT " + hri, e); + throw new IOException(e); + } + regionStates.updateRegionState(hri, RegionState.State.OFFLINE); + } else if (regionStates.isRegionInState( + hri, RegionState.State.SPLITTING_NEW, RegionState.State.MERGING_NEW)) { + regionStates.updateRegionState(hri, RegionState.State.OFFLINE); + } + regionsToAssignAggregator.add(hri); + // TODO: The below else if is different in branch-1 from master branch. + } else if (rit != null) { + if ((rit.isPendingCloseOrClosing() || rit.isOffline()) + && am.getTableStateManager().isTableState(hri.getTable(), + ZooKeeperProtos.Table.State.DISABLED, ZooKeeperProtos.Table.State.DISABLING) || + am.getReplicasToClose().contains(hri)) { + // If the table was partially disabled and the RS went down, we should clear the + // RIT and remove the node for the region. + // The rit that we use may be stale in case the table was in DISABLING state + // but though we did assign we will not be clearing the znode in CLOSING state. + // Doing this will have no harm. See HBASE-5927 + regionStates.updateRegionState(hri, RegionState.State.OFFLINE); + am.deleteClosingOrClosedNode(hri, rit.getServerName()); + am.offlineDisabledRegion(hri); + } else { + LOG.warn("THIS SHOULD NOT HAPPEN: unexpected region in transition " + + rit + " not to be assigned by SSH of server " + serverName); + } + } + } finally { + lock.unlock(); + } + } + } + return regionsToAssignAggregator; + } + + private boolean assign(final MasterProcedureEnv env, final List hris) + throws InterruptedIOException { + AssignmentManager am = env.getMasterServices().getAssignmentManager(); + try { + am.assign(hris); + } catch (InterruptedException ie) { + LOG.error("Caught " + ie + " during round-robin assignment"); + throw (InterruptedIOException)new InterruptedIOException().initCause(ie); + } catch (IOException ioe) { + LOG.info("Caught " + ioe + " during region assignment, will retry"); + return false; + } + return true; + } + + private boolean waitOnAssign(final MasterProcedureEnv env, final List hris) + throws InterruptedIOException { + int timeout = env.getMasterConfiguration().getInt(KEY_WAIT_ON_RIT, DEFAULT_WAIT_ON_RIT); + for (HRegionInfo hri: hris) { + // TODO: Blocks here. + if (!waitOnRegionToClearRegionsInTransition(env.getMasterServices().getAssignmentManager(), + hri, timeout)) { + return false; + } + } + return true; + } + + @Override + protected void rollbackState(MasterProcedureEnv env, ServerCrashState state) + throws IOException { + // Can't rollback. + throw new UnsupportedOperationException("unhandled state=" + state); + } + + @Override + protected ServerCrashState getState(int stateId) { + return ServerCrashState.valueOf(stateId); + } + + @Override + protected int getStateId(ServerCrashState state) { + return state.getNumber(); + } + + @Override + protected ServerCrashState getInitialState() { + return ServerCrashState.SERVER_CRASH_START; + } + + @Override + protected boolean abort(MasterProcedureEnv env) { + // TODO + return false; + } + + @Override + protected boolean acquireLock(final MasterProcedureEnv env) { + if (!env.getMasterServices().isServerCrashProcessingEnabled()) return false; + return env.getProcedureQueue().tryAcquireServerExclusiveLock(this); + } + + @Override + protected void releaseLock(final MasterProcedureEnv env) { + env.getProcedureQueue().releaseServerExclusiveLock(this); + } + + @Override + public void toStringClassDetails(StringBuilder sb) { + sb.append(getClass().getSimpleName()); + sb.append(" serverName="); + sb.append(this.serverName); + sb.append(", shouldSplitWal="); + sb.append(shouldSplitWal); + sb.append(", carryingMeta="); + sb.append(carryingMeta); + } + + @Override + public void serializeStateData(final OutputStream stream) throws IOException { + super.serializeStateData(stream); + + MasterProcedureProtos.ServerCrashStateData.Builder state = + MasterProcedureProtos.ServerCrashStateData.newBuilder(). + setServerName(ProtobufUtil.toServerName(this.serverName)). + setDistributedLogReplay(this.distributedLogReplay). + setCarryingMeta(this.carryingMeta). + setShouldSplitWal(this.shouldSplitWal); + if (this.regionsOnCrashedServer != null && !this.regionsOnCrashedServer.isEmpty()) { + for (HRegionInfo hri: this.regionsOnCrashedServer) { + state.addRegionsOnCrashedServer(HRegionInfo.convert(hri)); + } + } + if (this.regionsToAssign != null && !this.regionsToAssign.isEmpty()) { + for (HRegionInfo hri: this.regionsToAssign) { + state.addRegionsToAssign(HRegionInfo.convert(hri)); + } + } + state.build().writeDelimitedTo(stream); + } + + @Override + public void deserializeStateData(final InputStream stream) throws IOException { + super.deserializeStateData(stream); + + MasterProcedureProtos.ServerCrashStateData state = + MasterProcedureProtos.ServerCrashStateData.parseDelimitedFrom(stream); + this.serverName = ProtobufUtil.toServerName(state.getServerName()); + this.distributedLogReplay = state.hasDistributedLogReplay()? + state.getDistributedLogReplay(): false; + this.carryingMeta = state.hasCarryingMeta()? state.getCarryingMeta(): false; + // shouldSplitWAL has a default over in pb so this invocation will always work. + this.shouldSplitWal = state.getShouldSplitWal(); + int size = state.getRegionsOnCrashedServerCount(); + if (size > 0) { + this.regionsOnCrashedServer = new HashSet(size); + for (RegionInfo ri: state.getRegionsOnCrashedServerList()) { + this.regionsOnCrashedServer.add(HRegionInfo.convert(ri)); + } + } + size = state.getRegionsToAssignCount(); + if (size > 0) { + this.regionsToAssign = new ArrayList(size); + for (RegionInfo ri: state.getRegionsOnCrashedServerList()) { + this.regionsToAssign.add(HRegionInfo.convert(ri)); + } + } + } + + /** + * Process a dead region from a dead RS. Checks if the region is disabled or + * disabling or if the region has a partially completed split. + * @param hri + * @param assignmentManager + * @return Returns true if specified region should be assigned, false if not. + * @throws IOException + */ + private static boolean processDeadRegion(HRegionInfo hri, AssignmentManager assignmentManager) + throws IOException { + boolean tablePresent = assignmentManager.getTableStateManager().isTablePresent(hri.getTable()); + if (!tablePresent) { + LOG.info("The table " + hri.getTable() + " was deleted. Hence not proceeding."); + return false; + } + // If table is not disabled but the region is offlined, + boolean disabled = assignmentManager.getTableStateManager().isTableState(hri.getTable(), + ZooKeeperProtos.Table.State.DISABLED); + if (disabled){ + LOG.info("The table " + hri.getTable() + " was disabled. Hence not proceeding."); + return false; + } + if (hri.isOffline() && hri.isSplit()) { + // HBASE-7721: Split parent and daughters are inserted into hbase:meta as an atomic operation. + // If the meta scanner saw the parent split, then it should see the daughters as assigned + // to the dead server. We don't have to do anything. + return false; + } + boolean disabling = assignmentManager.getTableStateManager().isTableState(hri.getTable(), + ZooKeeperProtos.Table.State.DISABLING); + if (disabling) { + LOG.info("The table " + hri.getTable() + " is disabled. Hence not assigning region" + + hri.getEncodedName()); + return false; + } + return true; + } + + /** + * If hbase:meta is not assigned already, assign. + * @throws IOException + */ + private void verifyAndAssignMetaWithRetries(final MasterProcedureEnv env) throws IOException { + MasterServices services = env.getMasterServices(); + int iTimes = services.getConfiguration().getInt(KEY_RETRIES_ON_META, DEFAULT_RETRIES_ON_META); + // Just reuse same time as we have for short wait on meta. Adding another config is overkill. + long waitTime = + services.getConfiguration().getLong(KEY_SHORT_WAIT_ON_META, DEFAULT_SHORT_WAIT_ON_META); + int iFlag = 0; + while (true) { + try { + verifyAndAssignMeta(env); + break; + } catch (KeeperException e) { + services.abort("In server shutdown processing, assigning meta", e); + throw new IOException("Aborting", e); + } catch (Exception e) { + if (iFlag >= iTimes) { + services.abort("verifyAndAssignMeta failed after" + iTimes + " retries, aborting", e); + throw new IOException("Aborting", e); + } + try { + Thread.sleep(waitTime); + } catch (InterruptedException e1) { + LOG.warn("Interrupted when is the thread sleep", e1); + Thread.currentThread().interrupt(); + throw (InterruptedIOException)new InterruptedIOException().initCause(e1); + } + iFlag++; + } + } + } + + /** + * If hbase:meta is not assigned already, assign. + * @throws InterruptedException + * @throws IOException + * @throws KeeperException + */ + private void verifyAndAssignMeta(final MasterProcedureEnv env) + throws InterruptedException, IOException, KeeperException { + MasterServices services = env.getMasterServices(); + if (!isMetaAssignedQuickTest(env)) { + services.getAssignmentManager().assignMeta(HRegionInfo.FIRST_META_REGIONINFO); + } else if (serverName.equals(services.getMetaTableLocator(). + getMetaRegionLocation(services.getZooKeeper()))) { + throw new IOException("hbase:meta is onlined on the dead server " + this.serverName); + } else { + LOG.info("Skip assigning hbase:meta because it is online at " + + services.getMetaTableLocator().getMetaRegionLocation(services.getZooKeeper())); + } + } + + /** + * A quick test that hbase:meta is assigned; blocks for short time only. + * @return True if hbase:meta location is available and verified as good. + * @throws InterruptedException + * @throws IOException + */ + private boolean isMetaAssignedQuickTest(final MasterProcedureEnv env) + throws InterruptedException, IOException { + ZooKeeperWatcher zkw = env.getMasterServices().getZooKeeper(); + MetaTableLocator mtl = env.getMasterServices().getMetaTableLocator(); + boolean metaAssigned = false; + // Is hbase:meta location available yet? + if (mtl.isLocationAvailable(zkw)) { + ClusterConnection connection = env.getMasterServices().getConnection(); + // Is hbase:meta location good yet? + long timeout = + env.getMasterConfiguration().getLong(KEY_SHORT_WAIT_ON_META, DEFAULT_SHORT_WAIT_ON_META); + if (mtl.verifyMetaRegionLocation(connection, zkw, timeout)) { + metaAssigned = true; + } + } + return metaAssigned; + } + + @Override + public ServerName getServerName() { + return this.serverName; + } + + @Override + public boolean hasMetaTableRegion() { + return this.carryingMeta; + } + + /** + * For this procedure, yield at end of each successful flow step so that all crashed servers + * can make progress rather than do the default which has each procedure running to completion + * before we move to the next. For crashed servers, especially if running with distributed log + * replay, we will want all servers to come along; we do not want the scenario where a server is + * stuck waiting for regions to online so it can replay edits. + */ + @Override + protected boolean isYieldBeforeExecuteFromState(MasterProcedureEnv env, ServerCrashState state) { + return true; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerProcedureInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerProcedureInterface.java new file mode 100644 index 0000000..5b0c45f --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ServerProcedureInterface.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.procedure; + +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * Procedures that handle servers -- e.g. server crash -- must implement this Interface. + * It is used by the procedure runner to figure locking and what queuing. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public interface ServerProcedureInterface { + /** + * @return Name of this server instance. + */ + ServerName getServerName(); + + /** + * @return True if this server has an hbase:meta table region. + */ + boolean hasMetaTableRegion(); +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java index 6928d02..cc088f3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java @@ -45,4 +45,4 @@ public interface TableProcedureInterface { * @return the operation type that the procedure is executing. */ TableOperationType getTableOperationType(); -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java index c69bd8f..1a5b9ae 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateTableProcedure.java @@ -68,7 +68,8 @@ public class TruncateTableProcedure } @Override - protected Flow executeFromState(final MasterProcedureEnv env, TruncateTableState state) { + protected Flow executeFromState(final MasterProcedureEnv env, TruncateTableState state) + throws InterruptedException { if (LOG.isTraceEnabled()) { LOG.trace(this + " execute state=" + state); } @@ -130,9 +131,6 @@ public class TruncateTableProcedure } } catch (HBaseException|IOException e) { LOG.warn("Retriable error trying to truncate table=" + getTableName() + " state=" + state, e); - } catch (InterruptedException e) { - // if the interrupt is real, the executor will be stopped. - LOG.warn("Interrupted trying to truncate table=" + getTableName() + " state=" + state, e); } return Flow.HAS_MORE_STATE; } @@ -183,12 +181,12 @@ public class TruncateTableProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { if (!env.isInitialized()) return false; - return env.getProcedureQueue().tryAcquireTableWrite(getTableName(), "truncate table"); + return env.getProcedureQueue().tryAcquireTableExclusiveLock(getTableName(), "truncate table"); } @Override protected void releaseLock(final MasterProcedureEnv env) { - env.getProcedureQueue().releaseTableWrite(getTableName()); + env.getProcedureQueue().releaseTableExclusiveLock(getTableName()); } @Override @@ -287,4 +285,4 @@ public class TruncateTableProcedure }); } } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java index 3da0c0b..a0766ef 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java @@ -99,6 +99,7 @@ public class DefaultMemStore implements MemStore { volatile MemStoreLAB allocator; volatile MemStoreLAB snapshotAllocator; volatile long snapshotId; + volatile boolean tagsPresent; /** * Default constructor. Used for tests. @@ -170,8 +171,11 @@ public class DefaultMemStore implements MemStore { timeOfOldestEdit = Long.MAX_VALUE; } } - return new MemStoreSnapshot(this.snapshotId, snapshot.size(), this.snapshotSize, - this.snapshotTimeRangeTracker, new CollectionBackedScanner(snapshot, this.comparator)); + MemStoreSnapshot memStoreSnapshot = new MemStoreSnapshot(this.snapshotId, snapshot.size(), this.snapshotSize, + this.snapshotTimeRangeTracker, new CollectionBackedScanner(snapshot, this.comparator), + this.tagsPresent); + this.tagsPresent = false; + return memStoreSnapshot; } /** @@ -233,6 +237,13 @@ public class DefaultMemStore implements MemStore { private boolean addToCellSet(Cell e) { boolean b = this.cellSet.add(e); + // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call. + // When we use ACL CP or Visibility CP which deals with Tags during + // mutation, the TagRewriteCell.getTagsLength() is a cheaper call. We do not + // parse the byte[] to identify the tags length. + if(e.getTagsLength() > 0) { + tagsPresent = true; + } setOldestEditTimeToNow(); return b; } @@ -1005,8 +1016,8 @@ public class DefaultMemStore implements MemStore { } } - public final static long FIXED_OVERHEAD = ClassSize.align( - ClassSize.OBJECT + (9 * ClassSize.REFERENCE) + (3 * Bytes.SIZEOF_LONG)); + public final static long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT + + (9 * ClassSize.REFERENCE) + (3 * Bytes.SIZEOF_LONG) + Bytes.SIZEOF_BOOLEAN); public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD + ClassSize.ATOMIC_LONG + (2 * ClassSize.TIMERANGE_TRACKER) + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java index 73b8cb9..474a44a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java @@ -64,7 +64,7 @@ public class DefaultStoreFlusher extends StoreFlusher { status.setStatus("Flushing " + store + ": creating writer"); // Write the map out to the disk writer = store.createWriterInTmp( - cellsCount, store.getFamily().getCompression(), false, true, true); + cellsCount, store.getFamily().getCompression(), false, true, snapshot.isTagsPresent()); writer.setTimeRangeTracker(snapshot.getTimeRangeTracker()); IOException e = null; try { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushLargeStoresPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushLargeStoresPolicy.java index 7e0e54c..328e890 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushLargeStoresPolicy.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushLargeStoresPolicy.java @@ -76,9 +76,9 @@ public class FlushLargeStoresPolicy extends FlushPolicy { private boolean shouldFlush(Store store) { if (store.getMemStoreSize() > this.flushSizeLowerBound) { if (LOG.isDebugEnabled()) { - LOG.debug("Column Family: " + store.getColumnFamilyName() + " of region " + region - + " will be flushed because of memstoreSize(" + store.getMemStoreSize() - + ") is larger than lower bound(" + this.flushSizeLowerBound + ")"); + LOG.debug("Flush Column Family " + store.getColumnFamilyName() + " of " + + region.getRegionInfo().getEncodedName() + " because memstoreSize=" + + store.getMemStoreSize() + " > lower bound=" + this.flushSizeLowerBound); } return true; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 22fdc78..9c41ec6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -22,7 +22,6 @@ import java.io.EOFException; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; -import java.io.UnsupportedEncodingException; import java.lang.reflect.Constructor; import java.text.ParseException; import java.util.AbstractList; @@ -351,7 +350,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi private boolean disallowWritesInRecovering = false; // when a region is in recovering state, it can only accept writes not reads - private volatile boolean isRecovering = false; + private volatile boolean recovering = false; private volatile Optional configurationManager; @@ -608,6 +607,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi * is new), then read them from the supplied path. * @param htd the table descriptor * @param rsServices reference to {@link RegionServerServices} or null + * @deprecated Use other constructors. */ @Deprecated public HRegion(final Path tableDir, final WAL wal, final FileSystem fs, @@ -711,7 +711,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi Map recoveringRegions = rsServices.getRecoveringRegions(); String encodedName = getRegionInfo().getEncodedName(); if (recoveringRegions != null && recoveringRegions.containsKey(encodedName)) { - this.isRecovering = true; + this.recovering = true; recoveringRegions.put(encodedName, this); } } else { @@ -841,7 +841,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi // overlaps used sequence numbers if (this.writestate.writesEnabled) { nextSeqid = WALSplitter.writeRegionSequenceIdFile(this.fs.getFileSystem(), this.fs - .getRegionDir(), nextSeqid, (this.isRecovering ? (this.flushPerChanges + 10000000) : 1)); + .getRegionDir(), nextSeqid, (this.recovering ? (this.flushPerChanges + 10000000) : 1)); } else { nextSeqid++; } @@ -1153,7 +1153,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi * Reset recovering state of current region */ public void setRecovering(boolean newState) { - boolean wasRecovering = this.isRecovering; + boolean wasRecovering = this.recovering; // before we flip the recovering switch (enabling reads) we should write the region open // event to WAL if needed if (wal != null && getRegionServerServices() != null && !writestate.readOnly @@ -1194,8 +1194,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi } } - this.isRecovering = newState; - if (wasRecovering && !isRecovering) { + this.recovering = newState; + if (wasRecovering && !recovering) { // Call only when wal replay is over. coprocessorHost.postLogReplay(); } @@ -1203,7 +1203,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi @Override public boolean isRecovering() { - return this.isRecovering; + return this.recovering; } @Override @@ -1610,16 +1610,13 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi byte[] encodedRegionName = this.getRegionInfo().getEncodedNameAsBytes(); regionLoadBldr.clearStoreCompleteSequenceId(); for (byte[] familyName : this.stores.keySet()) { - long oldestUnflushedSeqId = this.wal.getEarliestMemstoreSeqNum(encodedRegionName, familyName); - // no oldestUnflushedSeqId means no data has written to the store after last flush, so we use - // lastFlushOpSeqId as complete sequence id for the store. + long completeSequenceId = getFlushedSequenceId(encodedRegionName, lastFlushOpSeqIdLocal); regionLoadBldr.addStoreCompleteSequenceId(StoreSequenceId .newBuilder() .setFamilyName(ByteString.copyFrom(familyName)) - .setSequenceId( - oldestUnflushedSeqId < 0 ? lastFlushOpSeqIdLocal : oldestUnflushedSeqId - 1).build()); + .setSequenceId(completeSequenceId).build()); } - return regionLoadBldr.setCompleteSequenceId(this.maxFlushedSeqId); + return regionLoadBldr.setCompleteSequenceId(getMaxFlushedSeqId()); } ////////////////////////////////////////////////////////////////////////////// @@ -1912,27 +1909,25 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi * returns true which will make a lot of flush requests. */ boolean shouldFlushStore(Store store) { - long maxFlushedSeqId = - this.wal.getEarliestMemstoreSeqNum(getRegionInfo().getEncodedNameAsBytes(), store - .getFamily().getName()) - 1; - if (maxFlushedSeqId > 0 && maxFlushedSeqId + flushPerChanges < sequenceId.get()) { + long earliest = this.wal.getEarliestMemstoreSeqNum(getRegionInfo().getEncodedNameAsBytes(), + store.getFamily().getName()) - 1; + if (earliest > 0 && earliest + flushPerChanges < sequenceId.get()) { if (LOG.isDebugEnabled()) { - LOG.debug("Column Family: " + store.getColumnFamilyName() + " of region " + this - + " will be flushed because its max flushed seqId(" + maxFlushedSeqId - + ") is far away from current(" + sequenceId.get() + "), max allowed is " - + flushPerChanges); + LOG.debug("Flush column family " + store.getColumnFamilyName() + " of " + + getRegionInfo().getEncodedName() + " because unflushed sequenceid=" + earliest + + " is > " + this.flushPerChanges + " from current=" + sequenceId.get()); } return true; } - if (flushCheckInterval <= 0) { + if (this.flushCheckInterval <= 0) { return false; } long now = EnvironmentEdgeManager.currentTime(); - if (store.timeOfOldestEdit() < now - flushCheckInterval) { + if (store.timeOfOldestEdit() < now - this.flushCheckInterval) { if (LOG.isDebugEnabled()) { - LOG.debug("Column Family: " + store.getColumnFamilyName() + " of region " + this - + " will be flushed because time of its oldest edit (" + store.timeOfOldestEdit() - + ") is far away from now(" + now + "), max allowed is " + flushCheckInterval); + LOG.debug("Flush column family: " + store.getColumnFamilyName() + " of " + + getRegionInfo().getEncodedName() + " because time of oldest edit=" + + store.timeOfOldestEdit() + " is > " + this.flushCheckInterval + " from now =" + now); } return true; } @@ -2036,6 +2031,20 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi } } + /** + * @param encodedRegionName + * @param alternative What to use if no edits currently in memstore. + * @return The sequence id just before the earliest edit in memstore. Use this as place to + * restart replay of edits from. + */ + private long getFlushedSequenceId(final byte [] encodedRegionName, final long alternative) { + long earliest = wal.getEarliestMemstoreSeqNum(encodedRegionName); + // If earliest == HConstants.NO_SEQNUM, it means no edits in memstore. Substract 1 from + // earliest to get a sequenceid that is before the earliest one in memstore; this is where we + // want replay to start at. + return (earliest == HConstants.NO_SEQNUM)? alternative: earliest - 1; + } + protected PrepareFlushResult internalPrepareFlushCache( final WAL wal, final long myseqid, final Collection storesToFlush, MonitoredTask status, boolean writeFlushWalMarker) @@ -2086,18 +2095,21 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi } if (LOG.isInfoEnabled()) { - LOG.info("Started memstore flush for " + this + ", current region memstore size " - + StringUtils.byteDesc(this.memstoreSize.get()) + ", and " + storesToFlush.size() + "/" - + stores.size() + " column families' memstores are being flushed." - + ((wal != null) ? "" : "; wal is null, using passed sequenceid=" + myseqid)); - // only log when we are not flushing all stores. + // Log a fat line detailing what is being flushed. + StringBuilder perCfExtras = null; if (this.stores.size() > storesToFlush.size()) { + perCfExtras = new StringBuilder(); for (Store store: storesToFlush) { - LOG.info("Flushing Column Family: " + store.getColumnFamilyName() - + " which was occupying " - + StringUtils.byteDesc(store.getMemStoreSize()) + " of memstore."); + perCfExtras.append("; "); + perCfExtras.append(store.getColumnFamilyName()); + perCfExtras.append("="); + perCfExtras.append(StringUtils.byteDesc(store.getMemStoreSize())); } } + LOG.info("Flushing " + + storesToFlush.size() + "/" + stores.size() + + " column families, memstore=" + StringUtils.byteDesc(this.memstoreSize.get()) + + ((perCfExtras != null && perCfExtras.length() > 0)? perCfExtras.toString(): "") + + ((wal != null) ? "" : "; WAL is null, using passed sequenceid=" + myseqid)); } // Stop updates while we snapshot the memstore of all of these regions' stores. We only have // to do this for a moment. It is quick. We also set the memstore size to zero here before we @@ -2145,11 +2157,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi myseqid); } flushOpSeqId = getNextSequenceId(wal); - long oldestUnflushedSeqId = wal.getEarliestMemstoreSeqNum(encodedRegionName); - // no oldestUnflushedSeqId means we flushed all stores. - // or the unflushed stores are all empty. - flushedSeqId = (oldestUnflushedSeqId == HConstants.NO_SEQNUM) ? flushOpSeqId - : oldestUnflushedSeqId - 1; + // This will be the sequence id that we will have flushed to if this flush succeeds. + flushedSeqId = getFlushedSequenceId(encodedRegionName, flushOpSeqId); } else { // use the provided sequence Id as WAL is not being used for this flush. flushedSeqId = flushOpSeqId = myseqid; @@ -3704,7 +3713,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi // Make request outside of synchronize block; HBASE-818. this.rsServices.getFlushRequester().requestFlush(this, false); if (LOG.isDebugEnabled()) { - LOG.debug("Flush requested on " + this); + LOG.debug("Flush requested on " + this.getRegionInfo().getEncodedName()); } } @@ -4438,7 +4447,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi + seqId + " is greater than current seqId:" + currentSeqId); // Prepare flush (take a snapshot) and then abort (drop the snapshot) - if (store == null ) { + if (store == null) { for (Store s : stores.values()) { totalFreedSize += doDropStoreMemstoreContentsForSeqId(s, currentSeqId); } @@ -5432,11 +5441,11 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi return scannerContext.setScannerState(NextState.BATCH_LIMIT_REACHED).hasMoreValues(); } else if (scannerContext.checkSizeLimit(limitScope)) { ScannerContext.NextState state = - moreCellsInRow ? NextState.SIZE_LIMIT_REACHED_MID_ROW : NextState.SIZE_LIMIT_REACHED; + moreCellsInRow? NextState.SIZE_LIMIT_REACHED_MID_ROW: NextState.SIZE_LIMIT_REACHED; return scannerContext.setScannerState(state).hasMoreValues(); } else if (scannerContext.checkTimeLimit(limitScope)) { ScannerContext.NextState state = - moreCellsInRow ? NextState.TIME_LIMIT_REACHED_MID_ROW : NextState.TIME_LIMIT_REACHED; + moreCellsInRow? NextState.TIME_LIMIT_REACHED_MID_ROW: NextState.TIME_LIMIT_REACHED; return scannerContext.setScannerState(state).hasMoreValues(); } } while (moreCellsInRow); @@ -5815,7 +5824,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi LOG.error(msg); LOG.error("unable to refresh store files", e); abortRegionServer(msg); - return new NotServingRegionException(getRegionInfo().getRegionNameAsString() +" is closing"); + return new NotServingRegionException(getRegionInfo().getRegionNameAsString() + " closing"); } } @@ -5968,12 +5977,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi * @return new HRegion * @throws IOException */ - public static HRegion createHRegion(final HRegionInfo info, final Path rootDir, final Path tableDir, - final Configuration conf, - final HTableDescriptor hTableDescriptor, - final WAL wal, - final boolean initialize, final boolean ignoreWAL) - throws IOException { + public static HRegion createHRegion(final HRegionInfo info, final Path rootDir, + final Path tableDir, final Configuration conf, final HTableDescriptor hTableDescriptor, + final WAL wal, final boolean initialize, final boolean ignoreWAL) + throws IOException { LOG.info("creating HRegion " + info.getTable().getNameAsString() + " HTD == " + hTableDescriptor + " RootDir = " + rootDir + " Table name == " + info.getTable().getNameAsString()); @@ -6212,7 +6219,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi this.openSeqNum = initialize(reporter); this.setSequenceId(openSeqNum); if (wal != null && getRegionServerServices() != null && !writestate.readOnly - && !isRecovering) { + && !recovering) { // Only write the region open event marker to WAL if (1) we are not read-only // (2) dist log replay is off or we are not recovering. In case region is // recovering, the open event will be written at setRecovering(false) @@ -6338,6 +6345,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi * @param tabledir qualified path for table * @param name ENCODED region name * @return Path of HRegion directory + * @deprecated For tests only; to be removed. */ @Deprecated public static Path getRegionDir(final Path tabledir, final String name) { @@ -6350,6 +6358,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi * @param rootdir qualified path of HBase root directory * @param info HRegionInfo for the region * @return qualified path of region directory + * @deprecated For tests only; to be removed. */ @Deprecated @VisibleForTesting @@ -7170,7 +7179,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi newTags.add(itr.next()); } } - if (i < ( edits.size() - 1) && !CellUtil.matchingQualifier(cell, edits.get(i + 1))) + if (i < (edits.size() - 1) && !CellUtil.matchingQualifier(cell, edits.get(i + 1))) idx++; } @@ -7752,6 +7761,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi // sync the WAL edit (SYNC and FSYNC treated the same for now) this.wal.sync(txid); break; + default: + throw new RuntimeException("Unknown durability " + durability); } } } @@ -7840,8 +7851,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi @Override public long getOldestSeqIdOfStore(byte[] familyName) { - return wal.getEarliestMemstoreSeqNum(getRegionInfo() - .getEncodedNameAsBytes(), familyName); + return wal.getEarliestMemstoreSeqNum(getRegionInfo().getEncodedNameAsBytes(), familyName); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 98dab42..0c6b2f0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -111,16 +111,6 @@ import com.google.common.collect.Sets; * services is compaction services where files are aggregated once they pass * a configurable threshold. * - *

    The only thing having to do with logs that Store needs to deal with is - * the reconstructionLog. This is a segment of an HRegion's log that might - * NOT be present upon startup. If the param is NULL, there's nothing to do. - * If the param is non-NULL, we need to process the log to reconstruct - * a TreeMap that might not have been written to disk before the process - * died. - * - *

    It's assumed that after this constructor returns, the reconstructionLog - * file will be deleted (by whoever has instantiated the Store). - * *

    Locking and transactions are handled at a higher level. This API should * not be called directly but by an HRegion manager. */ @@ -898,8 +888,7 @@ public class HStore implements Store { } /** - * Write out current snapshot. Presumes {@link #snapshot()} has been called - * previously. + * Write out current snapshot. Presumes {@link #snapshot()} has been called previously. * @param logCacheFlushId flush sequence number * @param snapshot * @param status diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java index 619cff5..be853c5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java @@ -32,14 +32,16 @@ public class MemStoreSnapshot { private final long size; private final TimeRangeTracker timeRangeTracker; private final KeyValueScanner scanner; + private final boolean tagsPresent; public MemStoreSnapshot(long id, int cellsCount, long size, TimeRangeTracker timeRangeTracker, - KeyValueScanner scanner) { + KeyValueScanner scanner, boolean tagsPresent) { this.id = id; this.cellsCount = cellsCount; this.size = size; this.timeRangeTracker = timeRangeTracker; this.scanner = scanner; + this.tagsPresent = tagsPresent; } /** @@ -76,4 +78,11 @@ public class MemStoreSnapshot { public KeyValueScanner getScanner() { return this.scanner; } + + /** + * @return true if tags are present in this snapshot + */ + public boolean isTagsPresent() { + return this.tagsPresent; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java index 8910042..274f09a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServic import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.WALSplitter.MutationReplay; +import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.Message; import com.google.protobuf.RpcController; import com.google.protobuf.Service; @@ -127,6 +128,7 @@ public interface Region extends ConfigurationObserver { long getMaxFlushedSeqId(); /** @return the oldest sequence id found in the store for the given family */ + @VisibleForTesting public long getOldestSeqIdOfStore(byte[] familyName); /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index aa722a0..94814b6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -29,16 +29,13 @@ import java.net.URLEncoder; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.NavigableMap; import java.util.Set; -import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; @@ -66,7 +63,6 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.Bytes; @@ -91,7 +87,6 @@ import org.apache.htrace.Trace; import org.apache.htrace.TraceScope; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Maps; import com.lmax.disruptor.BlockingWaitStrategy; import com.lmax.disruptor.EventHandler; import com.lmax.disruptor.ExceptionHandler; @@ -156,7 +151,7 @@ public class FSHLog implements WAL { private static final Log LOG = LogFactory.getLog(FSHLog.class); private static final int DEFAULT_SLOW_SYNC_TIME_MS = 100; // in ms - + /** * The nexus at which all incoming handlers meet. Does appends and sync with an ordering. * Appends and syncs are each put on the ring which means handlers need to @@ -168,7 +163,7 @@ public class FSHLog implements WAL { private final Disruptor disruptor; /** - * An executorservice that runs the disrutpor AppendEventHandler append executor. + * An executorservice that runs the disruptor AppendEventHandler append executor. */ private final ExecutorService appendExecutor; @@ -210,6 +205,7 @@ public class FSHLog implements WAL { * WAL directory, where all WAL files would be placed. */ private final Path fullPathLogDir; + /** * dir path where old logs are kept. */ @@ -241,6 +237,7 @@ public class FSHLog implements WAL { * conf object */ protected final Configuration conf; + /** Listeners that are called on WAL events. */ private final List listeners = new CopyOnWriteArrayList(); @@ -258,6 +255,7 @@ public class FSHLog implements WAL { public WALCoprocessorHost getCoprocessorHost() { return coprocessorHost; } + /** * FSDataOutputStream associated with the current SequenceFile.writer */ @@ -289,6 +287,13 @@ public class FSHLog implements WAL { private volatile boolean lowReplicationRollEnabled = true; /** + * Class that does accounting of sequenceids in WAL subsystem. Holds oldest outstanding + * sequence id as yet not flushed as well as the most recent edit sequence id appended to the + * WAL. Has facility for answering questions such as "Is it safe to GC a WAL?". + */ + private SequenceIdAccounting sequenceIdAccounting = new SequenceIdAccounting(); + + /** * Current log file. */ volatile Writer writer; @@ -334,52 +339,6 @@ public class FSHLog implements WAL { private final AtomicInteger closeErrorCount = new AtomicInteger(); - // Region sequence id accounting across flushes and for knowing when we can GC a WAL. These - // sequence id numbers are by region and unrelated to the ring buffer sequence number accounting - // done above in failedSequence, highest sequence, etc. - /** - * This lock ties all operations on lowestFlushingStoreSequenceIds and - * oldestUnflushedStoreSequenceIds Maps with the exception of append's putIfAbsent call into - * oldestUnflushedStoreSequenceIds. We use these Maps to find out the low bound regions - * sequence id, or to find regions with old sequence ids to force flush; we are interested in - * old stuff not the new additions (TODO: IS THIS SAFE? CHECK!). - */ - private final Object regionSequenceIdLock = new Object(); - - /** - * Map of encoded region names and family names to their OLDEST -- i.e. their first, - * the longest-lived -- sequence id in memstore. Note that this sequence id is the region - * sequence id. This is not related to the id we use above for {@link #highestSyncedSequence} - * and {@link #highestUnsyncedSequence} which is the sequence from the disruptor - * ring buffer. - */ - private final ConcurrentMap> oldestUnflushedStoreSequenceIds - = new ConcurrentSkipListMap>( - Bytes.BYTES_COMPARATOR); - - /** - * Map of encoded region names and family names to their lowest or OLDEST sequence/edit id in - * memstore currently being flushed out to hfiles. Entries are moved here from - * {@link #oldestUnflushedStoreSequenceIds} while the lock {@link #regionSequenceIdLock} is held - * (so movement between the Maps is atomic). This is not related to the id we use above for - * {@link #highestSyncedSequence} and {@link #highestUnsyncedSequence} which is the sequence from - * the disruptor ring buffer, an internal detail. - */ - private final Map> lowestFlushingStoreSequenceIds = - new TreeMap>(Bytes.BYTES_COMPARATOR); - - /** - * Map of region encoded names to the latest region sequence id. Updated on each append of - * WALEdits to the WAL. We create one map for each WAL file at the time it is rolled. - *

    When deciding whether to archive a WAL file, we compare the sequence IDs in this map to - * {@link #lowestFlushingRegionSequenceIds} and {@link #oldestUnflushedRegionSequenceIds}. - * See {@link FSHLog#areAllRegionsFlushed(Map, Map, Map)} for more info. - *

    - * This map uses byte[] as the key, and uses reference equality. It works in our use case as we - * use {@link HRegionInfo#getEncodedNameAsBytes()} as keys. For a given region, it always returns - * the same array. - */ - private Map highestRegionSequenceIds = new HashMap(); /** * WAL Comparator; it compares the timestamp (log filenum), present in the log file name. @@ -396,7 +355,7 @@ public class FSHLog implements WAL { }; /** - * Map of wal log file to the latest sequence ids of all regions it has entries of. + * Map of WAL log file to the latest sequence ids of all regions it has entries of. * The map is sorted by the log file creation timestamp (contained in the log file name). */ private NavigableMap> byWalRegionSequenceIds = @@ -542,7 +501,7 @@ public class FSHLog implements WAL { (long)(blocksize * conf.getFloat("hbase.regionserver.logroll.multiplier", 0.95f)); this.maxLogs = conf.getInt("hbase.regionserver.maxlogs", 32); - this.minTolerableReplication = conf.getInt( "hbase.regionserver.hlog.tolerable.lowreplication", + this.minTolerableReplication = conf.getInt("hbase.regionserver.hlog.tolerable.lowreplication", FSUtils.getDefaultReplication(fs, this.fullPathLogDir)); this.lowReplicationRollLimit = conf.getInt("hbase.regionserver.hlog.lowreplication.rolllimit", 5); @@ -745,128 +704,37 @@ public class FSHLog implements WAL { return DefaultWALProvider.createWriter(conf, fs, path, false); } - private long getLowestSeqId(Map seqIdMap) { - long result = HConstants.NO_SEQNUM; - for (Long seqNum: seqIdMap.values()) { - if (result == HConstants.NO_SEQNUM || seqNum.longValue() < result) { - result = seqNum.longValue(); - } - } - return result; - } - - private > Map copyMapWithLowestSeqId( - Map mapToCopy) { - Map copied = Maps.newHashMap(); - for (Map.Entry entry: mapToCopy.entrySet()) { - long lowestSeqId = getLowestSeqId(entry.getValue()); - if (lowestSeqId != HConstants.NO_SEQNUM) { - copied.put(entry.getKey(), lowestSeqId); - } - } - return copied; - } - /** - * Archive old logs that could be archived: a log is eligible for archiving if all its WALEdits - * have been flushed to hfiles. - *

    - * For each log file, it compares its region to sequenceId map - * (@link {@link FSHLog#highestRegionSequenceIds} with corresponding region entries in - * {@link FSHLog#lowestFlushingRegionSequenceIds} and - * {@link FSHLog#oldestUnflushedRegionSequenceIds}. If all the regions in the map are flushed - * past of their value, then the wal is eligible for archiving. + * Archive old logs. A WAL is eligible for archiving if all its WALEdits have been flushed. * @throws IOException */ private void cleanOldLogs() throws IOException { - Map lowestFlushingRegionSequenceIdsLocal = null; - Map oldestUnflushedRegionSequenceIdsLocal = null; - List logsToArchive = new ArrayList(); - // make a local copy so as to avoid locking when we iterate over these maps. - synchronized (regionSequenceIdLock) { - lowestFlushingRegionSequenceIdsLocal = - copyMapWithLowestSeqId(this.lowestFlushingStoreSequenceIds); - oldestUnflushedRegionSequenceIdsLocal = - copyMapWithLowestSeqId(this.oldestUnflushedStoreSequenceIds); - } - for (Map.Entry> e : byWalRegionSequenceIds.entrySet()) { - // iterate over the log file. + List logsToArchive = null; + // For each log file, look at its Map of regions to highest sequence id; if all sequence ids + // are older than what is currently in memory, the WAL can be GC'd. + for (Map.Entry> e : this.byWalRegionSequenceIds.entrySet()) { Path log = e.getKey(); Map sequenceNums = e.getValue(); - // iterate over the map for this log file, and tell whether it should be archive or not. - if (areAllRegionsFlushed(sequenceNums, lowestFlushingRegionSequenceIdsLocal, - oldestUnflushedRegionSequenceIdsLocal)) { + if (this.sequenceIdAccounting.areAllLower(sequenceNums)) { + if (logsToArchive == null) logsToArchive = new ArrayList(); logsToArchive.add(log); - LOG.debug("WAL file ready for archiving " + log); + if (LOG.isTraceEnabled()) LOG.trace("WAL file ready for archiving " + log); } } - for (Path p : logsToArchive) { - this.totalLogSize.addAndGet(-this.fs.getFileStatus(p).getLen()); - archiveLogFile(p); - this.byWalRegionSequenceIds.remove(p); - } - } - - /** - * Takes a region:sequenceId map for a WAL file, and checks whether the file can be archived. - * It compares the region entries present in the passed sequenceNums map with the local copy of - * {@link #oldestUnflushedRegionSequenceIds} and {@link #lowestFlushingRegionSequenceIds}. If, - * for all regions, the value is lesser than the minimum of values present in the - * oldestFlushing/UnflushedSeqNums, then the wal file is eligible for archiving. - * @param sequenceNums for a WAL, at the time when it was rolled. - * @param oldestFlushingMap - * @param oldestUnflushedMap - * @return true if wal is eligible for archiving, false otherwise. - */ - static boolean areAllRegionsFlushed(Map sequenceNums, - Map oldestFlushingMap, Map oldestUnflushedMap) { - for (Map.Entry regionSeqIdEntry : sequenceNums.entrySet()) { - // find region entries in the flushing/unflushed map. If there is no entry, it meansj - // a region doesn't have any unflushed entry. - long oldestFlushing = oldestFlushingMap.containsKey(regionSeqIdEntry.getKey()) ? - oldestFlushingMap.get(regionSeqIdEntry.getKey()) : Long.MAX_VALUE; - long oldestUnFlushed = oldestUnflushedMap.containsKey(regionSeqIdEntry.getKey()) ? - oldestUnflushedMap.get(regionSeqIdEntry.getKey()) : Long.MAX_VALUE; - // do a minimum to be sure to contain oldest sequence Id - long minSeqNum = Math.min(oldestFlushing, oldestUnFlushed); - if (minSeqNum <= regionSeqIdEntry.getValue()) return false;// can't archive - } - return true; - } - - /** - * Iterates over the given map of regions, and compares their sequence numbers with corresponding - * entries in {@link #oldestUnflushedRegionSequenceIds}. If the sequence number is greater or - * equal, the region is eligible to flush, otherwise, there is no benefit to flush (from the - * perspective of passed regionsSequenceNums map), because the region has already flushed the - * entries present in the WAL file for which this method is called for (typically, the oldest - * wal file). - * @param regionsSequenceNums - * @return regions which should be flushed (whose sequence numbers are larger than their - * corresponding un-flushed entries. - */ - private byte[][] findEligibleMemstoresToFlush(Map regionsSequenceNums) { - List regionsToFlush = null; - // Keeping the old behavior of iterating unflushedSeqNums under oldestSeqNumsLock. - synchronized (regionSequenceIdLock) { - for (Map.Entry e: regionsSequenceNums.entrySet()) { - long unFlushedVal = getEarliestMemstoreSeqNum(e.getKey()); - if (unFlushedVal != HConstants.NO_SEQNUM && unFlushedVal <= e.getValue()) { - if (regionsToFlush == null) - regionsToFlush = new ArrayList(); - regionsToFlush.add(e.getKey()); - } + if (logsToArchive != null) { + for (Path p : logsToArchive) { + this.totalLogSize.addAndGet(-this.fs.getFileStatus(p).getLen()); + archiveLogFile(p); + this.byWalRegionSequenceIds.remove(p); } } - return regionsToFlush == null ? null : regionsToFlush - .toArray(new byte[][] { HConstants.EMPTY_BYTE_ARRAY }); } /** - * If the number of un-archived WAL files is greater than maximum allowed, it checks - * the first (oldest) WAL file, and returns the regions which should be flushed so that it could + * If the number of un-archived WAL files is greater than maximum allowed, check the first + * (oldest) WAL file, and returns those regions which should be flushed so that it can * be archived. - * @return regions to flush in order to archive oldest wal file. + * @return regions (encodedRegionNames) to flush in order to archive oldest WAL file. * @throws IOException */ byte[][] findRegionsToForceFlush() throws IOException { @@ -875,7 +743,7 @@ public class FSHLog implements WAL { if (logCount > this.maxLogs && logCount > 0) { Map.Entry> firstWALEntry = this.byWalRegionSequenceIds.firstEntry(); - regions = findEligibleMemstoresToFlush(firstWALEntry.getValue()); + regions = this.sequenceIdAccounting.findLower(firstWALEntry.getValue()); } if (regions != null) { StringBuilder sb = new StringBuilder(); @@ -883,9 +751,8 @@ public class FSHLog implements WAL { if (i > 0) sb.append(", "); sb.append(Bytes.toStringBinary(regions[i])); } - LOG.info("Too many wals: logs=" + logCount + ", maxlogs=" + - this.maxLogs + "; forcing flush of " + regions.length + " regions(s): " + - sb.toString()); + LOG.info("Too many WALs; count=" + logCount + ", max=" + this.maxLogs + + "; forcing flush of " + regions.length + " regions(s): " + sb.toString()); } return regions; } @@ -963,8 +830,7 @@ public class FSHLog implements WAL { this.numEntries.set(0); final String newPathString = (null == newPath ? null : FSUtils.getPath(newPath)); if (oldPath != null) { - this.byWalRegionSequenceIds.put(oldPath, this.highestRegionSequenceIds); - this.highestRegionSequenceIds = new HashMap(); + this.byWalRegionSequenceIds.put(oldPath, this.sequenceIdAccounting.resetHighest()); long oldFileLen = this.fs.getFileStatus(oldPath).getLen(); this.totalLogSize.addAndGet(oldFileLen); LOG.info("Rolled WAL " + FSUtils.getPath(oldPath) + " with entries=" + oldNumEntries + @@ -1108,7 +974,7 @@ public class FSHLog implements WAL { LOG.debug("Moved " + files.length + " WAL file(s) to " + FSUtils.getPath(this.fullPathArchiveDir)); } - LOG.info("Closed WAL: " + toString() ); + LOG.info("Closed WAL: " + toString()); } @Override @@ -1631,108 +1497,25 @@ public class FSHLog implements WAL { } @Override - public boolean startCacheFlush(final byte[] encodedRegionName, - Set flushedFamilyNames) { - Map oldStoreSeqNum = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); + public boolean startCacheFlush(final byte[] encodedRegionName, Set flushedFamilyNames) { if (!closeBarrier.beginOp()) { - LOG.info("Flush will not be started for " + Bytes.toString(encodedRegionName) + - " - because the server is closing."); + LOG.info("Flush not started for " + Bytes.toString(encodedRegionName) + "; server closing."); return false; } - synchronized (regionSequenceIdLock) { - ConcurrentMap oldestUnflushedStoreSequenceIdsOfRegion = - oldestUnflushedStoreSequenceIds.get(encodedRegionName); - if (oldestUnflushedStoreSequenceIdsOfRegion != null) { - for (byte[] familyName: flushedFamilyNames) { - Long seqId = oldestUnflushedStoreSequenceIdsOfRegion.remove(familyName); - if (seqId != null) { - oldStoreSeqNum.put(familyName, seqId); - } - } - if (!oldStoreSeqNum.isEmpty()) { - Map oldValue = this.lowestFlushingStoreSequenceIds.put( - encodedRegionName, oldStoreSeqNum); - assert oldValue == null: "Flushing map not cleaned up for " - + Bytes.toString(encodedRegionName); - } - if (oldestUnflushedStoreSequenceIdsOfRegion.isEmpty()) { - // Remove it otherwise it will be in oldestUnflushedStoreSequenceIds for ever - // even if the region is already moved to other server. - // Do not worry about data racing, we held write lock of region when calling - // startCacheFlush, so no one can add value to the map we removed. - oldestUnflushedStoreSequenceIds.remove(encodedRegionName); - } - } - } - if (oldStoreSeqNum.isEmpty()) { - // TODO: if we have no oldStoreSeqNum, and WAL is not disabled, presumably either - // the region is already flushing (which would make this call invalid), or there - // were no appends after last flush, so why are we starting flush? Maybe we should - // assert not empty. Less rigorous, but safer, alternative is telling the caller to stop. - // For now preserve old logic. - LOG.warn("Couldn't find oldest seqNum for the region we are about to flush: [" - + Bytes.toString(encodedRegionName) + "]"); - } + this.sequenceIdAccounting.startCacheFlush(encodedRegionName, flushedFamilyNames); return true; } @Override public void completeCacheFlush(final byte [] encodedRegionName) { - synchronized (regionSequenceIdLock) { - this.lowestFlushingStoreSequenceIds.remove(encodedRegionName); - } + this.sequenceIdAccounting.completeCacheFlush(encodedRegionName); closeBarrier.endOp(); } - private ConcurrentMap getOrCreateOldestUnflushedStoreSequenceIdsOfRegion( - byte[] encodedRegionName) { - ConcurrentMap oldestUnflushedStoreSequenceIdsOfRegion = - oldestUnflushedStoreSequenceIds.get(encodedRegionName); - if (oldestUnflushedStoreSequenceIdsOfRegion != null) { - return oldestUnflushedStoreSequenceIdsOfRegion; - } - oldestUnflushedStoreSequenceIdsOfRegion = - new ConcurrentSkipListMap(Bytes.BYTES_COMPARATOR); - ConcurrentMap alreadyPut = - oldestUnflushedStoreSequenceIds.putIfAbsent(encodedRegionName, - oldestUnflushedStoreSequenceIdsOfRegion); - return alreadyPut == null ? oldestUnflushedStoreSequenceIdsOfRegion : alreadyPut; - } - @Override public void abortCacheFlush(byte[] encodedRegionName) { - Map storeSeqNumsBeforeFlushStarts; - Map currentStoreSeqNums = new TreeMap(Bytes.BYTES_COMPARATOR); - synchronized (regionSequenceIdLock) { - storeSeqNumsBeforeFlushStarts = this.lowestFlushingStoreSequenceIds.remove( - encodedRegionName); - if (storeSeqNumsBeforeFlushStarts != null) { - ConcurrentMap oldestUnflushedStoreSequenceIdsOfRegion = - getOrCreateOldestUnflushedStoreSequenceIdsOfRegion(encodedRegionName); - for (Map.Entry familyNameAndSeqId: storeSeqNumsBeforeFlushStarts - .entrySet()) { - currentStoreSeqNums.put(familyNameAndSeqId.getKey(), - oldestUnflushedStoreSequenceIdsOfRegion.put(familyNameAndSeqId.getKey(), - familyNameAndSeqId.getValue())); - } - } - } + this.sequenceIdAccounting.abortCacheFlush(encodedRegionName); closeBarrier.endOp(); - if (storeSeqNumsBeforeFlushStarts != null) { - for (Map.Entry familyNameAndSeqId : storeSeqNumsBeforeFlushStarts.entrySet()) { - Long currentSeqNum = currentStoreSeqNums.get(familyNameAndSeqId.getKey()); - if (currentSeqNum != null - && currentSeqNum.longValue() <= familyNameAndSeqId.getValue().longValue()) { - String errorStr = - "Region " + Bytes.toString(encodedRegionName) + " family " - + Bytes.toString(familyNameAndSeqId.getKey()) - + " acquired edits out of order current memstore seq=" + currentSeqNum - + ", previous oldest unflushed id=" + familyNameAndSeqId.getValue(); - LOG.error(errorStr); - Runtime.getRuntime().halt(1); - } - } - } } @VisibleForTesting @@ -1762,23 +1545,22 @@ public class FSHLog implements WAL { @Override public long getEarliestMemstoreSeqNum(byte[] encodedRegionName) { - ConcurrentMap oldestUnflushedStoreSequenceIdsOfRegion = - this.oldestUnflushedStoreSequenceIds.get(encodedRegionName); - return oldestUnflushedStoreSequenceIdsOfRegion != null ? - getLowestSeqId(oldestUnflushedStoreSequenceIdsOfRegion) : HConstants.NO_SEQNUM; + // This method is called from two places; when we are preparing to flush and when we are + // reporting oldest sequenceid for a region we are closing (it is also used by tests). + return this.sequenceIdAccounting.getLowestSequenceId(encodedRegionName); } @Override - public long getEarliestMemstoreSeqNum(byte[] encodedRegionName, - byte[] familyName) { - ConcurrentMap oldestUnflushedStoreSequenceIdsOfRegion = - this.oldestUnflushedStoreSequenceIds.get(encodedRegionName); - if (oldestUnflushedStoreSequenceIdsOfRegion != null) { - Long result = oldestUnflushedStoreSequenceIdsOfRegion.get(familyName); - return result != null ? result.longValue() : HConstants.NO_SEQNUM; - } else { - return HConstants.NO_SEQNUM; - } + public long getEarliestMemstoreSeqNum(byte[] encodedRegionName, byte[] familyName) { + // This method is used by tests and for figuring if we should flush or not because our + // sequenceids are too old. It is also used reporting the master our oldest sequenceid for use + // figuring what edits can be skipped during log recovery. getEarliestMemStoreSequenceId + // from this.sequenceIdAccounting is looking first in flushingOldestStoreSequenceIds, the + // currently flushing sequence ids, and if anything found there, it is returning these. This is + // the right thing to do for the reporting oldest sequenceids to master; we won't skip edits if + // we crash during the flush. For figuring what to flush, we might get requeued if our sequence + // id is old even though we are currently flushing. This may mean we do too much flushing. + return this.sequenceIdAccounting.getLowestSequenceId(encodedRegionName, familyName); } /** @@ -1820,10 +1602,10 @@ public class FSHLog implements WAL { /** * For Thread A to call when it is ready to wait on the 'safe point' to be attained. * Thread A will be held in here until Thread B calls {@link #safePointAttained()} - * @throws InterruptedException - * @throws ExecutionException * @param syncFuture We need this as barometer on outstanding syncs. If it comes home with * an exception, then something is up w/ our syncing. + * @throws InterruptedException + * @throws ExecutionException * @return The passed syncFuture * @throws FailedSyncBeforeLogCloseException */ @@ -2014,15 +1796,6 @@ public class FSHLog implements WAL { } } - private void updateOldestUnflushedSequenceIds(byte[] encodedRegionName, - Set familyNameSet, Long lRegionSequenceId) { - ConcurrentMap oldestUnflushedStoreSequenceIdsOfRegion = - getOrCreateOldestUnflushedStoreSequenceIdsOfRegion(encodedRegionName); - for (byte[] familyName : familyNameSet) { - oldestUnflushedStoreSequenceIdsOfRegion.putIfAbsent(familyName, lRegionSequenceId); - } - } - /** * Append to the WAL. Does all CP and WAL listener calls. * @param entry @@ -2040,14 +1813,14 @@ public class FSHLog implements WAL { // here inside this single appending/writing thread. Events are ordered on the ringbuffer // so region sequenceids will also be in order. regionSequenceId = entry.stampRegionSequenceId(); - + // Edits are empty, there is nothing to append. Maybe empty when we are looking for a // region sequence id only, a region edit/sequence id that is not associated with an actual // edit. It has to go through all the rigmarole to be sure we have the right ordering. if (entry.getEdit().isEmpty()) { return; } - + // Coprocessor hook. if (!coprocessorHost.preWALWrite(entry.getHRegionInfo(), entry.getKey(), entry.getEdit())) { @@ -2067,13 +1840,8 @@ public class FSHLog implements WAL { writer.append(entry); assert highestUnsyncedSequence < entry.getSequence(); highestUnsyncedSequence = entry.getSequence(); - Long lRegionSequenceId = Long.valueOf(regionSequenceId); - highestRegionSequenceIds.put(encodedRegionName, lRegionSequenceId); - if (entry.isInMemstore()) { - updateOldestUnflushedSequenceIds(encodedRegionName, - entry.getFamilyNames(), lRegionSequenceId); - } - + sequenceIdAccounting.update(encodedRegionName, entry.getFamilyNames(), regionSequenceId, + entry.isInMemstore()); coprocessorHost.postWALWrite(entry.getHRegionInfo(), entry.getKey(), entry.getEdit()); // Update metrics. postAppend(entry, EnvironmentEdgeManager.currentTime() - start); @@ -2203,4 +1971,4 @@ public class FSHLog implements WAL { } return new DatanodeInfo[0]; } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java new file mode 100644 index 0000000..5fa4a83 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceIdAccounting.java @@ -0,0 +1,349 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.util.Bytes; + +import com.google.common.collect.Maps; + +/** + * Accounting of sequence ids per region and then by column family. So we can our accounting + * current, call startCacheFlush and then finishedCacheFlush or abortCacheFlush so this instance + * can keep abreast of the state of sequence id persistence. Also call update per append. + */ +class SequenceIdAccounting { + private static final Log LOG = LogFactory.getLog(SequenceIdAccounting.class); + /** + * This lock ties all operations on {@link SequenceIdAccounting#flushingSequenceIds} and + * {@link #lowestUnflushedSequenceIds} Maps. {@link #lowestUnflushedSequenceIds} has the + * lowest outstanding sequence ids EXCEPT when flushing. When we flush, the current + * lowest set for the region/column family are moved (atomically because of this lock) to + * {@link #flushingSequenceIds}. + * + *

    The two Maps are tied by this locking object EXCEPT when we go to update the lowest + * entry; see {@link #lowest(byte[], Set, Long)}. In here is a putIfAbsent call on + * {@link #lowestUnflushedSequenceIds}. In this latter case, we will add this lowest + * sequence id if we find that there is no entry for the current column family. There will be no + * entry only if we just came up OR we have moved aside current set of lowest sequence ids + * because the current set are being flushed (by putting them into {@link #flushingSequenceIds}). + * This is how we pick up the next 'lowest' sequence id per region per column family to be used + * figuring what is in the next flush. + */ + private final Object tieLock = new Object(); + + /** + * Map of encoded region names and family names to their OLDEST -- i.e. their first, + * the longest-lived, their 'earliest', the 'lowest' -- sequence id. + * + *

    When we flush, the current lowest sequence ids get cleared and added to + * {@link #flushingSequenceIds}. The next append that comes in, is then added + * here to {@link #lowestUnflushedSequenceIds} as the next lowest sequenceid. + * + *

    If flush fails, currently server is aborted so no need to restore previous sequence ids. + *

    Needs to be concurrent Maps because we use putIfAbsent updating oldest. + */ + private final ConcurrentMap> lowestUnflushedSequenceIds + = new ConcurrentSkipListMap>( + Bytes.BYTES_COMPARATOR); + + /** + * Map of encoded region names and family names to their lowest or OLDEST sequence/edit id + * currently being flushed out to hfiles. Entries are moved here from + * {@link #lowestUnflushedSequenceIds} while the lock {@link #tieLock} is held + * (so movement between the Maps is atomic). + */ + private final Map> flushingSequenceIds = + new TreeMap>(Bytes.BYTES_COMPARATOR); + + /** + * Map of region encoded names to the latest/highest region sequence id. Updated on each + * call to append. + *

    + * This map uses byte[] as the key, and uses reference equality. It works in our use case as we + * use {@link HRegionInfo#getEncodedNameAsBytes()} as keys. For a given region, it always returns + * the same array. + */ + private Map highestSequenceIds = new HashMap(); + + /** + * @param encodedRegionName + * @return Lowest outstanding sequenceid for encodedRegionname. Will return + * {@link HConstants#NO_SEQNUM} when none. + */ + long getLowestSequenceId(final byte [] encodedRegionName) { + synchronized (this.tieLock) { + // A flush may be going on and this.flushingSequenceIds may be populated with + // sequenceids. We cannot use it though for it even if the below + // this.lowestUnflushedSequenceIds is empty because it may have a subset of all column + // families in this region only -- we can't tell in this context. + Map m = this.lowestUnflushedSequenceIds.get(encodedRegionName); + return m != null? getLowestSequenceId(m): HConstants.NO_SEQNUM; + } + } + + /** + * @param encodedRegionName + * @param familyName + * @return Lowest outstanding sequenceid for encodedRegionname and + * familyName. Returned sequenceid may be for an edit currently being flushed. + */ + long getLowestSequenceId(final byte [] encodedRegionName, final byte [] familyName) { + synchronized (this.tieLock) { + Map m = this.flushingSequenceIds.get(encodedRegionName); + if (m != null) { + Long lowest = m.get(familyName); + if (lowest != null) return lowest; + } + m = this.lowestUnflushedSequenceIds.get(encodedRegionName); + if (m != null) { + Long lowest = m.get(familyName); + if (lowest != null) return lowest; + } + } + return HConstants.NO_SEQNUM; + } + + /** + * Reset the accounting of highest sequenceid by regionname. + * @return Return the previous accounting Map of regions to the last sequence id written into + * each. + */ + Map resetHighest() { + Map old = this.highestSequenceIds; + this.highestSequenceIds = new HashMap(); + return old; + } + + /** + * We've been passed a new sequenceid for the region. Set it as highest seen for this region and + * if we are to record oldest, or lowest sequenceids, save it as oldest seen if nothing + * currently older. + * @param encodedRegionName + * @param families + * @param sequenceid + * @param lowest Whether to keep running account of oldest sequence id. + */ + void update(byte[] encodedRegionName, Set families, long sequenceid, + final boolean lowest) { + Long l = Long.valueOf(sequenceid); + this.highestSequenceIds.put(encodedRegionName, l); + if (lowest) { + ConcurrentMap m = getOrCreateLowestSequenceIds(encodedRegionName); + for (byte[] familyName : families) { + m.putIfAbsent(familyName, l); + } + } + } + + ConcurrentMap getOrCreateLowestSequenceIds(byte[] encodedRegionName) { + // Intentionally, this access is done outside of this.regionSequenceIdLock. Done per append. + ConcurrentMap m = this.lowestUnflushedSequenceIds.get(encodedRegionName); + if (m != null) return m; + m = new ConcurrentSkipListMap(Bytes.BYTES_COMPARATOR); + // Another thread may have added it ahead of us. + ConcurrentMap alreadyPut = + this.lowestUnflushedSequenceIds.putIfAbsent(encodedRegionName, m); + return alreadyPut == null? m : alreadyPut; + } + + /** + * @param sequenceids Map to search for lowest value. + * @return Lowest value found in sequenceids. + */ + static long getLowestSequenceId(Map sequenceids) { + long lowest = HConstants.NO_SEQNUM; + for (Long sid: sequenceids.values()) { + if (lowest == HConstants.NO_SEQNUM || sid.longValue() < lowest) { + lowest = sid.longValue(); + } + } + return lowest; + } + + /** + * @param src + * @return New Map that has same keys as src but instead of a Map for a value, it + * instead has found the smallest sequence id and it returns that as the value instead. + */ + private > Map flattenToLowestSequenceId( + Map src) { + if (src == null || src.isEmpty()) return null; + Map tgt = Maps.newHashMap(); + for (Map.Entry entry: src.entrySet()) { + long lowestSeqId = getLowestSequenceId(entry.getValue()); + if (lowestSeqId != HConstants.NO_SEQNUM) { + tgt.put(entry.getKey(), lowestSeqId); + } + } + return tgt; + } + + void startCacheFlush(final byte[] encodedRegionName, Set families) { + Map oldSequenceIds = null; + Map m = null; + synchronized (tieLock) { + m = this.lowestUnflushedSequenceIds.get(encodedRegionName); + if (m != null) { + // NOTE: Removal from this.lowestUnflushedSequenceIds must be done in controlled + // circumstance because another concurrent thread now may add sequenceids for this family + // (see above in getOrCreateLowestSequenceId). Make sure you are ok with this. Usually it + // is fine because updates are blocked when this method is called. Make sure!!! + for (byte[] familyName: families) { + Long seqId = m.remove(familyName); + if (seqId != null) { + if (oldSequenceIds == null) oldSequenceIds = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); + oldSequenceIds.put(familyName, seqId); + } + } + if (!oldSequenceIds.isEmpty()) { + if (this.flushingSequenceIds.put(encodedRegionName, oldSequenceIds) != null) { + LOG.warn("Flushing Map not clean up for " + Bytes.toString(encodedRegionName) + + ", sequenceid=" + oldSequenceIds); + } + } + if (m.isEmpty()) { + // Remove it otherwise it will be in oldestUnflushedStoreSequenceIds for ever + // even if the region is already moved to other server. + // Do not worry about data racing, we held write lock of region when calling + // startCacheFlush, so no one can add value to the map we removed. + this.lowestUnflushedSequenceIds.remove(encodedRegionName); + } + } + } + // Do this check outside lock. m can be null legitimately at flush time + if (m != null && oldSequenceIds.isEmpty()) { + // TODO: if we have no oldStoreSeqNum, and WAL is not disabled, presumably either + // the region is already flushing (which would make this call invalid), or there + // were no appends after last flush, so why are we starting flush? Maybe we should + // assert not empty. Less rigorous, but safer, alternative is telling the caller to stop. + // For now preserve old logic. + LOG.warn("Couldn't find oldest sequenceid for " + Bytes.toString(encodedRegionName)); + } + } + + void completeCacheFlush(final byte [] encodedRegionName) { + synchronized (tieLock) { + this.flushingSequenceIds.remove(encodedRegionName); + } + } + + void abortCacheFlush(final byte[] encodedRegionName) { + // Method is called when we are crashing down because failed write flush AND it is called + // if we fail prepare. The below is for the fail prepare case; we restore the old sequence ids. + Map flushing = null; + Map tmpMap = new TreeMap(Bytes.BYTES_COMPARATOR); + // Here we are moving sequenceids from flushing back to unflushed; doing opposite of what + // happened in startCacheFlush. During prepare phase, we have update lock on the region so + // no edits should be coming in via append. + synchronized (tieLock) { + flushing = this.flushingSequenceIds.remove(encodedRegionName); + if (flushing != null) { + Map unflushed = getOrCreateLowestSequenceIds(encodedRegionName); + for (Map.Entry e: flushing.entrySet()) { + // Set into unflushed the 'old' oldest sequenceid and if any value in flushed with this + // value, it will now be in tmpMap. + tmpMap.put(e.getKey(), unflushed.put(e.getKey(), e.getValue())); + } + } + } + + // Here we are doing some 'test' to see if edits are going in out of order. What is it for? + // Carried over from old code. + if (flushing != null) { + for (Map.Entry e : flushing.entrySet()) { + Long currentId = tmpMap.get(e.getKey()); + if (currentId != null && currentId.longValue() <= e.getValue().longValue()) { + String errorStr = Bytes.toString(encodedRegionName) + " family " + + Bytes.toString(e.getKey()) + " acquired edits out of order current memstore seq=" + + currentId + ", previous oldest unflushed id=" + e.getValue(); + LOG.error(errorStr); + Runtime.getRuntime().halt(1); + } + } + } + } + + /** + * See if passed sequenceids are lower -- i.e. earlier -- than any outstanding + * sequenceids, sequenceids we are holding on to in this accounting instance. + * @param sequenceids Keyed by encoded region name. Cannot be null (doesn't make + * sense for it to be null). + * @return true if all sequenceids are lower, older than, the old sequenceids in this instance. + */ + boolean areAllLower(Map sequenceids) { + Map flushing = null; + Map unflushed = null; + synchronized (this.tieLock) { + // Get a flattened -- only the oldest sequenceid -- copy of current flushing and unflushed + // data structures to use in tests below. + flushing = flattenToLowestSequenceId(this.flushingSequenceIds); + unflushed = flattenToLowestSequenceId(this.lowestUnflushedSequenceIds); + } + for (Map.Entry e : sequenceids.entrySet()) { + long oldestFlushing = Long.MAX_VALUE; + long oldestUnflushed = Long.MAX_VALUE; + if (flushing != null) { + if (flushing.containsKey(e.getKey())) oldestFlushing = flushing.get(e.getKey()); + } + if (unflushed != null) { + if (unflushed.containsKey(e.getKey())) oldestUnflushed = unflushed.get(e.getKey()); + } + long min = Math.min(oldestFlushing, oldestUnflushed); + if (min <= e.getValue()) return false; + } + return true; + } + + /** + * Iterates over the given Map and compares sequence ids with corresponding + * entries in {@link #oldestUnflushedRegionSequenceIds}. If a region in + * {@link #oldestUnflushedRegionSequenceIds} has a sequence id less than that passed + * in sequenceids then return it. + * @param sequenceids Sequenceids keyed by encoded region name. + * @return regions found in this instance with sequence ids less than those passed in. + */ + byte[][] findLower(Map sequenceids) { + List toFlush = null; + // Keeping the old behavior of iterating unflushedSeqNums under oldestSeqNumsLock. + synchronized (tieLock) { + for (Map.Entry e: sequenceids.entrySet()) { + Map m = this.lowestUnflushedSequenceIds.get(e.getKey()); + if (m == null) continue; + // The lowest sequence id outstanding for this region. + long lowest = getLowestSequenceId(m); + if (lowest != HConstants.NO_SEQNUM && lowest <= e.getValue()) { + if (toFlush == null) toFlush = new ArrayList(); + toFlush.add(e.getKey()); + } + } + } + return toFlush == null? null: toFlush.toArray(new byte[][] { HConstants.EMPTY_BYTE_ARRAY }); + } +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java index 57adbc0..367952b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/TableAuthManager.java @@ -391,7 +391,7 @@ public class TableAuthManager { public boolean authorize(User user, String namespace, Permission.Action action) { // Global authorizations supercede namespace level - if (authorizeUser(user, action)) { + if (authorize(user, action)) { return true; } // Check namespace permissions @@ -430,14 +430,6 @@ public class TableAuthManager { } /** - * Checks global authorization for a specific action for a user, based on the - * stored user permissions. - */ - public boolean authorizeUser(User user, Permission.Action action) { - return authorize(globalCache.getUser(user.getShortName()), action); - } - - /** * Checks authorization to a given table and column family for a user, based on the * stored user permissions. * diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConfigurationUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConfigurationUtil.java new file mode 100644 index 0000000..598d973 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/ConfigurationUtil.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import com.google.common.collect.Lists; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +import java.util.AbstractMap; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * Utilities for storing more complex collection types in + * {@link org.apache.hadoop.conf.Configuration} instances. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public final class ConfigurationUtil { + // TODO: hopefully this is a good delimiter; it's not in the base64 alphabet, + // nor is it valid for paths + public static final char KVP_DELIMITER = '^'; + + // Disallow instantiation + private ConfigurationUtil() { + + } + + /** + * Store a collection of Map.Entry's in conf, with each entry separated by ',' + * and key values delimited by {@link #KVP_DELIMITER} + * + * @param conf configuration to store the collection in + * @param key overall key to store keyValues under + * @param keyValues kvps to be stored under key in conf + */ + public static void setKeyValues(Configuration conf, String key, + Collection> keyValues) { + setKeyValues(conf, key, keyValues, KVP_DELIMITER); + } + + /** + * Store a collection of Map.Entry's in conf, with each entry separated by ',' + * and key values delimited by delimiter. + * + * @param conf configuration to store the collection in + * @param key overall key to store keyValues under + * @param keyValues kvps to be stored under key in conf + * @param delimiter character used to separate each kvp + */ + public static void setKeyValues(Configuration conf, String key, + Collection> keyValues, char delimiter) { + List serializedKvps = Lists.newArrayList(); + + for (Map.Entry kvp : keyValues) { + serializedKvps.add(kvp.getKey() + delimiter + kvp.getValue()); + } + + conf.setStrings(key, serializedKvps.toArray(new String[serializedKvps.size()])); + } + + /** + * Retrieve a list of key value pairs from configuration, stored under the provided key + * + * @param conf configuration to retrieve kvps from + * @param key key under which the key values are stored + * @return the list of kvps stored under key in conf, or null if the key isn't present. + * @see #setKeyValues(Configuration, String, Collection, char) + */ + public static List> getKeyValues(Configuration conf, String key) { + return getKeyValues(conf, key, KVP_DELIMITER); + } + + /** + * Retrieve a list of key value pairs from configuration, stored under the provided key + * + * @param conf configuration to retrieve kvps from + * @param key key under which the key values are stored + * @param delimiter character used to separate each kvp + * @return the list of kvps stored under key in conf, or null if the key isn't present. + * @see #setKeyValues(Configuration, String, Collection, char) + */ + public static List> getKeyValues(Configuration conf, String key, + char delimiter) { + String[] kvps = conf.getStrings(key); + + if (kvps == null) { + return null; + } + + List> rtn = Lists.newArrayList(); + + for (String kvp : kvps) { + String[] splitKvp = StringUtils.split(kvp, delimiter); + + if (splitKvp.length != 2) { + throw new IllegalArgumentException( + "Expected key value pair for configuration key '" + key + "'" + " to be of form '" + + delimiter + "; was " + kvp + " instead"); + } + + rtn.add(new AbstractMap.SimpleImmutableEntry(splitKvp[0], splitKvp[1])); + } + return rtn; + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java index 0fffcc6..1360fb2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java @@ -170,7 +170,7 @@ public class FSHDFSUtils extends FSUtils { boolean recoverDFSFileLease(final DistributedFileSystem dfs, final Path p, final Configuration conf, final CancelableProgressable reporter) throws IOException { - LOG.info("Recovering lease on dfs file " + p); + LOG.info("Recover lease on dfs file " + p); long startWaiting = EnvironmentEdgeManager.currentTime(); // Default is 15 minutes. It's huge, but the idea is that if we have a major issue, HDFS // usually needs 10 minutes before marking the nodes as dead. So we're putting ourselves @@ -259,7 +259,7 @@ public class FSHDFSUtils extends FSUtils { boolean recovered = false; try { recovered = dfs.recoverLease(p); - LOG.info("recoverLease=" + recovered + ", " + + LOG.info((recovered? "Recovered lease, ": "Failed to recover lease, ") + getLogMessageDetail(nbAttempt, p, startWaiting)); } catch (IOException e) { if (e instanceof LeaseExpiredException && e.getMessage().contains("File does not exist")) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java index ae689d1..d0b1f42 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java @@ -505,7 +505,7 @@ public abstract class FSUtils { /** * We use reflection because {@link DistributedFileSystem#setSafeMode( - * FSConstants.SafeModeAction action, boolean isChecked)} is not in hadoop 1.1 + * HdfsConstants.SafeModeAction action, boolean isChecked)} is not in hadoop 1.1 * * @param dfs * @return whether we're in safe mode @@ -515,15 +515,15 @@ public abstract class FSUtils { boolean inSafeMode = false; try { Method m = DistributedFileSystem.class.getMethod("setSafeMode", new Class []{ - org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.class, boolean.class}); + org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction.class, boolean.class}); inSafeMode = (Boolean) m.invoke(dfs, - org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET, true); + org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction.SAFEMODE_GET, true); } catch (Exception e) { if (e instanceof IOException) throw (IOException) e; // Check whether dfs is on safemode. inSafeMode = dfs.setSafeMode( - org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction.SAFEMODE_GET); + org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction.SAFEMODE_GET); } return inSafeMode; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index b4548f6..5244900 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -63,6 +63,7 @@ import com.google.common.collect.Ordering; import com.google.common.collect.TreeMultimap; import com.google.protobuf.ServiceException; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -138,7 +139,6 @@ import org.apache.hadoop.hbase.zookeeper.ZKTableStateManager; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; -import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; @@ -203,7 +203,13 @@ public class HBaseFsck extends Configured implements Closeable { private static final String TO_BE_LOADED = "to_be_loaded"; private static final String HBCK_LOCK_FILE = "hbase-hbck.lock"; private static final int DEFAULT_MAX_LOCK_FILE_ATTEMPTS = 5; - private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; + private static final int DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL = 200; // milliseconds + private static final int DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME = 5000; // milliseconds + // We have to set the timeout value > HdfsConstants.LEASE_SOFTLIMIT_PERIOD. + // In HADOOP-2.6 and later, the Namenode proxy now created with custom RetryPolicy for + // AlreadyBeingCreatedException which is implies timeout on this operations up to + // HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds). + private static final int DEFAULT_WAIT_FOR_LOCK_TIMEOUT = 80; // seconds /********************** * Internal resources @@ -327,9 +333,11 @@ public class HBaseFsck extends Configured implements Closeable { int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS); executor = new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck")); lockFileRetryCounterFactory = new RetryCounterFactory( - getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS), - getConf().getInt("hbase.hbck.lockfile.attempt.sleep.interval", - DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL)); + getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS), + getConf().getInt( + "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL), + getConf().getInt( + "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME)); } /** @@ -348,10 +356,13 @@ public class HBaseFsck extends Configured implements Closeable { errors = getErrorReporter(getConf()); this.executor = exec; lockFileRetryCounterFactory = new RetryCounterFactory( - getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS), - getConf().getInt("hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL)); + getConf().getInt("hbase.hbck.lockfile.attempts", DEFAULT_MAX_LOCK_FILE_ATTEMPTS), + getConf().getInt( + "hbase.hbck.lockfile.attempt.sleep.interval", DEFAULT_LOCK_FILE_ATTEMPT_SLEEP_INTERVAL), + getConf().getInt( + "hbase.hbck.lockfile.attempt.maxsleeptime", DEFAULT_LOCK_FILE_ATTEMPT_MAX_SLEEP_TIME)); } - + private class FileLockCallable implements Callable { RetryCounter retryCounter; @@ -421,10 +432,11 @@ public class HBaseFsck extends Configured implements Closeable { ExecutorService executor = Executors.newFixedThreadPool(1); FutureTask futureTask = new FutureTask(callable); executor.execute(futureTask); - final int timeoutInSeconds = 30; + final int timeoutInSeconds = getConf().getInt( + "hbase.hbck.lockfile.maxwaittime", DEFAULT_WAIT_FOR_LOCK_TIMEOUT); FSDataOutputStream stream = null; try { - stream = futureTask.get(30, TimeUnit.SECONDS); + stream = futureTask.get(timeoutInSeconds, TimeUnit.SECONDS); } catch (ExecutionException ee) { LOG.warn("Encountered exception when opening lock file", ee); } catch (InterruptedException ie) { @@ -445,9 +457,10 @@ public class HBaseFsck extends Configured implements Closeable { RetryCounter retryCounter = lockFileRetryCounterFactory.create(); do { try { - IOUtils.closeStream(hbckOutFd); + IOUtils.closeQuietly(hbckOutFd); FSUtils.delete(FSUtils.getCurrentFileSystem(getConf()), HBCK_LOCK_PATH, true); + LOG.info("Finishing hbck"); return; } catch (IOException ioe) { LOG.info("Failed to delete " + HBCK_LOCK_PATH + ", try=" @@ -464,7 +477,6 @@ public class HBaseFsck extends Configured implements Closeable { } } } while (retryCounter.shouldRetry()); - } } @@ -487,17 +499,18 @@ public class HBaseFsck extends Configured implements Closeable { // Make sure to cleanup the lock hbckLockCleanup.set(true); - // Add a shutdown hook to this thread, incase user tries to + // Add a shutdown hook to this thread, in case user tries to // kill the hbck with a ctrl-c, we want to cleanup the lock so that // it is available for further calls Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { - IOUtils.closeStream(HBaseFsck.this); + IOUtils.closeQuietly(HBaseFsck.this); unlockHbck(); } }); - LOG.debug("Launching hbck"); + + LOG.info("Launching hbck"); connection = (ClusterConnection)ConnectionFactory.createConnection(getConf()); admin = connection.getAdmin(); @@ -720,7 +733,9 @@ public class HBaseFsck extends Configured implements Closeable { @Override public void close() throws IOException { - IOUtils.cleanup(null, admin, meta, connection); + IOUtils.closeQuietly(admin); + IOUtils.closeQuietly(meta); + IOUtils.closeQuietly(connection); } private static class RegionBoundariesInformation { @@ -3334,7 +3349,11 @@ public class HBaseFsck extends Configured implements Closeable { KeeperException { undeployRegions(hi); ZooKeeperWatcher zkw = createZooKeeperWatcher(); - ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(hi.metaEntry.getReplicaId())); + try { + ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(hi.metaEntry.getReplicaId())); + } finally { + zkw.close(); + } } private void assignMetaReplica(int replicaId) @@ -4694,7 +4713,7 @@ public class HBaseFsck extends Configured implements Closeable { setRetCode(code); } } finally { - IOUtils.cleanup(null, this); + IOUtils.closeQuietly(this); } return this; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java index 5a2b08d..eb94a5b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WAL.java @@ -141,30 +141,36 @@ public interface WAL { /** * WAL keeps track of the sequence numbers that were not yet flushed from memstores - * in order to be able to do cleanup. This method tells WAL that some region is about - * to flush memstore. + * in order to be able to do accounting to figure which WALs can be let go. This method tells WAL + * that some region is about to flush memstore. * *

    We stash the oldest seqNum for the region, and let the the next edit inserted in this * region be recorded in {@link #append(HTableDescriptor, HRegionInfo, WALKey, WALEdit, * AtomicLong, boolean, List)} as new oldest seqnum. - * In case of flush being aborted, we put the stashed value back; in case of flush succeeding, + * + *

    In case of flush being aborted, we put the stashed value back; in case of flush succeeding, * the seqNum of that first edit after start becomes the valid oldest seqNum for this region. + * + *

    Currently, it expects that the update lock is held for the region; it will not work if + * appends are going on concurrently. See implementation for detail. * - * @return true if the flush can proceed, false in case wal is closing (ususally, when server is + * @return true if the flush can proceed, false in case wal is closing (usually, when server is * closing) and flush couldn't be started. + * @see #completeCacheFlush(byte[]) */ boolean startCacheFlush(final byte[] encodedRegionName, Set flushedFamilyNames); /** * Complete the cache flush. * @param encodedRegionName Encoded region name. + * @see #startCacheFlush(byte[], Set) */ void completeCacheFlush(final byte[] encodedRegionName); /** * Abort a cache flush. Call if the flush fails. Note that the only recovery * for an aborted flush currently is a restart of the regionserver so the - * snapshot content dropped by the failure gets restored to the memstore.v + * snapshot content dropped by the failure gets restored to the memstore. * @param encodedRegionName Encoded region name. */ void abortCacheFlush(byte[] encodedRegionName); @@ -175,15 +181,15 @@ public interface WAL { WALCoprocessorHost getCoprocessorHost(); - /** Gets the earliest sequence number in the memstore for this particular region. - * This can serve as best-effort "recent" WAL number for this region. + /** + * Gets the oldest unflushed sequence id in the memstore for this particular region. * @param encodedRegionName The region to get the number for. - * @return The number if present, HConstants.NO_SEQNUM if absent. + * @return The sequence number if present, HConstants.NO_SEQNUM if absent. */ long getEarliestMemstoreSeqNum(byte[] encodedRegionName); /** - * Gets the earliest sequence number in the memstore for this particular region and store. + * Gets the oldest unflushed sequence id in the memstore for this particular region and store. * @param encodedRegionName The region to get the number for. * @param familyName The family to get the number for. * @return The number if present, HConstants.NO_SEQNUM if absent. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java index c55280b..f8104d2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java @@ -124,6 +124,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.protobuf.ServiceException; +import com.google.protobuf.TextFormat; /** * This class is responsible for splitting up a bunch of regionserver commit log @@ -324,15 +325,19 @@ public class WALSplitter { failedServerName = (serverName == null) ? "" : serverName.getServerName(); while ((entry = getNextLogLine(in, logPath, skipErrors)) != null) { byte[] region = entry.getKey().getEncodedRegionName(); - String key = Bytes.toString(region); - lastFlushedSequenceId = lastFlushedSequenceIds.get(key); + String encodedRegionNameAsStr = Bytes.toString(region); + lastFlushedSequenceId = lastFlushedSequenceIds.get(encodedRegionNameAsStr); if (lastFlushedSequenceId == null) { if (this.distributedLogReplay) { RegionStoreSequenceIds ids = csm.getSplitLogWorkerCoordination().getRegionFlushedSequenceId(failedServerName, - key); + encodedRegionNameAsStr); if (ids != null) { lastFlushedSequenceId = ids.getLastFlushedSequenceId(); + if (LOG.isDebugEnabled()) { + LOG.debug("DLR Last flushed sequenceid for " + encodedRegionNameAsStr + ": " + + TextFormat.shortDebugString(ids)); + } } } else if (sequenceIdChecker != null) { RegionStoreSequenceIds ids = sequenceIdChecker.getLastSequenceId(region); @@ -341,13 +346,17 @@ public class WALSplitter { maxSeqIdInStores.put(storeSeqId.getFamilyName().toByteArray(), storeSeqId.getSequenceId()); } - regionMaxSeqIdInStores.put(key, maxSeqIdInStores); + regionMaxSeqIdInStores.put(encodedRegionNameAsStr, maxSeqIdInStores); lastFlushedSequenceId = ids.getLastFlushedSequenceId(); + if (LOG.isDebugEnabled()) { + LOG.debug("DLS Last flushed sequenceid for " + encodedRegionNameAsStr + ": " + + TextFormat.shortDebugString(ids)); + } } if (lastFlushedSequenceId == null) { lastFlushedSequenceId = -1L; } - lastFlushedSequenceIds.put(key, lastFlushedSequenceId); + lastFlushedSequenceIds.put(encodedRegionNameAsStr, lastFlushedSequenceId); } if (lastFlushedSequenceId >= entry.getKey().getLogSeqNum()) { editsSkipped++; @@ -1071,7 +1080,7 @@ public class WALSplitter { } private void doRun() throws IOException { - LOG.debug("Writer thread " + this + ": starting"); + if (LOG.isTraceEnabled()) LOG.trace("Writer thread starting"); while (true) { RegionEntryBuffer buffer = entryBuffers.getChunkToWrite(); if (buffer == null) { @@ -1226,7 +1235,8 @@ public class WALSplitter { } } controller.checkForErrors(); - LOG.info("Split writers finished"); + LOG.info((this.writerThreads == null? 0: this.writerThreads.size()) + + " split writers finished; closing..."); return (!progress_failed); } @@ -1317,12 +1327,14 @@ public class WALSplitter { CompletionService completionService = new ExecutorCompletionService(closeThreadPool); for (final Map.Entry writersEntry : writers.entrySet()) { - LOG.debug("Submitting close of " + ((WriterAndPath)writersEntry.getValue()).p); + if (LOG.isTraceEnabled()) { + LOG.trace("Submitting close of " + ((WriterAndPath)writersEntry.getValue()).p); + } completionService.submit(new Callable() { @Override public Void call() throws Exception { WriterAndPath wap = (WriterAndPath) writersEntry.getValue(); - LOG.debug("Closing " + wap.p); + if (LOG.isTraceEnabled()) LOG.trace("Closing " + wap.p); try { wap.w.close(); } catch (IOException ioe) { @@ -1330,8 +1342,8 @@ public class WALSplitter { thrown.add(ioe); return null; } - LOG.info("Closed wap " + wap.p + " (wrote " + wap.editsWritten + " edits in " - + (wap.nanosSpent / 1000 / 1000) + "ms)"); + LOG.info("Closed " + wap.p + "; wrote " + wap.editsWritten + " edit(s) in " + + (wap.nanosSpent / 1000 / 1000) + "ms"); if (wap.editsWritten == 0) { // just remove the empty recovered.edits file @@ -1490,8 +1502,8 @@ public class WALSplitter { } } Writer w = createWriter(regionedits); - LOG.info("Creating writer path=" + regionedits + " region=" + Bytes.toStringBinary(region)); - return (new WriterAndPath(regionedits, w)); + LOG.debug("Creating writer path=" + regionedits); + return new WriterAndPath(regionedits, w); } private void filterCellByStore(Entry logEntry) { @@ -1544,9 +1556,9 @@ public class WALSplitter { filterCellByStore(logEntry); if (!logEntry.getEdit().isEmpty()) { wap.w.append(logEntry); + this.updateRegionMaximumEditLogSeqNum(logEntry); + editsCount++; } - this.updateRegionMaximumEditLogSeqNum(logEntry); - editsCount++; } // Pass along summary statistics wap.incrementEdits(editsCount); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index fad724b..982c0b9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -131,7 +131,7 @@ import static org.junit.Assert.fail; * Create an instance and keep it around testing HBase. This class is * meant to be your one-stop shop for anything you might need testing. Manages * one cluster at a time only. Managed cluster can be an in-process - * {@link MiniHBaseCluster}, or a deployed cluster of type {@link DistributedHBaseCluster}. + * {@link MiniHBaseCluster}, or a deployed cluster of type {@link HBaseCluster}. * Not all methods work with the real cluster. * Depends on log4j being on classpath and * hbase-site.xml for logging and test-run configuration. It does not set @@ -283,6 +283,15 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { return htu; } + /** + * Close the Region {@code r}. For use in tests. + */ + public static void closeRegion(final Region r) throws IOException { + if (r != null) { + ((HRegion)r).close(); + } + } + /** * Returns this classes's instance of {@link Configuration}. Be careful how * you use the returned Configuration since {@link HConnection} instances @@ -1852,6 +1861,18 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { } /** + * Create an HRegion. Be sure to call {@link HBaseTestingUtility#closeRegion(Region)} + * when you're finished with it. + */ + public HRegion createHRegion( + final HRegionInfo info, + final Path rootDir, + final Configuration conf, + final HTableDescriptor htd) throws IOException { + return HRegion.createHRegion(info, rootDir, conf, htd); + } + + /** * Create an HRegion that writes to the local tmp dirs * @param desc * @param startKey @@ -1921,22 +1942,24 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { // ========================================================================== /** - * Provide an existing table name to truncate + * Provide an existing table name to truncate. + * Scans the table and issues a delete for each row read. * @param tableName existing table * @return HTable to that new table * @throws IOException */ - public HTable truncateTable(byte[] tableName) throws IOException { - return truncateTable(TableName.valueOf(tableName)); + public HTable deleteTableData(byte[] tableName) throws IOException { + return deleteTableData(TableName.valueOf(tableName)); } /** - * Provide an existing table name to truncate + * Provide an existing table name to truncate. + * Scans the table and issues a delete for each row read. * @param tableName existing table * @return HTable to that new table * @throws IOException */ - public HTable truncateTable(TableName tableName) throws IOException { + public HTable deleteTableData(TableName tableName) throws IOException { HTable table = new HTable(getConfiguration(), tableName); Scan scan = new Scan(); ResultScanner resScan = table.getScanner(scan); @@ -1950,6 +1973,58 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { } /** + * Truncate a table using the admin command. + * Effectively disables, deletes, and recreates the table. + * @param tableName table which must exist. + * @param preserveRegions keep the existing split points + * @return HTable for the new table + */ + public HTable truncateTable(final TableName tableName, final boolean preserveRegions) + throws IOException { + Admin admin = getHBaseAdmin(); + admin.truncateTable(tableName, preserveRegions); + return new HTable(getConfiguration(), tableName); + } + + /** + * Truncate a table using the admin command. + * Effectively disables, deletes, and recreates the table. + * For previous behavior of issuing row deletes, see + * deleteTableData. + * Expressly does not preserve regions of existing table. + * @param tableName table which must exist. + * @return HTable for the new table + */ + public HTable truncateTable(final TableName tableName) throws IOException { + return truncateTable(tableName, false); + } + + /** + * Truncate a table using the admin command. + * Effectively disables, deletes, and recreates the table. + * @param tableName table which must exist. + * @param preserveRegions keep the existing split points + * @return HTable for the new table + */ + public HTable truncateTable(final byte[] tableName, final boolean preserveRegions) + throws IOException { + return truncateTable(TableName.valueOf(tableName), preserveRegions); + } + + /** + * Truncate a table using the admin command. + * Effectively disables, deletes, and recreates the table. + * For previous behavior of issuing row deletes, see + * deleteTableData. + * Expressly does not preserve regions of existing table. + * @param tableName table which must exist. + * @return HTable for the new table + */ + public HTable truncateTable(final byte[] tableName) throws IOException { + return truncateTable(tableName, false); + } + + /** * Load table with rows from 'aaa' to 'zzz'. * @param t Table * @param f Family diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java index cc5c4c3..ec89974 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestHFileArchiving.java @@ -305,7 +305,7 @@ public class TestHFileArchiving { public void testArchiveOnTableFamilyDelete() throws Exception { TableName TABLE_NAME = TableName.valueOf("testArchiveOnTableFamilyDelete"); - UTIL.createTable(TABLE_NAME, TEST_FAM); + UTIL.createTable(TABLE_NAME, new byte[][] {TEST_FAM, Bytes.toBytes("fam2")}); List servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME); // make sure we only have 1 region serving this table diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java index 6f347dd..a2d8690 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java @@ -1296,11 +1296,11 @@ public class TestAdmin1 { @Test (timeout=300000) public void testEnableDisableAddColumnDeleteColumn() throws Exception { - ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL); - TableName tableName = TableName.valueOf("testMasterAdmin"); + ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL); + TableName tableName = TableName.valueOf("testEnableDisableAddColumnDeleteColumn"); TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); while (!ZKTableStateClientSideReader.isEnabledTable(zkw, - TableName.valueOf("testMasterAdmin"))) { + TableName.valueOf("testEnableDisableAddColumnDeleteColumn"))) { Thread.sleep(10); } this.admin.disableTable(tableName); @@ -1320,4 +1320,33 @@ public class TestAdmin1 { this.admin.disableTable(tableName); this.admin.deleteTable(tableName); } + + @Test (timeout=300000) + public void testDeleteLastColumnFamily() throws Exception { + TableName tableName = TableName.valueOf("testDeleteLastColumnFamily"); + TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); + while (!this.admin.isTableEnabled(TableName.valueOf("testDeleteLastColumnFamily"))) { + Thread.sleep(10); + } + + // test for enabled table + try { + this.admin.deleteColumn(tableName, HConstants.CATALOG_FAMILY); + fail("Should have failed to delete the only column family of a table"); + } catch (InvalidFamilyOperationException ex) { + // expected + } + + // test for disabled table + this.admin.disableTable(tableName); + + try { + this.admin.deleteColumn(tableName, HConstants.CATALOG_FAMILY); + fail("Should have failed to delete the only column family of a table"); + } catch (InvalidFamilyOperationException ex) { + // expected + } + + this.admin.deleteTable(tableName); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java index 1fea802..02eca77 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java @@ -5566,6 +5566,14 @@ public class TestFromClientSide { } checkTableIsLegal(htd); + // HBASE-13776 Setting illegal versions for HColumnDescriptor + // does not throw IllegalArgumentException + // finally, minVersions must be less than or equal to maxVersions + hcd.setMaxVersions(4); + hcd.setMinVersions(5); + checkTableIsIllegal(htd); + hcd.setMinVersions(3); + hcd.setScope(-1); checkTableIsIllegal(htd); hcd.setScope(0); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java index 975103f..bb2784d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide3.java @@ -96,7 +96,10 @@ public class TestFromClientSide3 { */ @After public void tearDown() throws Exception { - // Nothing to do. + for (HTableDescriptor htd: TEST_UTIL.getHBaseAdmin().listTables()) { + LOG.info("Tear down, remove table=" + htd.getTableName()); + TEST_UTIL.deleteTable(htd.getTableName()); + } } private void randomCFPuts(Table table, byte[] row, byte[] family, int nPuts) @@ -276,20 +279,20 @@ public class TestFromClientSide3 { // create an empty Put Put put1 = new Put(ROW); actions.add(put1); - + Put put2 = new Put(ANOTHERROW); put2.add(FAMILY, QUALIFIER, VALUE); actions.add(put2); - + table.batch(actions, results); fail("Empty Put should have failed the batch call"); } catch (IllegalArgumentException iae) { - + } finally { table.close(); } } - + @Test public void testHTableExistsMethodSingleRegionSingleGet() throws Exception { @@ -333,6 +336,61 @@ public class TestFromClientSide3 { } @Test + public void testHTableExistsBeforeGet() throws Exception { + Table table = TEST_UTIL.createTable( + Bytes.toBytes("testHTableExistsBeforeGet"), new byte[][] { FAMILY }); + try { + Put put = new Put(ROW); + put.add(FAMILY, QUALIFIER, VALUE); + table.put(put); + + Get get = new Get(ROW); + + boolean exist = table.exists(get); + assertEquals(true, exist); + + Result result = table.get(get); + assertEquals(false, result.isEmpty()); + assertTrue(Bytes.equals(VALUE, result.getValue(FAMILY, QUALIFIER))); + } finally { + table.close(); + } + } + + @Test + public void testHTableExistsAllBeforeGet() throws Exception { + final byte[] ROW2 = Bytes.add(ROW, Bytes.toBytes("2")); + Table table = TEST_UTIL.createTable( + Bytes.toBytes("testHTableExistsAllBeforeGet"), new byte[][] { FAMILY }); + try { + Put put = new Put(ROW); + put.add(FAMILY, QUALIFIER, VALUE); + table.put(put); + put = new Put(ROW2); + put.add(FAMILY, QUALIFIER, VALUE); + table.put(put); + + Get get = new Get(ROW); + Get get2 = new Get(ROW2); + ArrayList getList = new ArrayList(2); + getList.add(get); + getList.add(get2); + + boolean[] exists = table.existsAll(getList); + assertEquals(true, exists[0]); + assertEquals(true, exists[1]); + + Result[] result = table.get(getList); + assertEquals(false, result[0].isEmpty()); + assertTrue(Bytes.equals(VALUE, result[0].getValue(FAMILY, QUALIFIER))); + assertEquals(false, result[1].isEmpty()); + assertTrue(Bytes.equals(VALUE, result[1].getValue(FAMILY, QUALIFIER))); + } finally { + table.close(); + } + } + + @Test public void testHTableExistsMethodMultipleRegionsSingleGet() throws Exception { Table table = TEST_UTIL.createTable( @@ -355,7 +413,7 @@ public class TestFromClientSide3 { @Test public void testHTableExistsMethodMultipleRegionsMultipleGets() throws Exception { HTable table = TEST_UTIL.createTable( - TableName.valueOf("testHTableExistsMethodMultipleRegionsMultipleGets"), + TableName.valueOf("testHTableExistsMethodMultipleRegionsMultipleGets"), new byte[][] { FAMILY }, 1, new byte[] { 0x00 }, new byte[] { (byte) 0xff }, 255); Put put = new Put(ROW); put.add(FAMILY, QUALIFIER, VALUE); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java index f871123..4e44d2c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilter.java @@ -28,232 +28,291 @@ import org.junit.experimental.categories.Category; @Category(SmallTests.class) public class TestFuzzyRowFilter { @Test - public void testSatisfiesForward() { - Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, - FuzzyRowFilter.satisfies(false, - new byte[]{1, (byte) -128, 0, 0, 1}, // row to check - new byte[]{1, 0, 1}, // fuzzy row - new byte[]{0, 1, 0})); // mask + public void testSatisfiesNoUnsafeForward() { Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES, - FuzzyRowFilter.satisfies(false, + FuzzyRowFilter.satisfiesNoUnsafe(false, new byte[]{1, (byte) -128, 1, 0, 1}, + 0, 5, new byte[]{1, 0, 1}, new byte[]{0, 1, 0})); Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, - FuzzyRowFilter.satisfies(false, + FuzzyRowFilter.satisfiesNoUnsafe(false, new byte[]{1, (byte) -128, 2, 0, 1}, + 0, 5, new byte[]{1, 0, 1}, new byte[]{0, 1, 0})); - Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT, - FuzzyRowFilter.satisfies(false, - new byte[]{2, 3, 1, 1, 1}, - new byte[]{1, 0, 1}, - new byte[]{0, 1, 0})); Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES, - FuzzyRowFilter.satisfies(false, + FuzzyRowFilter.satisfiesNoUnsafe(false, new byte[]{1, 2, 1, 3, 3}, + 0, 5, new byte[]{1, 2, 0, 3}, new byte[]{0, 0, 1, 0})); Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, - FuzzyRowFilter.satisfies(false, + FuzzyRowFilter.satisfiesNoUnsafe(false, new byte[]{1, 1, 1, 3, 0}, // row to check + 0, 5, new byte[]{1, 2, 0, 3}, // fuzzy row new byte[]{0, 0, 1, 0})); // mask Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, - FuzzyRowFilter.satisfies(false, + FuzzyRowFilter.satisfiesNoUnsafe(false, new byte[]{1, 1, 1, 3, 0}, + 0, 5, new byte[]{1, (byte) 245, 0, 3}, new byte[]{0, 0, 1, 0})); - Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT, + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, + FuzzyRowFilter.satisfiesNoUnsafe(false, + new byte[]{1, 2, 1, 0, 1}, + 0, 5, + new byte[]{0, 1, 2}, + new byte[]{1, 0, 0})); + } + + @Test + public void testSatisfiesForward() { + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES, FuzzyRowFilter.satisfies(false, - new byte[]{1, (byte) 245, 1, 3, 0}, - new byte[]{1, 1, 0, 3}, - new byte[]{0, 0, 1, 0})); + new byte[]{1, (byte) -128, 1, 0, 1}, + new byte[]{1, 0, 1}, + new byte[]{-1, 0, -1})); - Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT, + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilter.satisfies(false, - new byte[]{1, 3, 1, 3, 0}, - new byte[]{1, 2, 0, 3}, - new byte[]{0, 0, 1, 0})); + new byte[]{1, (byte) -128, 2, 0, 1}, + new byte[]{1, 0, 1}, + new byte[]{-1, 0, -1})); + - Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT, + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES, FuzzyRowFilter.satisfies(false, - new byte[]{2, 1, 1, 1, 0}, + new byte[]{1, 2, 1, 3, 3}, new byte[]{1, 2, 0, 3}, - new byte[]{0, 0, 1, 0})); + new byte[]{-1, -1, 0, -1})); + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, + FuzzyRowFilter.satisfies(false, + new byte[]{1, 1, 1, 3, 0}, // row to check + new byte[]{1, 2, 0, 3}, // fuzzy row + new byte[]{-1, -1, 0, -1})); // mask + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, + FuzzyRowFilter.satisfies(false, + new byte[]{1, 1, 1, 3, 0}, + new byte[]{1, (byte) 245, 0, 3}, + new byte[]{-1, -1, 0, -1})); Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilter.satisfies(false, new byte[]{1, 2, 1, 0, 1}, new byte[]{0, 1, 2}, - new byte[]{1, 0, 0})); + new byte[]{0, -1, -1})); } @Test public void testSatisfiesReverse() { - Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT, - FuzzyRowFilter.satisfies(true, - new byte[]{1, (byte) -128, 0, 0, 1}, // row to check - new byte[]{1, 0, 1}, // fuzzy row - new byte[]{0, 1, 0})); // mask - Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES, FuzzyRowFilter.satisfies(true, new byte[]{1, (byte) -128, 1, 0, 1}, new byte[]{1, 0, 1}, - new byte[]{0, 1, 0})); + new byte[]{-1, 0, -1})); Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilter.satisfies(true, new byte[]{1, (byte) -128, 2, 0, 1}, new byte[]{1, 0, 1}, - new byte[]{0, 1, 0})); + new byte[]{-1, 0, -1})); Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilter.satisfies(true, new byte[]{2, 3, 1, 1, 1}, new byte[]{1, 0, 1}, - new byte[]{0, 1, 0})); + new byte[]{-1, 0, -1})); Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES, FuzzyRowFilter.satisfies(true, new byte[]{1, 2, 1, 3, 3}, new byte[]{1, 2, 0, 3}, - new byte[]{0, 0, 1, 0})); + new byte[]{-1, -1, 0, -1})); + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, + FuzzyRowFilter.satisfies(true, + new byte[]{1, (byte) 245, 1, 3, 0}, + new byte[]{1, 1, 0, 3}, + new byte[]{-1, -1, 0, -1})); - Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT, + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilter.satisfies(true, - new byte[]{1, 1, 1, 3, 0}, // row to check - new byte[]{1, 2, 0, 3}, // fuzzy row - new byte[]{0, 0, 1, 0})); // mask + new byte[]{1, 3, 1, 3, 0}, + new byte[]{1, 2, 0, 3}, + new byte[]{-1, -1, 0, -1})); - Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NO_NEXT, + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilter.satisfies(true, - new byte[]{1, 1, 1, 3, 0}, - new byte[]{1, (byte) 245, 0, 3}, - new byte[]{0, 0, 1, 0})); + new byte[]{2, 1, 1, 1, 0}, + new byte[]{1, 2, 0, 3}, + new byte[]{-1, -1, 0, -1})); Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, FuzzyRowFilter.satisfies(true, + new byte[]{1, 2, 1, 0, 1}, + new byte[]{0, 1, 2}, + new byte[]{0, -1, -1})); + } + + @Test + public void testSatisfiesNoUnsafeReverse() { + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES, + FuzzyRowFilter.satisfiesNoUnsafe(true, + new byte[]{1, (byte) -128, 1, 0, 1}, + 0, 5, + new byte[]{1, 0, 1}, + new byte[]{0, 1, 0})); + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, + FuzzyRowFilter.satisfiesNoUnsafe(true, + new byte[]{1, (byte) -128, 2, 0, 1}, + 0, 5, + new byte[]{1, 0, 1}, + new byte[]{0, 1, 0})); + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, + FuzzyRowFilter.satisfiesNoUnsafe(true, + new byte[]{2, 3, 1, 1, 1}, + 0, 5, + new byte[]{1, 0, 1}, + new byte[]{0, 1, 0})); + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.YES, + FuzzyRowFilter.satisfiesNoUnsafe(true, + new byte[]{1, 2, 1, 3, 3}, + 0, 5, + new byte[]{1, 2, 0, 3}, + new byte[]{0, 0, 1, 0})); + + Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, + FuzzyRowFilter.satisfiesNoUnsafe(true, new byte[]{1, (byte) 245, 1, 3, 0}, + 0, 5, new byte[]{1, 1, 0, 3}, new byte[]{0, 0, 1, 0})); Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, - FuzzyRowFilter.satisfies(true, + FuzzyRowFilter.satisfiesNoUnsafe(true, new byte[]{1, 3, 1, 3, 0}, + 0, 5, new byte[]{1, 2, 0, 3}, new byte[]{0, 0, 1, 0})); Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, - FuzzyRowFilter.satisfies(true, + FuzzyRowFilter.satisfiesNoUnsafe(true, new byte[]{2, 1, 1, 1, 0}, + 0, 5, new byte[]{1, 2, 0, 3}, new byte[]{0, 0, 1, 0})); Assert.assertEquals(FuzzyRowFilter.SatisfiesCode.NEXT_EXISTS, - FuzzyRowFilter.satisfies(true, + FuzzyRowFilter.satisfiesNoUnsafe(true, new byte[]{1, 2, 1, 0, 1}, + 0, 5, new byte[]{0, 1, 2}, new byte[]{1, 0, 0})); - } - + } @Test public void testGetNextForFuzzyRuleForward() { assertNext(false, new byte[]{0, 1, 2}, // fuzzy row - new byte[]{1, 0, 0}, // mask + new byte[]{0, -1, -1}, // mask new byte[]{1, 2, 1, 0, 1}, // current new byte[]{2, 1, 2, 0, 0}); // expected next assertNext(false, new byte[]{0, 1, 2}, // fuzzy row - new byte[]{1, 0, 0}, // mask + new byte[]{0, -1, -1}, // mask new byte[]{1, 1, 2, 0, 1}, // current new byte[]{1, 1, 2, 0, 2}); // expected next assertNext(false, new byte[]{0, 1, 0, 2, 0}, // fuzzy row - new byte[]{1, 0, 1, 0, 1}, // mask + new byte[]{0, -1, 0, -1, 0}, // mask new byte[]{1, 0, 2, 0, 1}, // current new byte[]{1, 1, 0, 2, 0}); // expected next assertNext(false, new byte[]{1, 0, 1}, - new byte[]{0, 1, 0}, + new byte[]{-1, 0, -1}, new byte[]{1, (byte) 128, 2, 0, 1}, new byte[]{1, (byte) 129, 1, 0, 0}); assertNext(false, new byte[]{0, 1, 0, 1}, - new byte[]{1, 0, 1, 0}, + new byte[]{0, -1, 0, -1}, new byte[]{5, 1, 0, 1}, new byte[]{5, 1, 1, 1}); assertNext(false, new byte[]{0, 1, 0, 1}, - new byte[]{1, 0, 1, 0}, + new byte[]{0, -1, 0, -1}, new byte[]{5, 1, 0, 1, 1}, new byte[]{5, 1, 0, 1, 2}); assertNext(false, new byte[]{0, 1, 0, 0}, // fuzzy row - new byte[]{1, 0, 1, 1}, // mask + new byte[]{0, -1, 0, 0}, // mask new byte[]{5, 1, (byte) 255, 1}, // current new byte[]{5, 1, (byte) 255, 2}); // expected next assertNext(false, new byte[]{0, 1, 0, 1}, // fuzzy row - new byte[]{1, 0, 1, 0}, // mask + new byte[]{0, -1, 0, -1}, // mask new byte[]{5, 1, (byte) 255, 1}, // current new byte[]{6, 1, 0, 1}); // expected next assertNext(false, new byte[]{0, 1, 0, 1}, // fuzzy row - new byte[]{1, 0, 1, 0}, // mask + new byte[]{0, -1, 0, -1}, // mask new byte[]{5, 1, (byte) 255, 0}, // current new byte[]{5, 1, (byte) 255, 1}); // expected next assertNext(false, new byte[]{5, 1, 1, 0}, - new byte[]{0, 0, 1, 1}, + new byte[]{-1, -1, 0, 0}, new byte[]{5, 1, (byte) 255, 1}, new byte[]{5, 1, (byte) 255, 2}); assertNext(false, new byte[]{1, 1, 1, 1}, - new byte[]{0, 0, 1, 1}, + new byte[]{-1, -1, 0, 0}, new byte[]{1, 1, 2, 2}, new byte[]{1, 1, 2, 3}); assertNext(false, new byte[]{1, 1, 1, 1}, - new byte[]{0, 0, 1, 1}, + new byte[]{-1, -1, 0, 0}, new byte[]{1, 1, 3, 2}, new byte[]{1, 1, 3, 3}); assertNext(false, new byte[]{1, 1, 1, 1}, - new byte[]{1, 1, 1, 1}, + new byte[]{0, 0, 0, 0}, new byte[]{1, 1, 2, 3}, new byte[]{1, 1, 2, 4}); assertNext(false, new byte[]{1, 1, 1, 1}, - new byte[]{1, 1, 1, 1}, + new byte[]{0, 0, 0, 0}, new byte[]{1, 1, 3, 2}, new byte[]{1, 1, 3, 3}); assertNext(false, new byte[]{1, 1, 0, 0}, - new byte[]{0, 0, 1, 1}, + new byte[]{-1, -1, 0, 0}, new byte[]{0, 1, 3, 2}, new byte[]{1, 1, 0, 0}); @@ -261,100 +320,100 @@ public class TestFuzzyRowFilter { Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule( new byte[]{2, 3, 1, 1, 1}, // row to check new byte[]{1, 0, 1}, // fuzzy row - new byte[]{0, 1, 0})); // mask + new byte[]{-1, 0, -1})); // mask Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule( new byte[]{1, (byte) 245, 1, 3, 0}, new byte[]{1, 1, 0, 3}, - new byte[]{0, 0, 1, 0})); + new byte[]{-1, -1, 0, -1})); Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule( new byte[]{1, 3, 1, 3, 0}, new byte[]{1, 2, 0, 3}, - new byte[]{0, 0, 1, 0})); + new byte[]{-1, -1, 0, -1})); Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule( new byte[]{2, 1, 1, 1, 0}, new byte[]{1, 2, 0, 3}, - new byte[]{0, 0, 1, 0})); + new byte[]{-1, -1, 0, -1})); } @Test public void testGetNextForFuzzyRuleReverse() { assertNext(true, new byte[]{0, 1, 2}, // fuzzy row - new byte[]{1, 0, 0}, // mask + new byte[]{0, -1, -1}, // mask new byte[]{1, 2, 1, 0, 1}, // current // TODO: should be {1, 1, 3} ? new byte[]{1, 1, 2, (byte) 0xFF, (byte) 0xFF}); // expected next assertNext(true, new byte[]{0, 1, 0, 2, 0}, // fuzzy row - new byte[]{1, 0, 1, 0, 1}, // mask + new byte[]{0, -1, 0, -1, 0}, // mask new byte[]{1, 2, 1, 3, 1}, // current // TODO: should be {1, 1, 1, 3} ? new byte[]{1, 1, 0, 2, 0}); // expected next assertNext(true, new byte[]{1, 0, 1}, - new byte[]{0, 1, 0}, + new byte[]{-1, 0, -1}, new byte[]{1, (byte) 128, 2, 0, 1}, // TODO: should be {1, (byte) 128, 2} ? new byte[]{1, (byte) 128, 1, (byte) 0xFF, (byte) 0xFF}); assertNext(true, new byte[]{0, 1, 0, 1}, - new byte[]{1, 0, 1, 0}, + new byte[]{0, -1, 0, -1}, new byte[]{5, 1, 0, 2, 1}, // TODO: should be {5, 1, 0, 2} ? new byte[]{5, 1, 0, 1, (byte) 0xFF}); assertNext(true, new byte[]{0, 1, 0, 0}, // fuzzy row - new byte[]{1, 0, 1, 1}, // mask + new byte[]{0, -1, 0, 0}, // mask new byte[]{5, 1, (byte) 255, 1}, // current new byte[]{5, 1, (byte) 255, 0}); // expected next assertNext(true, new byte[]{0, 1, 0, 1}, // fuzzy row - new byte[]{1, 0, 1, 0}, // mask + new byte[]{0, -1, 0, -1}, // mask new byte[]{5, 1, 0, 1}, // current new byte[]{4, 1, (byte) 255, 1}); // expected next assertNext(true, new byte[]{0, 1, 0, 1}, // fuzzy row - new byte[]{1, 0, 1, 0}, // mask + new byte[]{0, -1, 0, -1}, // mask new byte[]{5, 1, (byte) 255, 0}, // current new byte[]{5, 1, (byte) 254, 1}); // expected next assertNext(true, new byte[]{1, 1, 0, 0}, - new byte[]{0, 0, 1, 1}, + new byte[]{-1, -1, 0, 0}, new byte[]{2, 1, 3, 2}, // TODO: should be {1, 0} ? new byte[]{1, 1, 0, 0}); assertNext(true, new byte[]{1, 0, 1}, // fuzzy row - new byte[]{0, 1, 0}, // mask + new byte[]{-1, 0, -1}, // mask new byte[]{2, 3, 1, 1, 1}, // row to check // TODO: should be {1, (byte) 0xFF, 2} ? new byte[]{1, 0, 1, (byte) 0xFF, (byte) 0xFF}); assertNext(true, new byte[]{1, 1, 0, 3}, - new byte[]{0, 0, 1, 0}, + new byte[]{-1, -1, 0, -1}, new byte[]{1, (byte) 245, 1, 3, 0}, // TODO: should be {1, 1, (byte) 255, 4} ? new byte[]{1, 1, 0, 3, (byte) 0xFF}); assertNext(true, new byte[]{1, 2, 0, 3}, - new byte[]{0, 0, 1, 0}, + new byte[]{-1, -1, 0, -1}, new byte[]{1, 3, 1, 3, 0}, // TODO: should be 1, 2, (byte) 255, 4 ? new byte[]{1, 2, 0, 3, (byte) 0xFF}); assertNext(true, new byte[]{1, 2, 0, 3}, - new byte[]{0, 0, 1, 0}, + new byte[]{-1, -1, 0, -1}, new byte[]{2, 1, 1, 1, 0}, // TODO: should be {1, 2, (byte) 255, 4} ? new byte[]{1, 2, 0, 3, (byte) 0xFF}); @@ -362,42 +421,42 @@ public class TestFuzzyRowFilter { assertNext(true, // TODO: should be null? new byte[]{1, 0, 1}, - new byte[]{0, 1, 0}, + new byte[]{-1, 0, -1}, new byte[]{1, (byte) 128, 2}, new byte[]{1, (byte) 128, 1}); assertNext(true, // TODO: should be null? new byte[]{0, 1, 0, 1}, - new byte[]{1, 0, 1, 0}, + new byte[]{0, -1, 0, -1}, new byte[]{5, 1, 0, 2}, new byte[]{5, 1, 0, 1}); assertNext(true, // TODO: should be null? new byte[]{5, 1, 1, 0}, - new byte[]{0, 0, 1, 1}, + new byte[]{-1, -1, 0, 0}, new byte[]{5, 1, (byte) 0xFF, 1}, new byte[]{5, 1, (byte) 0xFF, 0}); assertNext(true, // TODO: should be null? new byte[]{1, 1, 1, 1}, - new byte[]{0, 0, 1, 1}, + new byte[]{-1, -1, 0, 0}, new byte[]{1, 1, 2, 2}, new byte[]{1, 1, 2, 1}); assertNext(true, // TODO: should be null? new byte[]{1, 1, 1, 1}, - new byte[]{1, 1, 1, 1}, + new byte[]{0, 0, 0, 0}, new byte[]{1, 1, 2, 3}, new byte[]{1, 1, 2, 2}); Assert.assertNull(FuzzyRowFilter.getNextForFuzzyRule(true, new byte[]{1, 1, 1, 3, 0}, new byte[]{1, 2, 0, 3}, - new byte[]{0, 0, 1, 0})); + new byte[]{-1, -1, 0, -1})); } private static void assertNext(boolean reverse, byte[] fuzzyRow, byte[] mask, byte[] current, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java new file mode 100644 index 0000000..1ff49a7 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFuzzyRowFilterEndToEnd.java @@ -0,0 +1,312 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Durability; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.filter.FilterList.Operator; +import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.RegionScanner; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; + +/** + */ +@Category(MediumTests.class) +public class TestFuzzyRowFilterEndToEnd { + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final Log LOG = LogFactory.getLog(TestFuzzyRowFilterEndToEnd.class); + + private static int firstPartCardinality = 50; + private static int secondPartCardinality = 40; + private static int colQualifiersTotal = 50; + private static int totalFuzzyKeys = secondPartCardinality / 2; + + private static String table = "TestFuzzyRowFilterEndToEnd"; + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + conf.setInt("hbase.client.scanner.caching", 1000); + conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, + ConstantSizeRegionSplitPolicy.class.getName()); + // set no splits + conf.setLong(HConstants.HREGION_MAX_FILESIZE, ((long) 1024) * 1024 * 1024 * 10); + + TEST_UTIL.startMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + // Nothing to do. + } + + /** + * @throws java.lang.Exception + */ + @After + public void tearDown() throws Exception { + // Nothing to do. + } + + @Test + public void testEndToEnd() throws Exception { + String cf = "f"; + + HTable ht = + TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE); + + // 10 byte row key - (2 bytes 4 bytes 4 bytes) + // 4 byte qualifier + // 4 byte value + + for (int i1 = 0; i1 < firstPartCardinality; i1++) { + if ((i1 % 1000) == 0) LOG.info("put " + i1); + + for (int i2 = 0; i2 < secondPartCardinality; i2++) { + byte[] rk = new byte[10]; + + ByteBuffer buf = ByteBuffer.wrap(rk); + buf.clear(); + buf.putShort((short) 2); + buf.putInt(i1); + buf.putInt(i2); + for (int c = 0; c < colQualifiersTotal; c++) { + byte[] cq = new byte[4]; + Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4); + + Put p = new Put(rk); + p.setDurability(Durability.SKIP_WAL); + p.add(cf.getBytes(), cq, Bytes.toBytes(c)); + ht.put(p); + } + } + } + + TEST_UTIL.flush(); + + // test passes + runTest(ht); + + } + + private void runTest(HTable hTable) throws IOException { + // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1] + + byte[] mask = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 }; + + List> list = new ArrayList>(); + for (int i = 0; i < totalFuzzyKeys; i++) { + byte[] fuzzyKey = new byte[10]; + ByteBuffer buf = ByteBuffer.wrap(fuzzyKey); + buf.clear(); + buf.putShort((short) 2); + for (int j = 0; j < 4; j++) { + buf.put((byte) 63); + } + buf.putInt(i); + + Pair pair = new Pair(fuzzyKey, mask); + list.add(pair); + } + + int expectedSize = firstPartCardinality * totalFuzzyKeys * colQualifiersTotal; + FuzzyRowFilter fuzzyRowFilter0 = new FuzzyRowFilter(list); + // Filters are not stateless - we can't reuse them + FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(list); + + // regular test + runScanner(hTable, expectedSize, fuzzyRowFilter0); + // optimized from block cache + runScanner(hTable, expectedSize, fuzzyRowFilter1); + + } + + private void runScanner(HTable hTable, int expectedSize, Filter filter) throws IOException { + + String cf = "f"; + Scan scan = new Scan(); + scan.addFamily(cf.getBytes()); + scan.setFilter(filter); + List regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes()); + HRegion first = regions.get(0); + first.getScanner(scan); + RegionScanner scanner = first.getScanner(scan); + List results = new ArrayList(); + // Result result; + long timeBeforeScan = System.currentTimeMillis(); + int found = 0; + while (scanner.next(results)) { + found += results.size(); + results.clear(); + } + found += results.size(); + long scanTime = System.currentTimeMillis() - timeBeforeScan; + scanner.close(); + + LOG.info("\nscan time = " + scanTime + "ms"); + LOG.info("found " + found + " results\n"); + + assertEquals(expectedSize, found); + } + + @SuppressWarnings("deprecation") + @Test + public void testFilterList() throws Exception { + String cf = "f"; + String table = "TestFuzzyRowFiltersInFilterList"; + HTable ht = + TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE); + + // 10 byte row key - (2 bytes 4 bytes 4 bytes) + // 4 byte qualifier + // 4 byte value + + for (int i1 = 0; i1 < 5; i1++) { + for (int i2 = 0; i2 < 5; i2++) { + byte[] rk = new byte[10]; + + ByteBuffer buf = ByteBuffer.wrap(rk); + buf.clear(); + buf.putShort((short) 2); + buf.putInt(i1); + buf.putInt(i2); + + // Each row contains 5 columns + for (int c = 0; c < 5; c++) { + byte[] cq = new byte[4]; + Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4); + + Put p = new Put(rk); + p.setDurability(Durability.SKIP_WAL); + p.add(cf.getBytes(), cq, Bytes.toBytes(c)); + ht.put(p); + LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: " + + Bytes.toStringBinary(cq)); + } + } + } + + TEST_UTIL.flush(); + + // test passes if we get back 5 KV's (1 row) + runTest(ht, 5); + + } + + @SuppressWarnings("unchecked") + private void runTest(HTable hTable, int expectedSize) throws IOException { + // [0, 2, ?, ?, ?, ?, 0, 0, 0, 1] + byte[] fuzzyKey1 = new byte[10]; + ByteBuffer buf = ByteBuffer.wrap(fuzzyKey1); + buf.clear(); + buf.putShort((short) 2); + for (int i = 0; i < 4; i++) + buf.put((byte) 63); + buf.putInt((short) 1); + byte[] mask1 = new byte[] { 0, 0, 1, 1, 1, 1, 0, 0, 0, 0 }; + + byte[] fuzzyKey2 = new byte[10]; + buf = ByteBuffer.wrap(fuzzyKey2); + buf.clear(); + buf.putShort((short) 2); + buf.putInt((short) 2); + for (int i = 0; i < 4; i++) + buf.put((byte) 63); + + byte[] mask2 = new byte[] { 0, 0, 0, 0, 0, 0, 1, 1, 1, 1 }; + + Pair pair1 = new Pair(fuzzyKey1, mask1); + Pair pair2 = new Pair(fuzzyKey2, mask2); + + FuzzyRowFilter fuzzyRowFilter1 = new FuzzyRowFilter(Lists.newArrayList(pair1)); + FuzzyRowFilter fuzzyRowFilter2 = new FuzzyRowFilter(Lists.newArrayList(pair2)); + // regular test - we expect 1 row back (5 KVs) + runScanner(hTable, expectedSize, fuzzyRowFilter1, fuzzyRowFilter2); + } + + private void runScanner(HTable hTable, int expectedSize, Filter filter1, Filter filter2) throws IOException { + String cf = "f"; + Scan scan = new Scan(); + scan.addFamily(cf.getBytes()); + FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2); + scan.setFilter(filterList); + + ResultScanner scanner = hTable.getScanner(scan); + List results = new ArrayList(); + Result result; + long timeBeforeScan = System.currentTimeMillis(); + while ((result = scanner.next()) != null) { + for (Cell kv : result.listCells()) { + LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: " + + Bytes.toStringBinary(CellUtil.cloneQualifier(kv))); + results.add(kv); + } + } + long scanTime = System.currentTimeMillis() - timeBeforeScan; + scanner.close(); + + LOG.info("scan time = " + scanTime + "ms"); + LOG.info("found " + results.size() + " results"); + + assertEquals(expectedSize, results.size()); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestMultiTableSnapshotInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestMultiTableSnapshotInputFormat.java new file mode 100644 index 0000000..3bb188d --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestMultiTableSnapshotInputFormat.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.mapred; + +import com.google.common.collect.Lists; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapred.FileOutputFormat; +import org.apache.hadoop.mapred.JobClient; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.OutputCollector; +import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.RunningJob; +import org.junit.experimental.categories.Category; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +import static org.junit.Assert.assertTrue; + +@Category({ LargeTests.class }) +public class TestMultiTableSnapshotInputFormat + extends org.apache.hadoop.hbase.mapreduce.TestMultiTableSnapshotInputFormat { + + private static final Log LOG = LogFactory.getLog(TestMultiTableSnapshotInputFormat.class); + + @Override + protected void runJob(String jobName, Configuration c, List scans) + throws IOException, InterruptedException, ClassNotFoundException { + JobConf job = new JobConf(TEST_UTIL.getConfiguration()); + + job.setJobName(jobName); + job.setMapperClass(Mapper.class); + job.setReducerClass(Reducer.class); + + TableMapReduceUtil.initMultiTableSnapshotMapperJob(getSnapshotScanMapping(scans), Mapper.class, + ImmutableBytesWritable.class, ImmutableBytesWritable.class, job, true, restoreDir); + + TableMapReduceUtil.addDependencyJars(job); + + job.setReducerClass(Reducer.class); + job.setNumReduceTasks(1); // one to get final "first" and "last" key + FileOutputFormat.setOutputPath(job, new Path(job.getJobName())); + LOG.info("Started " + job.getJobName()); + + RunningJob runningJob = JobClient.runJob(job); + runningJob.waitForCompletion(); + assertTrue(runningJob.isSuccessful()); + LOG.info("After map/reduce completion - job " + jobName); + } + + public static class Mapper extends TestMultiTableSnapshotInputFormat.ScanMapper + implements TableMap { + + @Override + public void map(ImmutableBytesWritable key, Result value, + OutputCollector outputCollector, + Reporter reporter) throws IOException { + makeAssertions(key, value); + outputCollector.collect(key, key); + } + + /** + * Closes this stream and releases any system resources associated + * with it. If the stream is already closed then invoking this + * method has no effect. + * + * @throws IOException if an I/O error occurs + */ + @Override + public void close() throws IOException { + } + + @Override + public void configure(JobConf jobConf) { + + } + } + + public static class Reducer extends TestMultiTableSnapshotInputFormat.ScanReducer implements + org.apache.hadoop.mapred.Reducer { + + private JobConf jobConf; + + @Override + public void reduce(ImmutableBytesWritable key, Iterator values, + OutputCollector outputCollector, Reporter reporter) + throws IOException { + makeAssertions(key, Lists.newArrayList(values)); + } + + /** + * Closes this stream and releases any system resources associated + * with it. If the stream is already closed then invoking this + * method has no effect. + * + * @throws IOException if an I/O error occurs + */ + @Override + public void close() throws IOException { + super.cleanup(this.jobConf); + } + + @Override + public void configure(JobConf jobConf) { + this.jobConf = jobConf; + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java new file mode 100644 index 0000000..130fa23 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.mapreduce; + +import com.google.common.collect.Lists; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Base set of tests and setup for input formats touching multiple tables. + */ +public abstract class MultiTableInputFormatTestBase { + static final Log LOG = LogFactory.getLog(MultiTableInputFormatTestBase.class); + public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + static final String TABLE_NAME = "scantest"; + static final byte[] INPUT_FAMILY = Bytes.toBytes("contents"); + static final String KEY_STARTROW = "startRow"; + static final String KEY_LASTROW = "stpRow"; + + static List TABLES = Lists.newArrayList(); + + static { + for (int i = 0; i < 3; i++) { + TABLES.add(TABLE_NAME + String.valueOf(i)); + } + } + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // switch TIF to log at DEBUG level + TEST_UTIL.enableDebug(MultiTableInputFormatBase.class); + // start mini hbase cluster + TEST_UTIL.startMiniCluster(3); + // create and fill table + for (String tableName : TABLES) { + HTable table = null; + try { + table = TEST_UTIL.createMultiRegionTable(TableName.valueOf(tableName), INPUT_FAMILY, 4); + TEST_UTIL.loadTable(table, INPUT_FAMILY, false); + } finally { + if (table != null) { + table.close(); + } + } + } + // start MR cluster + TEST_UTIL.startMiniMapReduceCluster(); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniMapReduceCluster(); + TEST_UTIL.shutdownMiniCluster(); + } + + @After + public void tearDown() throws Exception { + Configuration c = TEST_UTIL.getConfiguration(); + FileUtil.fullyDelete(new File(c.get("hadoop.tmp.dir"))); + } + + /** + * Pass the key and value to reducer. + */ + public static class ScanMapper extends + TableMapper { + /** + * Pass the key and value to reduce. + * + * @param key The key, here "aaa", "aab" etc. + * @param value The value is the same as the key. + * @param context The task context. + * @throws IOException When reading the rows fails. + */ + @Override + public void map(ImmutableBytesWritable key, Result value, Context context) + throws IOException, InterruptedException { + makeAssertions(key, value); + context.write(key, key); + } + + public void makeAssertions(ImmutableBytesWritable key, Result value) throws IOException { + if (value.size() != 1) { + throw new IOException("There should only be one input column"); + } + Map>> cf = + value.getMap(); + if (!cf.containsKey(INPUT_FAMILY)) { + throw new IOException("Wrong input columns. Missing: '" + + Bytes.toString(INPUT_FAMILY) + "'."); + } + String val = Bytes.toStringBinary(value.getValue(INPUT_FAMILY, null)); + LOG.debug("map: key -> " + Bytes.toStringBinary(key.get()) + + ", value -> " + val); + } + } + + /** + * Checks the last and first keys seen against the scanner boundaries. + */ + public static class ScanReducer + extends + Reducer { + private String first = null; + private String last = null; + + @Override + protected void reduce(ImmutableBytesWritable key, + Iterable values, Context context) + throws IOException, InterruptedException { + makeAssertions(key, values); + } + + protected void makeAssertions(ImmutableBytesWritable key, + Iterable values) { + int count = 0; + for (ImmutableBytesWritable value : values) { + String val = Bytes.toStringBinary(value.get()); + LOG.debug("reduce: key[" + count + "] -> " + + Bytes.toStringBinary(key.get()) + ", value -> " + val); + if (first == null) first = val; + last = val; + count++; + } + assertEquals(3, count); + } + + @Override + protected void cleanup(Context context) throws IOException, + InterruptedException { + Configuration c = context.getConfiguration(); + cleanup(c); + } + + protected void cleanup(Configuration c) { + String startRow = c.get(KEY_STARTROW); + String lastRow = c.get(KEY_LASTROW); + LOG.info("cleanup: first -> \"" + first + "\", start row -> \"" + + startRow + "\""); + LOG.info("cleanup: last -> \"" + last + "\", last row -> \"" + lastRow + + "\""); + if (startRow != null && startRow.length() > 0) { + assertEquals(startRow, first); + } + if (lastRow != null && lastRow.length() > 0) { + assertEquals(lastRow, last); + } + } + } + + @Test + public void testScanEmptyToEmpty() throws IOException, InterruptedException, + ClassNotFoundException { + testScan(null, null, null); + } + + @Test + public void testScanEmptyToAPP() throws IOException, InterruptedException, + ClassNotFoundException { + testScan(null, "app", "apo"); + } + + @Test + public void testScanOBBToOPP() throws IOException, InterruptedException, + ClassNotFoundException { + testScan("obb", "opp", "opo"); + } + + @Test + public void testScanYZYToEmpty() throws IOException, InterruptedException, + ClassNotFoundException { + testScan("yzy", null, "zzz"); + } + + /** + * Tests a MR scan using specific start and stop rows. + * + * @throws IOException + * @throws ClassNotFoundException + * @throws InterruptedException + */ + private void testScan(String start, String stop, String last) + throws IOException, InterruptedException, ClassNotFoundException { + String jobName = + "Scan" + (start != null ? start.toUpperCase() : "Empty") + "To" + + (stop != null ? stop.toUpperCase() : "Empty"); + LOG.info("Before map/reduce startup - job " + jobName); + Configuration c = new Configuration(TEST_UTIL.getConfiguration()); + + c.set(KEY_STARTROW, start != null ? start : ""); + c.set(KEY_LASTROW, last != null ? last : ""); + + List scans = new ArrayList(); + + for (String tableName : TABLES) { + Scan scan = new Scan(); + + scan.addFamily(INPUT_FAMILY); + scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(tableName)); + + if (start != null) { + scan.setStartRow(Bytes.toBytes(start)); + } + if (stop != null) { + scan.setStopRow(Bytes.toBytes(stop)); + } + + scans.add(scan); + + LOG.info("scan before: " + scan); + } + + runJob(jobName, c, scans); + } + + protected void runJob(String jobName, Configuration c, List scans) + throws IOException, InterruptedException, ClassNotFoundException { + Job job = new Job(c, jobName); + + initJob(scans, job); + job.setReducerClass(ScanReducer.class); + job.setNumReduceTasks(1); // one to get final "first" and "last" key + FileOutputFormat.setOutputPath(job, new Path(job.getJobName())); + LOG.info("Started " + job.getJobName()); + job.waitForCompletion(true); + assertTrue(job.isSuccessful()); + LOG.info("After map/reduce completion - job " + jobName); + } + + protected abstract void initJob(List scans, Job job) throws IOException; + + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormat.java new file mode 100644 index 0000000..f3e6d8d --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormat.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.mapreduce; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Multimaps; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.mapreduce.Job; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +@Category({ LargeTests.class }) +public class TestMultiTableSnapshotInputFormat extends MultiTableInputFormatTestBase { + + protected Path restoreDir; + + @BeforeClass + public static void setUpSnapshots() throws Exception { + + TEST_UTIL.enableDebug(MultiTableSnapshotInputFormat.class); + TEST_UTIL.enableDebug(MultiTableSnapshotInputFormatImpl.class); + + // take a snapshot of every table we have. + for (String tableName : TABLES) { + SnapshotTestingUtils + .createSnapshotAndValidate(TEST_UTIL.getHBaseAdmin(), TableName.valueOf(tableName), + ImmutableList.of(MultiTableInputFormatTestBase.INPUT_FAMILY), null, + snapshotNameForTable(tableName), FSUtils.getRootDir(TEST_UTIL.getConfiguration()), + TEST_UTIL.getTestFileSystem(), true); + } + } + + @Before + public void setUp() throws Exception { + this.restoreDir = new Path("/tmp"); + + } + + @Override + protected void initJob(List scans, Job job) throws IOException { + TableMapReduceUtil + .initMultiTableSnapshotMapperJob(getSnapshotScanMapping(scans), ScanMapper.class, + ImmutableBytesWritable.class, ImmutableBytesWritable.class, job, true, restoreDir); + } + + protected Map> getSnapshotScanMapping(final List scans) { + return Multimaps.index(scans, new Function() { + @Nullable + @Override + public String apply(Scan input) { + return snapshotNameForTable( + Bytes.toStringBinary(input.getAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME))); + } + }).asMap(); + } + + public static String snapshotNameForTable(String tableName) { + return tableName + "_snapshot"; + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormatImpl.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormatImpl.java new file mode 100644 index 0000000..b4b8056 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableSnapshotInputFormatImpl.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.mapreduce; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.verify; + +@Category({ SmallTests.class }) +public class TestMultiTableSnapshotInputFormatImpl { + + private MultiTableSnapshotInputFormatImpl subject; + private Map> snapshotScans; + private Path restoreDir; + private Configuration conf; + private Path rootDir; + + @Before + public void setUp() throws Exception { + this.subject = Mockito.spy(new MultiTableSnapshotInputFormatImpl()); + + // mock out restoreSnapshot + // TODO: this is kind of meh; it'd be much nicer to just inject the RestoreSnapshotHelper + // dependency into the + // input format. However, we need a new RestoreSnapshotHelper per snapshot in the current + // design, and it *also* + // feels weird to introduce a RestoreSnapshotHelperFactory and inject that, which would + // probably be the more "pure" + // way of doing things. This is the lesser of two evils, perhaps? + doNothing().when(this.subject). + restoreSnapshot(any(Configuration.class), any(String.class), any(Path.class), + any(Path.class), any(FileSystem.class)); + + this.conf = new Configuration(); + this.rootDir = new Path("file:///test-root-dir"); + FSUtils.setRootDir(conf, rootDir); + this.snapshotScans = ImmutableMap.>of("snapshot1", + ImmutableList.of(new Scan(Bytes.toBytes("1"), Bytes.toBytes("2"))), "snapshot2", + ImmutableList.of(new Scan(Bytes.toBytes("3"), Bytes.toBytes("4")), + new Scan(Bytes.toBytes("5"), Bytes.toBytes("6")))); + + this.restoreDir = new Path(FSUtils.getRootDir(conf), "restore-dir"); + + } + + public void callSetInput() throws IOException { + subject.setInput(this.conf, snapshotScans, restoreDir); + } + + public Map> toScanWithEquals( + Map> snapshotScans) throws IOException { + Map> rtn = Maps.newHashMap(); + + for (Map.Entry> entry : snapshotScans.entrySet()) { + List scans = Lists.newArrayList(); + + for (Scan scan : entry.getValue()) { + scans.add(new ScanWithEquals(scan)); + } + rtn.put(entry.getKey(), scans); + } + + return rtn; + } + + public static class ScanWithEquals { + + private final String startRow; + private final String stopRow; + + /** + * Creates a new instance of this class while copying all values. + * + * @param scan The scan instance to copy from. + * @throws java.io.IOException When copying the values fails. + */ + public ScanWithEquals(Scan scan) throws IOException { + this.startRow = Bytes.toStringBinary(scan.getStartRow()); + this.stopRow = Bytes.toStringBinary(scan.getStopRow()); + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof ScanWithEquals)) { + return false; + } + ScanWithEquals otherScan = (ScanWithEquals) obj; + return Objects.equals(this.startRow, otherScan.startRow) && Objects + .equals(this.stopRow, otherScan.stopRow); + } + + @Override + public String toString() { + return com.google.common.base.Objects.toStringHelper(this).add("startRow", startRow) + .add("stopRow", stopRow).toString(); + } + } + + @Test + public void testSetInputSetsSnapshotToScans() throws Exception { + + callSetInput(); + + Map> actual = subject.getSnapshotsToScans(conf); + + // convert to scans we can use .equals on + Map> actualWithEquals = toScanWithEquals(actual); + Map> expectedWithEquals = toScanWithEquals(snapshotScans); + + assertEquals(expectedWithEquals, actualWithEquals); + } + + @Test + public void testSetInputPushesRestoreDirectories() throws Exception { + callSetInput(); + + Map restoreDirs = subject.getSnapshotDirs(conf); + + assertEquals(this.snapshotScans.keySet(), restoreDirs.keySet()); + } + + @Test + public void testSetInputCreatesRestoreDirectoriesUnderRootRestoreDir() throws Exception { + callSetInput(); + + Map restoreDirs = subject.getSnapshotDirs(conf); + + for (Path snapshotDir : restoreDirs.values()) { + assertEquals("Expected " + snapshotDir + " to be a child of " + restoreDir, restoreDir, + snapshotDir.getParent()); + } + } + + @Test + public void testSetInputRestoresSnapshots() throws Exception { + callSetInput(); + + Map snapshotDirs = subject.getSnapshotDirs(conf); + + for (Map.Entry entry : snapshotDirs.entrySet()) { + verify(this.subject).restoreSnapshot(eq(this.conf), eq(entry.getKey()), eq(this.rootDir), + eq(entry.getValue()), any(FileSystem.class)); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java index cf780ba..361941d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java @@ -160,7 +160,7 @@ public class TestRowCounter { long ts; // clean up content of TABLE_NAME - HTable table = TEST_UTIL.truncateTable(TableName.valueOf(TABLE_NAME)); + HTable table = TEST_UTIL.deleteTableData(TableName.valueOf(TABLE_NAME)); ts = System.currentTimeMillis(); put1.add(family, col1, ts, Bytes.toBytes("val1")); table.put(put1); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan2.java index e022d6b..25e0a9d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScan2.java @@ -112,6 +112,6 @@ public class TestTableInputFormatScan2 extends TestTableInputFormatScanBase { @Test public void testScanFromConfiguration() throws IOException, InterruptedException, ClassNotFoundException { - testScanFromConfiguration("bba", "bbd", "bbc"); + testScan("bba", "bbd", "bbc"); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java index 9e8097e..79313b0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java @@ -24,6 +24,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -50,6 +52,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ZooKeeperConnectionException; import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionTestingUtility; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager; @@ -66,7 +69,9 @@ import org.apache.hadoop.hbase.master.TableLockManager.NullTableLockManager; import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; import org.apache.hadoop.hbase.master.balancer.SimpleLoadBalancer; import org.apache.hadoop.hbase.master.handler.EnableTableHandler; -import org.apache.hadoop.hbase.master.handler.ServerShutdownHandler; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure; +import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest; @@ -98,6 +103,7 @@ import org.mockito.Mockito; import org.mockito.internal.util.reflection.Whitebox; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; +import org.mortbay.log.Log; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -105,14 +111,20 @@ import com.google.protobuf.ServiceException; /** * Test {@link AssignmentManager} + * + * TODO: This test suite has rotted. It is too fragile. The smallest change throws it off. It is + * too brittle mocking up partial states in mockito trying to ensure we walk the right codepath + * to obtain expected result. Redo. */ @Category(MediumTests.class) public class TestAssignmentManager { private static final HBaseTestingUtility HTU = new HBaseTestingUtility(); - private static final ServerName SERVERNAME_A = - ServerName.valueOf("example.org", 1234, 5678); - private static final ServerName SERVERNAME_B = - ServerName.valueOf("example.org", 0, 5678); + // Let this be the server that is 'dead' in the tests below. + private static final ServerName SERVERNAME_DEAD = + ServerName.valueOf("dead.example.org", 1, 5678); + // This is the server that is 'live' in the tests below. + private static final ServerName SERVERNAME_LIVE = + ServerName.valueOf("live.example.org", 0, 5678); private static final HRegionInfo REGIONINFO = new HRegionInfo(TableName.valueOf("t"), HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW); @@ -177,12 +189,12 @@ public class TestAssignmentManager { // Mock a ServerManager. Say server SERVERNAME_{A,B} are online. Also // make it so if close or open, we return 'success'. this.serverManager = Mockito.mock(ServerManager.class); - Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true); - Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(true); + Mockito.when(this.serverManager.isServerOnline(SERVERNAME_DEAD)).thenReturn(true); + Mockito.when(this.serverManager.isServerOnline(SERVERNAME_LIVE)).thenReturn(true); Mockito.when(this.serverManager.getDeadServers()).thenReturn(new DeadServer()); final Map onlineServers = new HashMap(); - onlineServers.put(SERVERNAME_B, ServerLoad.EMPTY_SERVERLOAD); - onlineServers.put(SERVERNAME_A, ServerLoad.EMPTY_SERVERLOAD); + onlineServers.put(SERVERNAME_LIVE, ServerLoad.EMPTY_SERVERLOAD); + onlineServers.put(SERVERNAME_DEAD, ServerLoad.EMPTY_SERVERLOAD); Mockito.when(this.serverManager.getOnlineServersList()).thenReturn( new ArrayList(onlineServers.keySet())); Mockito.when(this.serverManager.getOnlineServers()).thenReturn(onlineServers); @@ -192,14 +204,14 @@ public class TestAssignmentManager { Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(avServers); Mockito.when(this.serverManager.createDestinationServersList(null)).thenReturn(avServers); - Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, REGIONINFO, -1)). + Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_DEAD, REGIONINFO, -1)). thenReturn(true); - Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_B, REGIONINFO, -1)). + Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_LIVE, REGIONINFO, -1)). thenReturn(true); // Ditto on open. - Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_A, REGIONINFO, -1, null)). + Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_DEAD, REGIONINFO, -1, null)). thenReturn(RegionOpeningState.OPENED); - Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_B, REGIONINFO, -1, null)). + Mockito.when(this.serverManager.sendRegionOpen(SERVERNAME_LIVE, REGIONINFO, -1, null)). thenReturn(RegionOpeningState.OPENED); this.master = Mockito.mock(HMaster.class); @@ -231,13 +243,13 @@ public class TestAssignmentManager { AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, this.serverManager); try { - createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO); + createRegionPlanAndBalance(am, SERVERNAME_DEAD, SERVERNAME_LIVE, REGIONINFO); startFakeFailedOverMasterAssignmentManager(am, this.watcher); while (!am.processRITInvoked) Thread.sleep(1); // As part of the failover cleanup, the balancing region plan is removed. // So a random server will be used to open the region. For testing purpose, // let's assume it is going to open on server b: - am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B)); + am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_LIVE)); Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO); @@ -247,7 +259,7 @@ public class TestAssignmentManager { // region handler duplicated here because its down deep in a private // method hard to expose. int versionid = - ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1); + ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_DEAD, -1); assertNotSame(versionid, -1); Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName()); @@ -257,12 +269,12 @@ public class TestAssignmentManager { assertNotSame(-1, versionid); // This uglyness below is what the openregionhandler on RS side does. versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO, - SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE, + SERVERNAME_LIVE, EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_OPENING, versionid); assertNotSame(-1, versionid); // Move znode from OPENING to OPENED as RS does on successful open. versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, - SERVERNAME_B, versionid); + SERVERNAME_LIVE, versionid); assertNotSame(-1, versionid); am.gate.set(false); // Block here until our znode is cleared or until this test times out. @@ -280,13 +292,13 @@ public class TestAssignmentManager { AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, this.serverManager); try { - createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO); + createRegionPlanAndBalance(am, SERVERNAME_DEAD, SERVERNAME_LIVE, REGIONINFO); startFakeFailedOverMasterAssignmentManager(am, this.watcher); while (!am.processRITInvoked) Thread.sleep(1); // As part of the failover cleanup, the balancing region plan is removed. // So a random server will be used to open the region. For testing purpose, // let's assume it is going to open on server b: - am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B)); + am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_LIVE)); Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO); @@ -296,7 +308,7 @@ public class TestAssignmentManager { // region handler duplicated here because its down deep in a private // method hard to expose. int versionid = - ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1); + ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_DEAD, -1); assertNotSame(versionid, -1); am.gate.set(false); Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName()); @@ -307,12 +319,12 @@ public class TestAssignmentManager { assertNotSame(-1, versionid); // This uglyness below is what the openregionhandler on RS side does. versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO, - SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE, + SERVERNAME_LIVE, EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_OPENING, versionid); assertNotSame(-1, versionid); // Move znode from OPENING to OPENED as RS does on successful open. versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, - SERVERNAME_B, versionid); + SERVERNAME_LIVE, versionid); assertNotSame(-1, versionid); // Block here until our znode is cleared or until this test timesout. @@ -330,13 +342,13 @@ public class TestAssignmentManager { AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, this.serverManager); try { - createRegionPlanAndBalance(am, SERVERNAME_A, SERVERNAME_B, REGIONINFO); + createRegionPlanAndBalance(am, SERVERNAME_DEAD, SERVERNAME_LIVE, REGIONINFO); startFakeFailedOverMasterAssignmentManager(am, this.watcher); while (!am.processRITInvoked) Thread.sleep(1); // As part of the failover cleanup, the balancing region plan is removed. // So a random server will be used to open the region. For testing purpose, // let's assume it is going to open on server b: - am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_B)); + am.addPlan(REGIONINFO.getEncodedName(), new RegionPlan(REGIONINFO, null, SERVERNAME_LIVE)); Mocking.waitForRegionFailedToCloseAndSetToPendingClose(am, REGIONINFO); @@ -346,7 +358,7 @@ public class TestAssignmentManager { // region handler duplicated here because its down deep in a private // method hard to expose. int versionid = - ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1); + ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_DEAD, -1); assertNotSame(versionid, -1); Mocking.waitForRegionPendingOpenInRIT(am, REGIONINFO.getEncodedName()); @@ -357,12 +369,12 @@ public class TestAssignmentManager { assertNotSame(-1, versionid); // This uglyness below is what the openregionhandler on RS side does. versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO, - SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE, + SERVERNAME_LIVE, EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_OPENING, versionid); assertNotSame(-1, versionid); // Move znode from OPENING to OPENED as RS does on successful open. versionid = ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, - SERVERNAME_B, versionid); + SERVERNAME_LIVE, versionid); assertNotSame(-1, versionid); // Block here until our znode is cleared or until this test timesout. ZKAssign.blockUntilNoRIT(watcher); @@ -410,9 +422,9 @@ public class TestAssignmentManager { this.watcher.registerListenerFirst(am); // Call the balance function but fake the region being online first at // SERVERNAME_A. Create a balance plan. - am.regionOnline(REGIONINFO, SERVERNAME_A); + am.regionOnline(REGIONINFO, SERVERNAME_DEAD); // Balance region from A to B. - RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_A, SERVERNAME_B); + RegionPlan plan = new RegionPlan(REGIONINFO, SERVERNAME_DEAD, SERVERNAME_LIVE); am.balance(plan); RegionStates regionStates = am.getRegionStates(); @@ -428,7 +440,7 @@ public class TestAssignmentManager { // region handler duplicated here because its down deep in a private // method hard to expose. int versionid = - ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_A, -1); + ZKAssign.transitionNodeClosed(this.watcher, REGIONINFO, SERVERNAME_DEAD, -1); assertNotSame(versionid, -1); // AM is going to notice above CLOSED and queue up a new assign. The // assign will go to open the region in the new location set by the @@ -442,12 +454,12 @@ public class TestAssignmentManager { assertNotSame(-1, versionid); // This uglyness below is what the openregionhandler on RS side does. versionid = ZKAssign.transitionNode(server.getZooKeeper(), REGIONINFO, - SERVERNAME_B, EventType.M_ZK_REGION_OFFLINE, + SERVERNAME_LIVE, EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_OPENING, versionid); assertNotSame(-1, versionid); // Move znode from OPENING to OPENED as RS does on successful open. versionid = - ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_B, versionid); + ZKAssign.transitionNodeOpened(this.watcher, REGIONINFO, SERVERNAME_LIVE, versionid); assertNotSame(-1, versionid); // Wait on the handler removing the OPENED znode. while(regionStates.isRegionInTransition(REGIONINFO)) Threads.sleep(1); @@ -463,10 +475,12 @@ public class TestAssignmentManager { * Run a simple server shutdown handler. * @throws KeeperException * @throws IOException + * @throws InterruptedException */ @Test (timeout=180000) public void testShutdownHandler() - throws KeeperException, IOException, CoordinatedStateException, ServiceException { + throws KeeperException, IOException, CoordinatedStateException, ServiceException, + InterruptedException { // Create and startup an executor. This is used by AssignmentManager // handling zk callbacks. ExecutorService executor = startupMasterExecutor("testShutdownHandler"); @@ -492,26 +506,56 @@ public class TestAssignmentManager { * @throws KeeperException * @throws IOException * @throws ServiceException + * @throws InterruptedException */ @Test (timeout=180000) - public void testSSHWhenDisableTableInProgress() throws KeeperException, IOException, - CoordinatedStateException, ServiceException { + public void testSSHWhenDisablingTableInProgress() throws KeeperException, IOException, + CoordinatedStateException, ServiceException, InterruptedException { testCaseWithPartiallyDisabledState(Table.State.DISABLING); - testCaseWithPartiallyDisabledState(Table.State.DISABLED); } + /** + * To test closed region handler to remove rit and delete corresponding znode + * if region in pending close or closing while processing shutdown of a region + * server.(HBASE-5927). + * + * @throws KeeperException + * @throws IOException + * @throws ServiceException + * @throws InterruptedException + */ + @Test (timeout=180000) + public void testSSHWhenDisabledTableInProgress() throws KeeperException, IOException, + CoordinatedStateException, ServiceException, InterruptedException { + testCaseWithPartiallyDisabledState(Table.State.DISABLED); + } /** - * To test if the split region is removed from RIT if the region was in SPLITTING state but the RS - * has actually completed the splitting in hbase:meta but went down. See HBASE-6070 and also HBASE-5806 + * To test if the split region is removed from RIT if the region was in SPLITTING state but the + * RS has actually completed the splitting in hbase:meta but went down. See HBASE-6070 and also + * HBASE-5806 * * @throws KeeperException * @throws IOException */ @Test (timeout=180000) - public void testSSHWhenSplitRegionInProgress() throws KeeperException, IOException, Exception { + public void testSSHWhenSplitRegionInProgressTrue() + throws KeeperException, IOException, Exception { // true indicates the region is split but still in RIT testCaseWithSplitRegionPartial(true); + } + + /** + * To test if the split region is removed from RIT if the region was in SPLITTING state but the + * RS has actually completed the splitting in hbase:meta but went down. See HBASE-6070 and also + * HBASE-5806 + * + * @throws KeeperException + * @throws IOException + */ + @Test (timeout=180000) + public void testSSHWhenSplitRegionInProgressFalse() + throws KeeperException, IOException, Exception { // false indicate the region is not split testCaseWithSplitRegionPartial(false); } @@ -529,14 +573,13 @@ public class TestAssignmentManager { AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager( this.server, this.serverManager); // adding region to regions and servers maps. - am.regionOnline(REGIONINFO, SERVERNAME_A); - // adding region in pending close. - am.getRegionStates().updateRegionState( - REGIONINFO, State.SPLITTING, SERVERNAME_A); - am.getTableStateManager().setTableState(REGIONINFO.getTable(), - Table.State.ENABLED); - RegionTransition data = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING, - REGIONINFO.getRegionName(), SERVERNAME_A); + am.regionOnline(REGIONINFO, SERVERNAME_DEAD); + // Adding region in SPLITTING state. + am.getRegionStates().updateRegionState(REGIONINFO, State.SPLITTING, SERVERNAME_DEAD); + am.getTableStateManager().setTableState(REGIONINFO.getTable(), Table.State.ENABLED); + RegionTransition data = + RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_SPLITTING, + REGIONINFO.getRegionName(), SERVERNAME_DEAD); String node = ZKAssign.getNodeName(this.watcher, REGIONINFO.getEncodedName()); // create znode in M_ZK_REGION_CLOSING state. ZKUtil.createAndWatch(this.watcher, node, data.toByteArray()); @@ -566,7 +609,7 @@ public class TestAssignmentManager { } private void testCaseWithPartiallyDisabledState(Table.State state) throws KeeperException, - IOException, CoordinatedStateException, ServiceException { + IOException, CoordinatedStateException, ServiceException, InterruptedException { // Create and startup an executor. This is used by AssignmentManager // handling zk callbacks. ExecutorService executor = startupMasterExecutor("testSSHWhenDisableTableInProgress"); @@ -577,7 +620,7 @@ public class TestAssignmentManager { AssignmentManager am = new AssignmentManager(this.server, this.serverManager, balancer, executor, null, master.getTableLockManager()); // adding region to regions and servers maps. - am.regionOnline(REGIONINFO, SERVERNAME_A); + am.regionOnline(REGIONINFO, SERVERNAME_DEAD); // adding region in pending close. am.getRegionStates().updateRegionState(REGIONINFO, State.PENDING_CLOSE); if (state == Table.State.DISABLING) { @@ -588,7 +631,7 @@ public class TestAssignmentManager { Table.State.DISABLED); } RegionTransition data = RegionTransition.createRegionTransition(EventType.M_ZK_REGION_CLOSING, - REGIONINFO.getRegionName(), SERVERNAME_A); + REGIONINFO.getRegionName(), SERVERNAME_DEAD); // RegionTransitionData data = new // RegionTransitionData(EventType.M_ZK_REGION_CLOSING, // REGIONINFO.getRegionName(), SERVERNAME_A); @@ -618,7 +661,7 @@ public class TestAssignmentManager { } private void processServerShutdownHandler(AssignmentManager am, boolean splitRegion) - throws IOException, ServiceException { + throws IOException, ServiceException, InterruptedException { // Make sure our new AM gets callbacks; once registered, can't unregister. // Thats ok because we make a new zk watcher for each test. this.watcher.registerListenerFirst(am); @@ -627,13 +670,13 @@ public class TestAssignmentManager { // Make an RS Interface implementation. Make it so a scanner can go against it. ClientProtos.ClientService.BlockingInterface implementation = Mockito.mock(ClientProtos.ClientService.BlockingInterface.class); - // Get a meta row result that has region up on SERVERNAME_A + // Get a meta row result that has region up on SERVERNAME_DEAD Result r; if (splitRegion) { - r = MetaMockingUtil.getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_A); + r = MetaMockingUtil.getMetaTableRowResultAsSplitRegion(REGIONINFO, SERVERNAME_DEAD); } else { - r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A); + r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_DEAD); } final ScanResponse.Builder builder = ScanResponse.newBuilder(); @@ -641,8 +684,7 @@ public class TestAssignmentManager { builder.addCellsPerResult(r.size()); final List cellScannables = new ArrayList(1); cellScannables.add(r); - Mockito.when(implementation.scan( - (RpcController)Mockito.any(), (ScanRequest)Mockito.any())). + Mockito.when(implementation.scan((RpcController)Mockito.any(), (ScanRequest)Mockito.any())). thenAnswer(new Answer() { @Override public ScanResponse answer(InvocationOnMock invocation) throws Throwable { @@ -658,7 +700,7 @@ public class TestAssignmentManager { // Get a connection w/ mocked up common methods. ClusterConnection connection = HConnectionTestingUtility.getMockedConnectionAndDecorate(HTU.getConfiguration(), - null, implementation, SERVERNAME_B, REGIONINFO); + null, implementation, SERVERNAME_LIVE, REGIONINFO); // These mocks were done up when all connections were managed. World is different now we // moved to unmanaged connections. It messes up the intercepts done in these tests. // Just mark connections as marked and then down in MetaTableAccessor, it will go the path @@ -670,26 +712,58 @@ public class TestAssignmentManager { // down in guts of server shutdown handler. Mockito.when(this.server.getConnection()).thenReturn(connection); - // Now make a server shutdown handler instance and invoke process. - // Have it that SERVERNAME_A died. + // Now make a server crash procedure instance and invoke it to process crashed SERVERNAME_A. + // Fake out system that SERVERNAME_A is down. DeadServer deadServers = new DeadServer(); - deadServers.add(SERVERNAME_A); - // I need a services instance that will return the AM + deadServers.add(SERVERNAME_DEAD); + Mockito.when(this.serverManager.getDeadServers()).thenReturn(deadServers); + final List liveServers = new ArrayList(1); + liveServers.add(SERVERNAME_LIVE); + Mockito.when(this.serverManager.createDestinationServersList()). + thenReturn(liveServers); + Mockito.when(this.serverManager.isServerOnline(SERVERNAME_DEAD)).thenReturn(false); + Mockito.when(this.serverManager.isServerReachable(SERVERNAME_DEAD)).thenReturn(false); + Mockito.when(this.serverManager.isServerOnline(SERVERNAME_LIVE)).thenReturn(true); + Mockito.when(this.serverManager.isServerReachable(SERVERNAME_LIVE)).thenReturn(true); + // Make it so we give right answers when log recovery get/set are called. MasterFileSystem fs = Mockito.mock(MasterFileSystem.class); Mockito.doNothing().when(fs).setLogRecoveryMode(); Mockito.when(fs.getLogRecoveryMode()).thenReturn(RecoveryMode.LOG_REPLAY); + // I need a services instance that will return the AM MasterServices services = Mockito.mock(MasterServices.class); Mockito.when(services.getAssignmentManager()).thenReturn(am); Mockito.when(services.getServerManager()).thenReturn(this.serverManager); Mockito.when(services.getZooKeeper()).thenReturn(this.watcher); Mockito.when(services.getMasterFileSystem()).thenReturn(fs); Mockito.when(services.getConnection()).thenReturn(connection); + MetaTableLocator mtl = Mockito.mock(MetaTableLocator.class); + Mockito.when(mtl.verifyMetaRegionLocation(Mockito.isA(HConnection.class), + Mockito.isA(ZooKeeperWatcher.class), Mockito.anyLong())). + thenReturn(true); + Mockito.when(mtl.isLocationAvailable(this.watcher)).thenReturn(true); + Mockito.when(services.getMetaTableLocator()).thenReturn(mtl); Configuration conf = server.getConfiguration(); Mockito.when(services.getConfiguration()).thenReturn(conf); - ServerShutdownHandler handler = new ServerShutdownHandler(this.server, - services, deadServers, SERVERNAME_A, false); + MasterProcedureEnv env = new MasterProcedureEnv(services); + ServerCrashProcedure procedure = new ServerCrashProcedure(SERVERNAME_DEAD, true, false); am.failoverCleanupDone.set(true); - handler.process(); + clearRITInBackground(am, REGIONINFO, SERVERNAME_LIVE); + Method protectedExecuteMethod = null; + try { + protectedExecuteMethod = + procedure.getClass().getSuperclass().getDeclaredMethod("execute", Object.class); + protectedExecuteMethod.setAccessible(true); + Procedure [] procedures = new Procedure [] {procedure}; + do { + // We know that ServerCrashProcedure does not return more than a single Procedure as + // result; it does not make children so the procedures[0] is safe. + procedures = (Procedure [])protectedExecuteMethod.invoke(procedures[0], env); + } while(procedures != null); + } catch (NoSuchMethodException | SecurityException | IllegalAccessException | + IllegalArgumentException | InvocationTargetException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } // The region in r will have been assigned. It'll be up in zk as unassigned. } finally { if (connection != null) connection.close(); @@ -697,6 +771,30 @@ public class TestAssignmentManager { } /** + * Start a background thread that will notice when a particular RIT arrives and that will then + * 'clear it' as though it had been successfully processed. + */ + private void clearRITInBackground(final AssignmentManager am, final HRegionInfo hri, + final ServerName sn) { + Thread t = new Thread() { + @Override + public void run() { + while (true) { + RegionState rs = am.getRegionStates().getRegionTransitionState(hri); + if (rs != null && rs.getServerName() != null) { + if (rs.getServerName().equals(sn)) { + am.regionOnline(REGIONINFO, sn); + break; + } + } + Threads.sleep(100); + } + } + }; + t.start(); + } + + /** * Create and startup executor pools. Start same set as master does (just * run a few less). * @param name Name to give our executor @@ -720,7 +818,7 @@ public class TestAssignmentManager { // First amend the servermanager mock so that when we do send close of the // first meta region on SERVERNAME_A, it will return true rather than // default null. - Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_A, hri, -1)).thenReturn(true); + Mockito.when(this.serverManager.sendRegionClose(SERVERNAME_DEAD, hri, -1)).thenReturn(true); // Need a mocked catalog tracker. LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(server .getConfiguration()); @@ -729,20 +827,20 @@ public class TestAssignmentManager { this.serverManager, balancer, null, null, master.getTableLockManager()); try { // First make sure my mock up basically works. Unassign a region. - unassign(am, SERVERNAME_A, hri); + unassign(am, SERVERNAME_DEAD, hri); // This delete will fail if the previous unassign did wrong thing. - ZKAssign.deleteClosingNode(this.watcher, hri, SERVERNAME_A); + ZKAssign.deleteClosingNode(this.watcher, hri, SERVERNAME_DEAD); // Now put a SPLITTING region in the way. I don't have to assert it // go put in place. This method puts it in place then asserts it still // owns it by moving state from SPLITTING to SPLITTING. - int version = createNodeSplitting(this.watcher, hri, SERVERNAME_A); + int version = createNodeSplitting(this.watcher, hri, SERVERNAME_DEAD); // Now, retry the unassign with the SPLTTING in place. It should just // complete without fail; a sort of 'silent' recognition that the // region to unassign has been split and no longer exists: TOOD: what if // the split fails and the parent region comes back to life? - unassign(am, SERVERNAME_A, hri); + unassign(am, SERVERNAME_DEAD, hri); // This transition should fail if the znode has been messed with. - ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_A, + ZKAssign.transitionNode(this.watcher, hri, SERVERNAME_DEAD, EventType.RS_ZK_REGION_SPLITTING, EventType.RS_ZK_REGION_SPLITTING, version); assertFalse(am.getRegionStates().isRegionInTransition(hri)); } finally { @@ -806,23 +904,23 @@ public class TestAssignmentManager { if (balancer instanceof MockedLoadBalancer) { ((MockedLoadBalancer) balancer).setGateVariable(gate); } - ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A); + ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_DEAD); int v = ZKAssign.getVersion(this.watcher, REGIONINFO); - ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, + ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_DEAD, EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_FAILED_OPEN, v); String path = ZKAssign.getNodeName(this.watcher, REGIONINFO .getEncodedName()); am.getRegionStates().updateRegionState( - REGIONINFO, State.OPENING, SERVERNAME_A); + REGIONINFO, State.OPENING, SERVERNAME_DEAD); // a dummy plan inserted into the regionPlans. This plan is cleared and // new one is formed am.regionPlans.put(REGIONINFO.getEncodedName(), new RegionPlan( - REGIONINFO, null, SERVERNAME_A)); + REGIONINFO, null, SERVERNAME_DEAD)); RegionPlan regionPlan = am.regionPlans.get(REGIONINFO.getEncodedName()); List serverList = new ArrayList(2); - serverList.add(SERVERNAME_B); + serverList.add(SERVERNAME_LIVE); Mockito.when( - this.serverManager.createDestinationServersList(SERVERNAME_A)) + this.serverManager.createDestinationServersList(SERVERNAME_DEAD)) .thenReturn(serverList); am.nodeDataChanged(path); // here we are waiting until the random assignment in the load balancer is @@ -886,22 +984,24 @@ public class TestAssignmentManager { /** * Test the scenario when the master is in failover and trying to process a * region which is in Opening state on a dead RS. Master will force offline the - * region and put it in transition. AM relies on SSH to reassign it. + * region and put it in transition. AM relies on ServerCrashProcedure to reassign it. */ @Test(timeout = 60000) public void testRegionInOpeningStateOnDeadRSWhileMasterFailover() throws IOException, KeeperException, ServiceException, CoordinatedStateException, InterruptedException { AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager( this.server, this.serverManager); - ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_A); + ZKAssign.createNodeOffline(this.watcher, REGIONINFO, SERVERNAME_DEAD); int version = ZKAssign.getVersion(this.watcher, REGIONINFO); - ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_A, EventType.M_ZK_REGION_OFFLINE, + ZKAssign.transitionNode(this.watcher, REGIONINFO, SERVERNAME_DEAD, + EventType.M_ZK_REGION_OFFLINE, EventType.RS_ZK_REGION_OPENING, version); RegionTransition rt = RegionTransition.createRegionTransition(EventType.RS_ZK_REGION_OPENING, - REGIONINFO.getRegionName(), SERVERNAME_A, HConstants.EMPTY_BYTE_ARRAY); + REGIONINFO.getRegionName(), SERVERNAME_DEAD, HConstants.EMPTY_BYTE_ARRAY); version = ZKAssign.getVersion(this.watcher, REGIONINFO); - Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(false); - am.getRegionStates().logSplit(SERVERNAME_A); // Assume log splitting is done + // This isServerOnlin is weird. It is just so the below processRegionsInTransition will walk + // the wanted code path. + Mockito.when(this.serverManager.isServerOnline(SERVERNAME_DEAD)).thenReturn(false); am.getRegionStates().createRegionState(REGIONINFO); am.gate.set(false); @@ -922,8 +1022,8 @@ public class TestAssignmentManager { while (!am.gate.get()) { Thread.sleep(10); } - assertTrue("The region should be assigned immediately.", null != am.regionPlans.get(REGIONINFO - .getEncodedName())); + assertFalse("The region should be assigned immediately.", + am.getRegionStates().isRegionInTransition(REGIONINFO.getEncodedName())); am.shutdown(); } @@ -943,7 +1043,7 @@ public class TestAssignmentManager { Mockito.when(this.serverManager.getOnlineServers()).thenReturn( new HashMap(0)); List destServers = new ArrayList(1); - destServers.add(SERVERNAME_A); + destServers.add(SERVERNAME_DEAD); Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers); // To avoid cast exception in DisableTableHandler process. HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0); @@ -995,12 +1095,13 @@ public class TestAssignmentManager { * @throws Exception */ @Test (timeout=180000) - public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception { + public void testMasterRestartWhenTableInEnabling() + throws KeeperException, IOException, Exception { enabling = true; List destServers = new ArrayList(1); - destServers.add(SERVERNAME_A); + destServers.add(SERVERNAME_DEAD); Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers); - Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true); + Mockito.when(this.serverManager.isServerOnline(SERVERNAME_DEAD)).thenReturn(true); HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0); CoordinatedStateManager csm = CoordinatedStateManagerFactory.getCoordinatedStateManager( HTU.getConfiguration()); @@ -1047,9 +1148,9 @@ public class TestAssignmentManager { public void testMasterRestartShouldRemoveStaleZnodesOfUnknownTableAsForMeta() throws Exception { List destServers = new ArrayList(1); - destServers.add(SERVERNAME_A); + destServers.add(SERVERNAME_DEAD); Mockito.when(this.serverManager.createDestinationServersList()).thenReturn(destServers); - Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true); + Mockito.when(this.serverManager.isServerOnline(SERVERNAME_DEAD)).thenReturn(true); HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0); CoordinatedStateManager csm = CoordinatedStateManagerFactory.getCoordinatedStateManager( HTU.getConfiguration()); @@ -1078,25 +1179,27 @@ public class TestAssignmentManager { } /** * When a region is in transition, if the region server opening the region goes down, - * the region assignment takes a long time normally (waiting for timeout monitor to trigger assign). - * This test is to make sure SSH reassigns it right away. + * the region assignment takes a long time normally (waiting for timeout monitor to trigger + * assign). This test is to make sure SSH reassigns it right away. + * @throws InterruptedException */ @Test (timeout=180000) public void testSSHTimesOutOpeningRegionTransition() - throws KeeperException, IOException, CoordinatedStateException, ServiceException { + throws KeeperException, IOException, CoordinatedStateException, ServiceException, + InterruptedException { // Create an AM. - AssignmentManagerWithExtrasForTesting am = + final AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, this.serverManager); - // adding region in pending open. + // First set up region as being online on SERVERNAME_B. + am.getRegionStates().regionOnline(REGIONINFO, SERVERNAME_LIVE); + // Now add region in pending open up in RIT RegionState state = new RegionState(REGIONINFO, - State.OPENING, System.currentTimeMillis(), SERVERNAME_A); - am.getRegionStates().regionOnline(REGIONINFO, SERVERNAME_B); + State.OPENING, System.currentTimeMillis(), SERVERNAME_DEAD); am.getRegionStates().regionsInTransition.put(REGIONINFO.getEncodedName(), state); - // adding region plan + // Add a region plan am.regionPlans.put(REGIONINFO.getEncodedName(), - new RegionPlan(REGIONINFO, SERVERNAME_B, SERVERNAME_A)); - am.getTableStateManager().setTableState(REGIONINFO.getTable(), - Table.State.ENABLED); + new RegionPlan(REGIONINFO, SERVERNAME_LIVE, SERVERNAME_DEAD)); + am.getTableStateManager().setTableState(REGIONINFO.getTable(), Table.State.ENABLED); try { am.assignInvoked = false; @@ -1113,8 +1216,8 @@ public class TestAssignmentManager { * Scenario:

      *
    • master starts a close, and creates a znode
    • *
    • it fails just at this moment, before contacting the RS
    • - *
    • while the second master is coming up, the targeted RS dies. But it's before ZK timeout so - * we don't know, and we have an exception.
    • + *
    • while the second master is coming up, the targeted RS dies. But it's before ZK timeout + * so we don't know, and we have an exception.
    • *
    • the master must handle this nicely and reassign. *
    */ @@ -1123,7 +1226,7 @@ public class TestAssignmentManager { AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(this.server, this.serverManager); - ZKAssign.createNodeClosing(this.watcher, REGIONINFO, SERVERNAME_A); + ZKAssign.createNodeClosing(this.watcher, REGIONINFO, SERVERNAME_DEAD); try { am.getRegionStates().createRegionState(REGIONINFO); @@ -1207,7 +1310,7 @@ public class TestAssignmentManager { ClientProtos.ClientService.BlockingInterface ri = Mockito.mock(ClientProtos.ClientService.BlockingInterface.class); // Get a meta row result that has region up on SERVERNAME_A for REGIONINFO - Result r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_A); + Result r = MetaMockingUtil.getMetaTableRowResult(REGIONINFO, SERVERNAME_DEAD); final ScanResponse.Builder builder = ScanResponse.newBuilder(); builder.setMoreResults(true); builder.addCellsPerResult(r.size()); @@ -1240,7 +1343,7 @@ public class TestAssignmentManager { // Get a connection w/ mocked up common methods. ClusterConnection connection = (ClusterConnection)HConnectionTestingUtility. getMockedConnectionAndDecorate(HTU.getConfiguration(), null, - ri, SERVERNAME_B, REGIONINFO); + ri, SERVERNAME_LIVE, REGIONINFO); // These mocks were done up when all connections were managed. World is different now we // moved to unmanaged connections. It messes up the intercepts done in these tests. // Just mark connections as marked and then down in MetaTableAccessor, it will go the path @@ -1288,7 +1391,7 @@ public class TestAssignmentManager { public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan) { if (enabling) { assignmentCount++; - this.regionOnline(region, SERVERNAME_A); + this.regionOnline(region, SERVERNAME_DEAD); } else { super.assign(region, setOfflineInZK, forceNewPlan); this.gate.set(true); @@ -1301,7 +1404,7 @@ public class TestAssignmentManager { if (enabling) { for (HRegionInfo region : regions) { assignmentCount++; - this.regionOnline(region, SERVERNAME_A); + this.regionOnline(region, SERVERNAME_DEAD); } return true; } @@ -1447,9 +1550,9 @@ public class TestAssignmentManager { this.watcher.registerListenerFirst(am); assertFalse("The region should not be in transition", am.getRegionStates().isRegionInTransition(hri)); - ZKAssign.createNodeOffline(this.watcher, hri, SERVERNAME_A); + ZKAssign.createNodeOffline(this.watcher, hri, SERVERNAME_DEAD); // Trigger a transition event - ZKAssign.transitionNodeOpening(this.watcher, hri, SERVERNAME_A); + ZKAssign.transitionNodeOpening(this.watcher, hri, SERVERNAME_DEAD); long startTime = EnvironmentEdgeManager.currentTime(); while (!zkEventProcessed.get()) { assertTrue("Timed out in waiting for ZK event to be processed", @@ -1475,7 +1578,7 @@ public class TestAssignmentManager { HRegionInfo hri = REGIONINFO; regionStates.createRegionState(hri); assertFalse(regionStates.isRegionInTransition(hri)); - RegionPlan plan = new RegionPlan(hri, SERVERNAME_A, SERVERNAME_B); + RegionPlan plan = new RegionPlan(hri, SERVERNAME_DEAD, SERVERNAME_LIVE); // Fake table is deleted regionStates.tableDeleted(hri.getTable()); am.balance(plan); @@ -1491,7 +1594,8 @@ public class TestAssignmentManager { @SuppressWarnings("unchecked") @Test (timeout=180000) public void testOpenCloseRegionRPCIntendedForPreviousServer() throws Exception { - Mockito.when(this.serverManager.sendRegionOpen(Mockito.eq(SERVERNAME_B), Mockito.eq(REGIONINFO), + Mockito.when(this.serverManager.sendRegionOpen(Mockito.eq(SERVERNAME_LIVE), + Mockito.eq(REGIONINFO), Mockito.anyInt(), (List)Mockito.any())) .thenThrow(new DoNotRetryIOException()); this.server.getConfiguration().setInt("hbase.assignment.maximum.attempts", 100); @@ -1505,12 +1609,12 @@ public class TestAssignmentManager { RegionStates regionStates = am.getRegionStates(); try { am.regionPlans.put(REGIONINFO.getEncodedName(), - new RegionPlan(REGIONINFO, null, SERVERNAME_B)); + new RegionPlan(REGIONINFO, null, SERVERNAME_LIVE)); // Should fail once, but succeed on the second attempt for the SERVERNAME_A am.assign(hri, true, false); } finally { - assertEquals(SERVERNAME_A, regionStates.getRegionState(REGIONINFO).getServerName()); + assertEquals(SERVERNAME_DEAD, regionStates.getRegionState(REGIONINFO).getServerName()); am.shutdown(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java index e892ce7..b682233 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java @@ -218,7 +218,7 @@ public class TestAssignmentManagerOnCluster { TEST_UTIL.deleteTable(Bytes.toBytes(table)); } } - + // Simulate a scenario where the AssignCallable and SSH are trying to assign a region @Test (timeout=60000) public void testAssignRegionBySSH() throws Exception { @@ -248,15 +248,15 @@ public class TestAssignmentManagerOnCluster { TEST_UTIL.getHBaseCluster().killRegionServer(controlledServer); TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(controlledServer, -1); AssignmentManager am = master.getAssignmentManager(); - + // Simulate the AssignCallable trying to assign the region. Have the region in OFFLINE state, - // but not in transition and the server is the dead 'controlledServer' + // but not in transition and the server is the dead 'controlledServer' regionStates.createRegionState(hri, State.OFFLINE, controlledServer, null); am.assign(hri, true, true); // Region should remain OFFLINE and go to transition assertEquals(State.OFFLINE, regionStates.getRegionState(hri).getState()); assertTrue (regionStates.isRegionInTransition(hri)); - + master.enableSSH(true); am.waitForAssignment(hri); assertTrue (regionStates.getRegionState(hri).isOpened()); @@ -336,7 +336,7 @@ public class TestAssignmentManagerOnCluster { TEST_UTIL.getMiniHBaseCluster().stopMaster(masterServerName); TEST_UTIL.getMiniHBaseCluster().startMaster(); // Wait till master is active and is initialized - while (TEST_UTIL.getMiniHBaseCluster().getMaster() == null || + while (TEST_UTIL.getMiniHBaseCluster().getMaster() == null || !TEST_UTIL.getMiniHBaseCluster().getMaster().isInitialized()) { Threads.sleep(1); } @@ -724,7 +724,7 @@ public class TestAssignmentManagerOnCluster { } am.getTableStateManager().setTableState(table, ZooKeeperProtos.Table.State.DISABLING); - List toAssignRegions = am.processServerShutdown(destServerName); + List toAssignRegions = am.cleanOutCrashedServerReferences(destServerName); assertTrue("Regions to be assigned should be empty.", toAssignRegions.isEmpty()); assertTrue("Regions to be assigned should be empty.", am.getRegionStates() .getRegionState(hri).isOffline()); @@ -847,7 +847,7 @@ public class TestAssignmentManagerOnCluster { List regions = new ArrayList(); regions.add(hri); am.assign(destServerName, regions); - + // let region open continue MyRegionObserver.postOpenEnabled.set(false); @@ -1324,8 +1324,8 @@ public class TestAssignmentManagerOnCluster { } @Override - public boolean isServerShutdownHandlerEnabled() { - return enabled.get() && super.isServerShutdownHandlerEnabled(); + public boolean isServerCrashProcessingEnabled() { + return enabled.get() && super.isServerCrashProcessingEnabled(); } public void enableSSH(boolean enabled) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java index dea5c3a..053dc99 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java @@ -364,7 +364,7 @@ public class TestCatalogJanitor { } @Override - public boolean isServerShutdownHandlerEnabled() { + public boolean isServerCrashProcessingEnabled() { return true; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java index 38e2613..83dd123 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -1727,5 +1727,4 @@ public class TestDistributedLogSplitting { return hrs; } - -} +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestGetLastFlushedSequenceId.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestGetLastFlushedSequenceId.java index 5e6bff8..abb6520 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestGetLastFlushedSequenceId.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestGetLastFlushedSequenceId.java @@ -91,6 +91,7 @@ public class TestGetLastFlushedSequenceId { testUtil.getHBaseCluster().getMaster() .getLastSequenceId(region.getRegionInfo().getEncodedNameAsBytes()); assertEquals(HConstants.NO_SEQNUM, ids.getLastFlushedSequenceId()); + // This will be the sequenceid just before that of the earliest edit in memstore. long storeSequenceId = ids.getStoreSequenceId(0).getSequenceId(); assertTrue(storeSequenceId > 0); testUtil.getHBaseAdmin().flush(tableName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java index 33fe65a..edd7b2d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java @@ -74,6 +74,7 @@ import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.hadoop.hbase.zookeeper.ZKTableStateManager; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.zookeeper.data.Stat; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -761,8 +762,7 @@ public class TestMasterFailover { } Thread.sleep(100); } - LOG.debug("\n\nRegion of enabled table was OPENED on dead RS\n" + - region + "\n\n"); + LOG.debug("\n\nRegion of enabled table was OPENED on dead RS\n" + region + "\n\n"); // Region of disabled table was opened on dead RS region = disabledRegions.remove(0); @@ -778,8 +778,7 @@ public class TestMasterFailover { } Thread.sleep(100); } - LOG.debug("\n\nRegion of disabled table was OPENED on dead RS\n" + - region + "\n\n"); + LOG.debug("\n\nRegion of disabled table was OPENED on dead RS\n" + region + "\n\n"); /* * ZK = NONE diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestZKLessAMOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestZKLessAMOnCluster.java index 3d13d54..0e49f1c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestZKLessAMOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestZKLessAMOnCluster.java @@ -39,4 +39,4 @@ public class TestZKLessAMOnCluster extends TestAssignmentManagerOnCluster { public static void tearDownAfterClass() throws Exception { TestAssignmentManagerOnCluster.tearDownAfterClass(); } -} +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java index 6be34b6..00f82f4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java @@ -192,15 +192,26 @@ public class MasterProcedureTestingUtility { assertTrue(tsm.isTableState(tableName, ZooKeeperProtos.Table.State.DISABLED)); } + /** + * Run through all procedure flow states TWICE while also restarting procedure executor at each + * step; i.e force a reread of procedure store. + * + *

    It does + *

    1. Execute step N - kill the executor before store update + *
    2. Restart executor/store + *
    3. Execute step N - and then save to store + *
    + * + *

    This is a good test for finding state that needs persisting and steps that are not + * idempotent. Use this version of the test when a procedure executes all flow steps from start to + * finish. + * @see #testRecoveryAndDoubleExecution(ProcedureExecutor, long) + */ public static void testRecoveryAndDoubleExecution( final ProcedureExecutor procExec, final long procId, final int numSteps, final TState[] states) throws Exception { ProcedureTestingUtility.waitProcedure(procExec, procId); assertEquals(false, procExec.isRunning()); - // Restart the executor and execute the step twice - // execute step N - kill before store update - // restart executor/store - // execute step N - save on store for (int i = 0; i < numSteps; ++i) { LOG.info("Restart "+ i +" exec state: " + states[i]); ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); @@ -211,6 +222,35 @@ public class MasterProcedureTestingUtility { ProcedureTestingUtility.assertProcNotFailed(procExec, procId); } + /** + * Run through all procedure flow states TWICE while also restarting procedure executor at each + * step; i.e force a reread of procedure store. + * + *

    It does + *

    1. Execute step N - kill the executor before store update + *
    2. Restart executor/store + *
    3. Execute step N - and then save to store + *
    + * + *

    This is a good test for finding state that needs persisting and steps that are not + * idempotent. Use this version of the test when the order in which flow steps are executed is + * not start to finish; where the procedure may vary the flow steps dependent on circumstance + * found. + * @see #testRecoveryAndDoubleExecution(ProcedureExecutor, long, int, Object[]) + */ + public static void testRecoveryAndDoubleExecution( + final ProcedureExecutor procExec, final long procId) + throws Exception { + ProcedureTestingUtility.waitProcedure(procExec, procId); + assertEquals(false, procExec.isRunning()); + while (!procExec.isFinished(procId)) { + ProcedureTestingUtility.restart(procExec); + ProcedureTestingUtility.waitProcedure(procExec, procId); + } + assertEquals(true, procExec.isRunning()); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + } + public static void testRollbackAndDoubleExecution( final ProcedureExecutor procExec, final long procId, final int lastStep, final TState[] states) throws Exception { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureQueue.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureQueue.java index 0d00ff2..349fa8e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureQueue.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureQueue.java @@ -18,14 +18,18 @@ package org.apache.hadoop.hbase.master.procedure; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.ConcurrentHashMap; import java.util.ArrayList; import java.util.HashSet; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -35,18 +39,11 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.master.TableLockManager; import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.testclassification.SmallTests; - import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - @Category(SmallTests.class) public class TestMasterProcedureQueue { private static final Log LOG = LogFactory.getLog(TestMasterProcedureQueue.class); @@ -118,12 +115,12 @@ public class TestMasterProcedureQueue { // fetch item and take a lock assertEquals(1, queue.poll().longValue()); // take the xlock - assertTrue(queue.tryAcquireTableWrite(tableName, "write")); + assertTrue(queue.tryAcquireTableExclusiveLock(tableName, "write")); // table can't be deleted because we have the lock assertEquals(0, queue.size()); assertFalse(queue.markTableAsDeleted(tableName)); // release the xlock - queue.releaseTableWrite(tableName); + queue.releaseTableExclusiveLock(tableName); // complete the table deletion assertTrue(queue.markTableAsDeleted(tableName)); } @@ -149,7 +146,7 @@ public class TestMasterProcedureQueue { // fetch item and take a lock assertEquals(i, queue.poll().longValue()); // take the rlock - assertTrue(queue.tryAcquireTableRead(tableName, "read " + i)); + assertTrue(queue.tryAcquireTableSharedLock(tableName, "read " + i)); // table can't be deleted because we have locks and/or items in the queue assertFalse(queue.markTableAsDeleted(tableName)); } @@ -158,7 +155,7 @@ public class TestMasterProcedureQueue { // table can't be deleted because we have locks assertFalse(queue.markTableAsDeleted(tableName)); // release the rlock - queue.releaseTableRead(tableName); + queue.releaseTableSharedLock(tableName); } // there are no items and no lock in the queeu @@ -187,47 +184,47 @@ public class TestMasterProcedureQueue { // Fetch the 1st item and take the write lock Long procId = queue.poll(); assertEquals(1, procId.longValue()); - assertEquals(true, queue.tryAcquireTableWrite(tableName, "write " + procId)); + assertEquals(true, queue.tryAcquireTableExclusiveLock(tableName, "write " + procId)); // Fetch the 2nd item and verify that the lock can't be acquired assertEquals(null, queue.poll()); // Release the write lock and acquire the read lock - queue.releaseTableWrite(tableName); + queue.releaseTableExclusiveLock(tableName); // Fetch the 2nd item and take the read lock procId = queue.poll(); assertEquals(2, procId.longValue()); - assertEquals(true, queue.tryAcquireTableRead(tableName, "read " + procId)); + assertEquals(true, queue.tryAcquireTableSharedLock(tableName, "read " + procId)); // Fetch the 3rd item and verify that the lock can't be acquired procId = queue.poll(); assertEquals(3, procId.longValue()); - assertEquals(false, queue.tryAcquireTableWrite(tableName, "write " + procId)); + assertEquals(false, queue.tryAcquireTableExclusiveLock(tableName, "write " + procId)); // release the rdlock of item 2 and take the wrlock for the 3d item - queue.releaseTableRead(tableName); - assertEquals(true, queue.tryAcquireTableWrite(tableName, "write " + procId)); + queue.releaseTableSharedLock(tableName); + assertEquals(true, queue.tryAcquireTableExclusiveLock(tableName, "write " + procId)); // Fetch 4th item and verify that the lock can't be acquired assertEquals(null, queue.poll()); // Release the write lock and acquire the read lock - queue.releaseTableWrite(tableName); + queue.releaseTableExclusiveLock(tableName); // Fetch the 4th item and take the read lock procId = queue.poll(); assertEquals(4, procId.longValue()); - assertEquals(true, queue.tryAcquireTableRead(tableName, "read " + procId)); + assertEquals(true, queue.tryAcquireTableSharedLock(tableName, "read " + procId)); // Fetch the 4th item and take the read lock procId = queue.poll(); assertEquals(5, procId.longValue()); - assertEquals(true, queue.tryAcquireTableRead(tableName, "read " + procId)); + assertEquals(true, queue.tryAcquireTableSharedLock(tableName, "read " + procId)); // Release 4th and 5th read-lock - queue.releaseTableRead(tableName); - queue.releaseTableRead(tableName); + queue.releaseTableSharedLock(tableName); + queue.releaseTableSharedLock(tableName); // remove table queue assertEquals(0, queue.size()); @@ -353,11 +350,11 @@ public class TestMasterProcedureQueue { case CREATE: case DELETE: case EDIT: - avail = queue.tryAcquireTableWrite(proc.getTableName(), + avail = queue.tryAcquireTableExclusiveLock(proc.getTableName(), "op="+ proc.getTableOperationType()); break; case READ: - avail = queue.tryAcquireTableRead(proc.getTableName(), + avail = queue.tryAcquireTableSharedLock(proc.getTableName(), "op="+ proc.getTableOperationType()); break; } @@ -374,10 +371,10 @@ public class TestMasterProcedureQueue { case CREATE: case DELETE: case EDIT: - queue.releaseTableWrite(proc.getTableName()); + queue.releaseTableExclusiveLock(proc.getTableName()); break; case READ: - queue.releaseTableRead(proc.getTableName()); + queue.releaseTableSharedLock(proc.getTableName()); break; } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerCrashProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerCrashProcedure.java new file mode 100644 index 0000000..97512ce --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestServerCrashProcedure.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.master.procedure; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Runs first with DLS and then with DLR. + */ +@Category(LargeTests.class) +@RunWith(Parameterized.class) +public class TestServerCrashProcedure { + // Ugly junit parameterization. I just want to pass false and then true but seems like needs + // to return sequences of two-element arrays. + @Parameters(name = "{index}: setting={0}") + public static Collection data() { + return Arrays.asList(new Object[] [] {{Boolean.FALSE, -1}, {Boolean.TRUE, -1}}); + } + + private final HBaseTestingUtility util = new HBaseTestingUtility(); + + @Before + public void setup() throws Exception { + this.util.startMiniCluster(3); + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate( + this.util.getHBaseCluster().getMaster().getMasterProcedureExecutor(), false); + } + + @After + public void tearDown() throws Exception { + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate( + this.util.getHBaseCluster().getMaster().getMasterProcedureExecutor(), false); + this.util.shutdownMiniCluster(); + } + + public TestServerCrashProcedure(final Boolean b, final int ignore) { + this.util.getConfiguration().setBoolean("hbase.master.distributed.log.replay", b); + this.util.getConfiguration().setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + } + + /** + * Run server crash procedure steps twice to test idempotency and that we are persisting all + * needed state. + * @throws Exception + */ + @Test(timeout = 300000) + public void testRecoveryAndDoubleExecutionOnline() throws Exception { + final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOnline"); + this.util.createTable(tableName, HBaseTestingUtility.COLUMNS, + HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); + try (Table t = this.util.getConnection().getTable(tableName)) { + // Load the table with a bit of data so some logs to split and some edits in each region. + this.util.loadTable(t, HBaseTestingUtility.COLUMNS[0]); + int count = countRows(t); + // Run the procedure executor outside the master so we can mess with it. Need to disable + // Master's running of the server crash processing. + HMaster master = this.util.getHBaseCluster().getMaster(); + final ProcedureExecutor procExec = master.getMasterProcedureExecutor(); + master.setServerCrashProcessingEnabled(false); + // Kill a server. Master will notice but do nothing other than add it to list of dead servers. + HRegionServer hrs = this.util.getHBaseCluster().getRegionServer(0); + boolean carryingMeta = master.getAssignmentManager().isCarryingMeta(hrs.getServerName()); + this.util.getHBaseCluster().killRegionServer(hrs.getServerName()); + hrs.join(); + // Wait until the expiration of the server has arrived at the master. We won't process it + // by queuing a ServerCrashProcedure because we have disabled crash processing... but wait + // here so ServerManager gets notice and adds expired server to appropriate queues. + while (!master.getServerManager().isServerDead(hrs.getServerName())) Threads.sleep(10); + // Now, reenable processing else we can't get a lock on the ServerCrashProcedure. + master.setServerCrashProcessingEnabled(true); + // Do some of the master processing of dead servers so when SCP runs, it has expected 'state'. + master.getServerManager().moveFromOnelineToDeadServers(hrs.getServerName()); + // Enable test flags and then queue the crash procedure. + ProcedureTestingUtility.waitNoProcedureRunning(procExec); + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); + long procId = + procExec.submitProcedure(new ServerCrashProcedure(hrs.getServerName(), true, carryingMeta)); + // Now run through the procedure twice crashing the executor on each step... + MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); + // Assert all data came back. + assertEquals(count, countRows(t)); + } + } + + int countRows(final Table t) throws IOException { + int count = 0; + try (ResultScanner scanner = t.getScanner(new Scan())) { + while(scanner.next() != null) count++; + } + return count; + } +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 32878d6..b61416c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -5796,6 +5796,23 @@ public class TestHRegion { } @Test + public void testFlushedFileWithNoTags() throws Exception { + String method = "testFlushedFileWithNoTags"; + HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(fam1)); + region = initHRegion(Bytes.toBytes(method), method, TEST_UTIL.getConfiguration(), fam1); + Put put = new Put(Bytes.toBytes("a-b-0-0")); + put.addColumn(fam1, qual1, Bytes.toBytes("c1-value")); + region.put(put); + region.flush(true); + Store store = region.getStore(fam1); + Collection storefiles = store.getStorefiles(); + for (StoreFile sf : storefiles) { + assertFalse("Tags should not be present " + ,sf.getReader().getHFileReader().getFileContext().isIncludesTags()); + } + } + @Test @SuppressWarnings("unchecked") public void testOpenRegionWrittenToWALForLogReplay() throws Exception { // similar to the above test but with distributed log replay diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java index a00c132..4d74024 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRowTooBig.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.regionserver; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; @@ -38,6 +39,7 @@ import java.io.IOException; @Category(MediumTests.class) public class TestRowTooBig { private final static HBaseTestingUtility HTU = HBaseTestingUtility.createLocalHTU(); + private static Path rootRegionDir; private static final HTableDescriptor TEST_HTD = new HTableDescriptor(TableName.valueOf(TestRowTooBig.class.getSimpleName())); @@ -46,6 +48,7 @@ public class TestRowTooBig { HTU.startMiniCluster(); HTU.getConfiguration().setLong(HConstants.TABLE_MAX_ROWSIZE_KEY, 10 * 1024 * 1024L); + rootRegionDir = HTU.getDataTestDirOnTestFS("TestRowTooBig"); } @AfterClass @@ -80,19 +83,22 @@ public class TestRowTooBig { final HRegionInfo hri = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_END_ROW, HConstants.EMPTY_END_ROW); - Region region = HTU.createLocalHRegion(hri, htd); - - // Add 5 cells to memstore - for (int i = 0; i < 5 ; i++) { - Put put = new Put(row1); + Region region = HTU.createHRegion(hri, rootRegionDir, HTU.getConfiguration(), htd); + try { + // Add 5 cells to memstore + for (int i = 0; i < 5 ; i++) { + Put put = new Put(row1); + + put.add(fam1, Bytes.toBytes("col_" + i ), new byte[5 * 1024 * 1024]); + region.put(put); + region.flush(true); + } - put.add(fam1, Bytes.toBytes("col_" + i ), new byte[5 * 1024 * 1024]); - region.put(put); - region.flush(true); + Get get = new Get(row1); + region.get(get); + } finally { + HBaseTestingUtility.closeRegion(region); } - - Get get = new Get(row1); - region.get(get); } /** @@ -122,20 +128,23 @@ public class TestRowTooBig { final HRegionInfo hri = new HRegionInfo(htd.getTableName(), HConstants.EMPTY_END_ROW, HConstants.EMPTY_END_ROW); - Region region = HTU.createLocalHRegion(hri, htd); - - // Add to memstore - for (int i = 0; i < 10; i++) { - Put put = new Put(row1); - for (int j = 0; j < 10 * 10000; j++) { - put.add(fam1, Bytes.toBytes("col_" + i + "_" + j), new byte[10]); + Region region = HTU.createHRegion(hri, rootRegionDir, HTU.getConfiguration(), htd); + try { + // Add to memstore + for (int i = 0; i < 10; i++) { + Put put = new Put(row1); + for (int j = 0; j < 10 * 10000; j++) { + put.add(fam1, Bytes.toBytes("col_" + i + "_" + j), new byte[10]); + } + region.put(put); + region.flush(true); } - region.put(put); - region.flush(true); - } - region.compact(true); + region.compact(true); - Get get = new Get(row1); - region.get(get); + Get get = new Get(row1); + region.get(get); + } finally { + HBaseTestingUtility.closeRegion(region); + } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java new file mode 100644 index 0000000..2b41882 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java @@ -0,0 +1,148 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.spy; + +import java.io.IOException; +import java.util.Collection; + +import org.apache.commons.lang.mutable.MutableBoolean; +import org.apache.hadoop.hbase.DroppedSnapshotException; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.monitoring.MonitoredTask; +import org.apache.hadoop.hbase.regionserver.HRegion.PrepareFlushResult; +import org.apache.hadoop.hbase.regionserver.Region.FlushResult; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.hbase.wal.WAL; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Matchers; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +/** + * Testcase for https://issues.apache.org/jira/browse/HBASE-13811 + */ +@Category({ MediumTests.class }) +public class TestSplitWalDataLoss { + + private final HBaseTestingUtility testUtil = new HBaseTestingUtility(); + + private NamespaceDescriptor namespace = NamespaceDescriptor.create(getClass().getSimpleName()) + .build(); + + private TableName tableName = TableName.valueOf(namespace.getName(), "dataloss"); + + private byte[] family = Bytes.toBytes("f"); + + private byte[] qualifier = Bytes.toBytes("q"); + + @Before + public void setUp() throws Exception { + testUtil.getConfiguration().setInt("hbase.regionserver.msginterval", 30000); + testUtil.getConfiguration().setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); + testUtil.startMiniCluster(2); + HBaseAdmin admin = testUtil.getHBaseAdmin(); + admin.createNamespace(namespace); + admin.createTable(new HTableDescriptor(tableName).addFamily(new HColumnDescriptor(family))); + testUtil.waitTableAvailable(tableName); + } + + @After + public void tearDown() throws Exception { + testUtil.shutdownMiniCluster(); + } + + @Test + public void test() throws IOException, InterruptedException { + final HRegionServer rs = testUtil.getRSForFirstRegionInTable(tableName); + final HRegion region = (HRegion) rs.getOnlineRegions(tableName).get(0); + HRegion spiedRegion = spy(region); + final MutableBoolean flushed = new MutableBoolean(false); + final MutableBoolean reported = new MutableBoolean(false); + doAnswer(new Answer() { + + @Override + public FlushResult answer(InvocationOnMock invocation) throws Throwable { + synchronized (flushed) { + flushed.setValue(true); + flushed.notifyAll(); + } + synchronized (reported) { + while (!reported.booleanValue()) { + reported.wait(); + } + } + rs.getWAL(region.getRegionInfo()).abortCacheFlush( + region.getRegionInfo().getEncodedNameAsBytes()); + throw new DroppedSnapshotException("testcase"); + } + }).when(spiedRegion).internalFlushCacheAndCommit(Matchers. any(), + Matchers. any(), Matchers. any(), + Matchers.> any()); + rs.onlineRegions.put(rs.onlineRegions.keySet().iterator().next(), spiedRegion); + Connection conn = testUtil.getConnection(); + + try (Table table = conn.getTable(tableName)) { + table.put(new Put(Bytes.toBytes("row0")).addColumn(family, qualifier, Bytes.toBytes("val0"))); + } + long oldestSeqIdOfStore = region.getOldestSeqIdOfStore(family); + assertTrue(oldestSeqIdOfStore > HConstants.NO_SEQNUM); + rs.cacheFlusher.requestFlush(spiedRegion, false); + synchronized (flushed) { + while (!flushed.booleanValue()) { + flushed.wait(); + } + } + try (Table table = conn.getTable(tableName)) { + table.put(new Put(Bytes.toBytes("row1")).addColumn(family, qualifier, Bytes.toBytes("val1"))); + } + long now = EnvironmentEdgeManager.currentTime(); + rs.tryRegionServerReport(now - 500, now); + synchronized (reported) { + reported.setValue(true); + reported.notifyAll(); + } + while (testUtil.getRSForFirstRegionInTable(tableName) == rs) { + Thread.sleep(100); + } + try (Table table = conn.getTable(tableName)) { + Result result = table.get(new Get(Bytes.toBytes("row0"))); + assertArrayEquals(Bytes.toBytes("val0"), result.getValue(family, qualifier)); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java index 4849efb..125bfa8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java @@ -304,7 +304,7 @@ public class TestStore { HColumnDescriptor hcd = new HColumnDescriptor(family); hcd.setMinVersions(minVersions); hcd.setTimeToLive(ttl); - init(name.getMethodName(), conf, hcd); + init(name.getMethodName() + "-" + minVersions, conf, hcd); long storeTtl = this.store.getScanInfo().getTtl(); long sleepTime = storeTtl / storeFileNum; @@ -344,6 +344,7 @@ public class TestStore { edge.incrementTime(sleepTime); } assertNull(this.store.requestCompaction()); + Collection sfs = this.store.getStorefiles(); // Assert the last expired file is not removed. if (minVersions == 0) { @@ -351,6 +352,10 @@ public class TestStore { } long ts = sfs.iterator().next().getReader().getMaxTimestamp(); assertTrue(ts < (edge.currentTime() - storeTtl)); + + for (StoreFile sf : sfs) { + sf.closeReader(true); + } } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java index cc5191c..b3b520a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestFSHLog.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.regionserver.wal; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -27,9 +26,7 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicLong; @@ -50,7 +47,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; @@ -58,6 +54,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.SampleRegionWALObserver; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdge; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -320,53 +317,6 @@ public class TestFSHLog { } } - /** - * Simulates WAL append ops for a region and tests - * {@link FSHLog#areAllRegionsFlushed(Map, Map, Map)} API. - * It compares the region sequenceIds with oldestFlushing and oldestUnFlushed entries. - * If a region's entries are larger than min of (oldestFlushing, oldestUnFlushed), then the - * region should be flushed before archiving this WAL. - */ - @Test - public void testAllRegionsFlushed() { - LOG.debug("testAllRegionsFlushed"); - Map oldestFlushingSeqNo = new HashMap(); - Map oldestUnFlushedSeqNo = new HashMap(); - Map seqNo = new HashMap(); - // create a table - TableName t1 = TableName.valueOf("t1"); - // create a region - HRegionInfo hri1 = new HRegionInfo(t1, HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW); - // variables to mock region sequenceIds - final AtomicLong sequenceId1 = new AtomicLong(1); - // test empty map - assertTrue(FSHLog.areAllRegionsFlushed(seqNo, oldestFlushingSeqNo, oldestUnFlushedSeqNo)); - // add entries in the region - seqNo.put(hri1.getEncodedNameAsBytes(), sequenceId1.incrementAndGet()); - oldestUnFlushedSeqNo.put(hri1.getEncodedNameAsBytes(), sequenceId1.get()); - // should say region1 is not flushed. - assertFalse(FSHLog.areAllRegionsFlushed(seqNo, oldestFlushingSeqNo, oldestUnFlushedSeqNo)); - // test with entries in oldestFlushing map. - oldestUnFlushedSeqNo.clear(); - oldestFlushingSeqNo.put(hri1.getEncodedNameAsBytes(), sequenceId1.get()); - assertFalse(FSHLog.areAllRegionsFlushed(seqNo, oldestFlushingSeqNo, oldestUnFlushedSeqNo)); - // simulate region flush, i.e., clear oldestFlushing and oldestUnflushed maps - oldestFlushingSeqNo.clear(); - oldestUnFlushedSeqNo.clear(); - assertTrue(FSHLog.areAllRegionsFlushed(seqNo, oldestFlushingSeqNo, oldestUnFlushedSeqNo)); - // insert some large values for region1 - oldestUnFlushedSeqNo.put(hri1.getEncodedNameAsBytes(), 1000l); - seqNo.put(hri1.getEncodedNameAsBytes(), 1500l); - assertFalse(FSHLog.areAllRegionsFlushed(seqNo, oldestFlushingSeqNo, oldestUnFlushedSeqNo)); - - // tests when oldestUnFlushed/oldestFlushing contains larger value. - // It means region is flushed. - oldestFlushingSeqNo.put(hri1.getEncodedNameAsBytes(), 1200l); - oldestUnFlushedSeqNo.clear(); - seqNo.put(hri1.getEncodedNameAsBytes(), 1199l); - assertTrue(FSHLog.areAllRegionsFlushed(seqNo, oldestFlushingSeqNo, oldestUnFlushedSeqNo)); - } - @Test(expected=IOException.class) public void testFailedToCreateWALIfParentRenamed() throws IOException { final String name = "testFailedToCreateWALIfParentRenamed"; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSequenceIdAccounting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSequenceIdAccounting.java new file mode 100644 index 0000000..d79716f --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestSequenceIdAccounting.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.regionserver.wal; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestSequenceIdAccounting { + private static final byte [] ENCODED_REGION_NAME = Bytes.toBytes("r"); + private static final byte [] FAMILY_NAME = Bytes.toBytes("cf"); + private static final Set FAMILIES; + static { + FAMILIES = new HashSet(); + FAMILIES.add(FAMILY_NAME); + } + + @Test + public void testAreAllLower() { + SequenceIdAccounting sida = new SequenceIdAccounting(); + sida.getOrCreateLowestSequenceIds(ENCODED_REGION_NAME); + Map m = new HashMap(); + m.put(ENCODED_REGION_NAME, HConstants.NO_SEQNUM); + assertTrue(sida.areAllLower(m)); + long sequenceid = 1; + sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid, true); + sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); + sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); + assertTrue(sida.areAllLower(m)); + m.put(ENCODED_REGION_NAME, sequenceid); + assertFalse(sida.areAllLower(m)); + long lowest = sida.getLowestSequenceId(ENCODED_REGION_NAME); + assertEquals("Lowest should be first sequence id inserted", 1, lowest); + m.put(ENCODED_REGION_NAME, lowest); + assertFalse(sida.areAllLower(m)); + // Now make sure above works when flushing. + sida.startCacheFlush(ENCODED_REGION_NAME, FAMILIES); + assertFalse(sida.areAllLower(m)); + m.put(ENCODED_REGION_NAME, HConstants.NO_SEQNUM); + assertTrue(sida.areAllLower(m)); + // Let the flush complete and if we ask if the sequenceid is lower, should be yes since no edits + sida.completeCacheFlush(ENCODED_REGION_NAME); + m.put(ENCODED_REGION_NAME, sequenceid); + assertTrue(sida.areAllLower(m)); + // Flush again but add sequenceids while we are flushing. + sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); + sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); + sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); + lowest = sida.getLowestSequenceId(ENCODED_REGION_NAME); + m.put(ENCODED_REGION_NAME, lowest); + assertFalse(sida.areAllLower(m)); + sida.startCacheFlush(ENCODED_REGION_NAME, FAMILIES); + sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); + sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); + sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); + assertEquals(lowest, sida.getLowestSequenceId(ENCODED_REGION_NAME)); + assertFalse(sida.areAllLower(m)); + m.put(ENCODED_REGION_NAME, lowest - 1); + assertTrue(sida.areAllLower(m)); + } + + @Test + public void testFindLower() { + SequenceIdAccounting sida = new SequenceIdAccounting(); + sida.getOrCreateLowestSequenceIds(ENCODED_REGION_NAME); + Map m = new HashMap(); + m.put(ENCODED_REGION_NAME, HConstants.NO_SEQNUM); + long sequenceid = 1; + sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid, true); + sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); + sida.update(ENCODED_REGION_NAME, FAMILIES, sequenceid++, true); + assertTrue(sida.findLower(m) == null); + m.put(ENCODED_REGION_NAME, sida.getLowestSequenceId(ENCODED_REGION_NAME)); + assertTrue(sida.findLower(m).length == 1); + m.put(ENCODED_REGION_NAME, sida.getLowestSequenceId(ENCODED_REGION_NAME) - 1); + assertTrue(sida.findLower(m) == null); + } +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationChangingPeerRegionservers.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationChangingPeerRegionservers.java index d858321..5397c6c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationChangingPeerRegionservers.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationChangingPeerRegionservers.java @@ -58,9 +58,9 @@ public class TestReplicationChangingPeerRegionservers extends TestReplicationBas utility1.getHBaseCluster().getRegionServerThreads()) { utility1.getHBaseAdmin().rollWALWriter(r.getRegionServer().getServerName()); } - utility1.truncateTable(tableName); + utility1.deleteTableData(tableName); // truncating the table will send one Delete per row to the slave cluster - // in an async fashion, which is why we cannot just call truncateTable on + // in an async fashion, which is why we cannot just call deleteTableData on // utility2 since late writes could make it to the slave in some way. // Instead, we truncate the first table and wait for all the Deletes to // make it to the slave. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java index d8d735f..06f78e3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java @@ -79,9 +79,9 @@ public class TestReplicationSmallTests extends TestReplicationBase { utility1.getHBaseCluster().getRegionServerThreads()) { utility1.getHBaseAdmin().rollWALWriter(r.getRegionServer().getServerName()); } - utility1.truncateTable(tableName); + utility1.deleteTableData(tableName); // truncating the table will send one Delete per row to the slave cluster - // in an async fashion, which is why we cannot just call truncateTable on + // in an async fashion, which is why we cannot just call deleteTableData on // utility2 since late writes could make it to the slave in some way. // Instead, we truncate the first table and wait for all the Deletes to // make it to the slave. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java index 7efb4e3..db58ccb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java @@ -118,8 +118,8 @@ public class TestReplicationSink { */ @Before public void setUp() throws Exception { - table1 = TEST_UTIL.truncateTable(TABLE_NAME1); - table2 = TEST_UTIL.truncateTable(TABLE_NAME2); + table1 = TEST_UTIL.deleteTableData(TABLE_NAME1); + table2 = TEST_UTIL.deleteTableData(TABLE_NAME2); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java index 75971d6..3b91554 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; @@ -105,14 +106,10 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.Permission.Action; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil; -import org.apache.hadoop.hbase.util.TestTableName; import org.apache.log4j.Level; import org.apache.log4j.Logger; -import org.junit.After; import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -136,7 +133,7 @@ public class TestAccessController extends SecureTestUtil { Logger.getLogger(TableAuthManager.class).setLevel(Level.TRACE); } - @Rule public TestTableName TEST_TABLE = new TestTableName(); + private static TableName TEST_TABLE = TableName.valueOf("testtable1"); private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static Configuration conf; @@ -175,7 +172,7 @@ public class TestAccessController extends SecureTestUtil { private static MasterCoprocessorEnvironment CP_ENV; private static AccessController ACCESS_CONTROLLER; private static RegionServerCoprocessorEnvironment RSCP_ENV; - private RegionCoprocessorEnvironment RCP_ENV; + private static RegionCoprocessorEnvironment RCP_ENV; @BeforeClass public static void setupBeforeClass() throws Exception { @@ -218,25 +215,24 @@ public class TestAccessController extends SecureTestUtil { USER_ADMIN_CF = User.createUserForTesting(conf, "col_family_admin", new String[0]); systemUserConnection = TEST_UTIL.getConnection(); + setUpTableAndUserPermissions(); } @AfterClass public static void tearDownAfterClass() throws Exception { + cleanUp(); TEST_UTIL.shutdownMiniCluster(); } - @Before - public void setUp() throws Exception { - // Create the test table (owner added to the _acl_ table) - Admin admin = TEST_UTIL.getHBaseAdmin(); - HTableDescriptor htd = new HTableDescriptor(TEST_TABLE.getTableName()); + private static void setUpTableAndUserPermissions() throws Exception { + HTableDescriptor htd = new HTableDescriptor(TEST_TABLE); HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY); hcd.setMaxVersions(100); htd.addFamily(hcd); htd.setOwner(USER_OWNER); createTable(TEST_UTIL, htd, new byte[][] { Bytes.toBytes("s") }); - Region region = TEST_UTIL.getHBaseCluster().getRegions(TEST_TABLE.getTableName()).get(0); + Region region = TEST_UTIL.getHBaseCluster().getRegions(TEST_TABLE).get(0); RegionCoprocessorHost rcpHost = region.getCoprocessorHost(); RCP_ENV = rcpHost.createEnvironment(AccessController.class, ACCESS_CONTROLLER, Coprocessor.PRIORITY_HIGHEST, 1, conf); @@ -250,26 +246,26 @@ public class TestAccessController extends SecureTestUtil { Permission.Action.WRITE); grantOnTable(TEST_UTIL, USER_RW.getShortName(), - TEST_TABLE.getTableName(), TEST_FAMILY, null, + TEST_TABLE, TEST_FAMILY, null, Permission.Action.READ, Permission.Action.WRITE); // USER_CREATE is USER_RW plus CREATE permissions grantOnTable(TEST_UTIL, USER_CREATE.getShortName(), - TEST_TABLE.getTableName(), null, null, + TEST_TABLE, null, null, Permission.Action.CREATE, Permission.Action.READ, Permission.Action.WRITE); grantOnTable(TEST_UTIL, USER_RO.getShortName(), - TEST_TABLE.getTableName(), TEST_FAMILY, null, + TEST_TABLE, TEST_FAMILY, null, Permission.Action.READ); grantOnTable(TEST_UTIL, USER_ADMIN_CF.getShortName(), - TEST_TABLE.getTableName(), TEST_FAMILY, + TEST_TABLE, TEST_FAMILY, null, Permission.Action.ADMIN, Permission.Action.CREATE); - assertEquals(5, AccessControlLists.getTablePermissions(conf, TEST_TABLE.getTableName()).size()); + assertEquals(5, AccessControlLists.getTablePermissions(conf, TEST_TABLE).size()); try { assertEquals(5, AccessControlClient.getUserPermissions(systemUserConnection, TEST_TABLE.toString()).size()); @@ -278,21 +274,20 @@ public class TestAccessController extends SecureTestUtil { } } - @After - public void tearDown() throws Exception { + private static void cleanUp() throws Exception { // Clean the _acl_ table try { - deleteTable(TEST_UTIL, TEST_TABLE.getTableName()); + deleteTable(TEST_UTIL, TEST_TABLE); } catch (TableNotFoundException ex) { // Test deleted the table, no problem - LOG.info("Test deleted table " + TEST_TABLE.getTableName()); + LOG.info("Test deleted table " + TEST_TABLE); } // Verify all table/namespace permissions are erased - assertEquals(0, AccessControlLists.getTablePermissions(conf, TEST_TABLE.getTableName()).size()); + assertEquals(0, AccessControlLists.getTablePermissions(conf, TEST_TABLE).size()); assertEquals( - 0, - AccessControlLists.getNamespacePermissions(conf, - TEST_TABLE.getTableName().getNamespaceAsString()).size()); + 0, + AccessControlLists.getNamespacePermissions(conf, + TEST_TABLE.getNamespaceAsString()).size()); } @Test @@ -319,11 +314,11 @@ public class TestAccessController extends SecureTestUtil { AccessTestAction modifyTable = new AccessTestAction() { @Override public Object run() throws Exception { - HTableDescriptor htd = new HTableDescriptor(TEST_TABLE.getTableName()); + HTableDescriptor htd = new HTableDescriptor(TEST_TABLE); htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); htd.addFamily(new HColumnDescriptor("fam_" + User.getCurrent().getShortName())); ACCESS_CONTROLLER.preModifyTable(ObserverContext.createAndPrepare(CP_ENV, null), - TEST_TABLE.getTableName(), htd); + TEST_TABLE, htd); return null; } }; @@ -338,7 +333,7 @@ public class TestAccessController extends SecureTestUtil { @Override public Object run() throws Exception { ACCESS_CONTROLLER - .preDeleteTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE.getTableName()); + .preDeleteTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE); return null; } }; @@ -354,7 +349,7 @@ public class TestAccessController extends SecureTestUtil { public Object run() throws Exception { ACCESS_CONTROLLER .preTruncateTable(ObserverContext.createAndPrepare(CP_ENV, null), - TEST_TABLE.getTableName()); + TEST_TABLE); return null; } }; @@ -369,8 +364,8 @@ public class TestAccessController extends SecureTestUtil { AccessTestAction action = new AccessTestAction() { @Override public Object run() throws Exception { - ACCESS_CONTROLLER.preAddColumn(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE.getTableName(), - hcd); + ACCESS_CONTROLLER.preAddColumn(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE, + hcd); return null; } }; @@ -387,7 +382,7 @@ public class TestAccessController extends SecureTestUtil { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preModifyColumn(ObserverContext.createAndPrepare(CP_ENV, null), - TEST_TABLE.getTableName(), hcd); + TEST_TABLE, hcd); return null; } }; @@ -402,7 +397,7 @@ public class TestAccessController extends SecureTestUtil { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preDeleteColumn(ObserverContext.createAndPrepare(CP_ENV, null), - TEST_TABLE.getTableName(), TEST_FAMILY); + TEST_TABLE, TEST_FAMILY); return null; } }; @@ -417,7 +412,7 @@ public class TestAccessController extends SecureTestUtil { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preDisableTable(ObserverContext.createAndPrepare(CP_ENV, null), - TEST_TABLE.getTableName()); + TEST_TABLE); return null; } }; @@ -444,7 +439,7 @@ public class TestAccessController extends SecureTestUtil { @Override public Object run() throws Exception { ACCESS_CONTROLLER - .preEnableTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE.getTableName()); + .preEnableTable(ObserverContext.createAndPrepare(CP_ENV, null), TEST_TABLE); return null; } }; @@ -456,7 +451,7 @@ public class TestAccessController extends SecureTestUtil { @Test public void testMove() throws Exception { List regions; - try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE.getTableName())) { + try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); @@ -466,7 +461,7 @@ public class TestAccessController extends SecureTestUtil { @Override public Object run() throws Exception { ACCESS_CONTROLLER.preMove(ObserverContext.createAndPrepare(CP_ENV, null), - hri, server, server); + hri, server, server); return null; } }; @@ -478,7 +473,7 @@ public class TestAccessController extends SecureTestUtil { @Test public void testAssign() throws Exception { List regions; - try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE.getTableName())) { + try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); @@ -498,7 +493,7 @@ public class TestAccessController extends SecureTestUtil { @Test public void testUnassign() throws Exception { List regions; - try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE.getTableName())) { + try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); @@ -518,7 +513,7 @@ public class TestAccessController extends SecureTestUtil { @Test public void testRegionOffline() throws Exception { List regions; - try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE.getTableName())) { + try (RegionLocator locator = systemUserConnection.getRegionLocator(TEST_TABLE)) { regions = locator.getAllRegionLocations(); } HRegionLocation location = regions.get(0); @@ -628,21 +623,26 @@ public class TestAccessController extends SecureTestUtil { @Test public void testMergeRegions() throws Exception { - final List regions = TEST_UTIL.getHBaseCluster().findRegionsForTable(TEST_TABLE.getTableName()); - assertTrue("not enough regions: " + regions.size(), regions.size() >= 2); + final TableName tname = TableName.valueOf("testMergeRegions"); + createTestTable(tname); + try { + final List regions = TEST_UTIL.getHBaseCluster().findRegionsForTable(tname); + assertTrue("not enough regions: " + regions.size(), regions.size() >= 2); - AccessTestAction action = new AccessTestAction() { - @Override - public Object run() throws Exception { - ACCESS_CONTROLLER.preMerge( - ObserverContext.createAndPrepare(RSCP_ENV, null), - regions.get(0),regions.get(1)); - return null; - } - }; + AccessTestAction action = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preMerge(ObserverContext.createAndPrepare(RSCP_ENV, null), + regions.get(0), regions.get(1)); + return null; + } + }; - verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER); - verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE); + verifyAllowed(action, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(action, USER_CREATE, USER_RW, USER_RO, USER_NONE); + } finally { + deleteTable(TEST_UTIL, tname); + } } @Test @@ -693,7 +693,7 @@ public class TestAccessController extends SecureTestUtil { Get g = new Get(TEST_ROW); g.addFamily(TEST_FAMILY); try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(TEST_TABLE.getTableName())) { + Table t = conn.getTable(TEST_TABLE)) { t.get(g); } return null; @@ -709,8 +709,8 @@ public class TestAccessController extends SecureTestUtil { s.addFamily(TEST_FAMILY); try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(TEST_TABLE.getTableName())) { - ResultScanner scanner = t.getScanner(s); + Table table = conn.getTable(TEST_TABLE)) { + ResultScanner scanner = table.getScanner(s); try { for (Result r = scanner.next(); r != null; r = scanner.next()) { // do nothing @@ -736,7 +736,7 @@ public class TestAccessController extends SecureTestUtil { Put p = new Put(TEST_ROW); p.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(1)); try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(TEST_TABLE.getTableName())) { + Table t = conn.getTable(TEST_TABLE)) { t.put(p); } return null; @@ -751,7 +751,7 @@ public class TestAccessController extends SecureTestUtil { Delete d = new Delete(TEST_ROW); d.deleteFamily(TEST_FAMILY); try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(TEST_TABLE.getTableName())) { + Table t = conn.getTable(TEST_TABLE)) { t.delete(d); } return null; @@ -766,7 +766,7 @@ public class TestAccessController extends SecureTestUtil { Increment inc = new Increment(TEST_ROW); inc.addColumn(TEST_FAMILY, TEST_QUALIFIER, 1); try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(TEST_TABLE.getTableName())) { + Table t = conn.getTable(TEST_TABLE);) { t.increment(inc); } return null; @@ -784,7 +784,7 @@ public class TestAccessController extends SecureTestUtil { Delete d = new Delete(TEST_ROW); d.deleteFamily(TEST_FAMILY); try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(TEST_TABLE.getTableName())) { + Table t = conn.getTable(TEST_TABLE);) { t.checkAndDelete(TEST_ROW, TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes("test_value"), d); } @@ -800,7 +800,7 @@ public class TestAccessController extends SecureTestUtil { Put p = new Put(TEST_ROW); p.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(1)); try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(TEST_TABLE.getTableName())) { + Table t = conn.getTable(TEST_TABLE);) { t.checkAndPut(TEST_ROW, TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes("test_value"), p); } @@ -812,37 +812,39 @@ public class TestAccessController extends SecureTestUtil { @Test public void testBulkLoad() throws Exception { - FileSystem fs = TEST_UTIL.getTestFileSystem(); - final Path dir = TEST_UTIL.getDataTestDirOnTestFS("testBulkLoad"); - fs.mkdirs(dir); - //need to make it globally writable - //so users creating HFiles have write permissions - fs.setPermission(dir, FsPermission.valueOf("-rwxrwxrwx")); - - AccessTestAction bulkLoadAction = new AccessTestAction() { - @Override - public Object run() throws Exception { - int numRows = 3; - - //Making the assumption that the test table won't split between the range - byte[][][] hfileRanges = {{{(byte)0}, {(byte)9}}}; + try { + FileSystem fs = TEST_UTIL.getTestFileSystem(); + final Path dir = TEST_UTIL.getDataTestDirOnTestFS("testBulkLoad"); + fs.mkdirs(dir); + // need to make it globally writable + // so users creating HFiles have write permissions + fs.setPermission(dir, FsPermission.valueOf("-rwxrwxrwx")); + + AccessTestAction bulkLoadAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + int numRows = 3; - Path bulkLoadBasePath = new Path(dir, new Path(User.getCurrent().getName())); - new BulkLoadHelper(bulkLoadBasePath) - .bulkLoadHFile(TEST_TABLE.getTableName(), TEST_FAMILY, TEST_QUALIFIER, hfileRanges, numRows); + // Making the assumption that the test table won't split between the range + byte[][][] hfileRanges = { { { (byte) 0 }, { (byte) 9 } } }; - return null; - } - }; + Path bulkLoadBasePath = new Path(dir, new Path(User.getCurrent().getName())); + new BulkLoadHelper(bulkLoadBasePath).bulkLoadHFile(TEST_TABLE, TEST_FAMILY, + TEST_QUALIFIER, hfileRanges, numRows); - // User performing bulk loads must have privilege to read table metadata - // (ADMIN or CREATE) - verifyAllowed(bulkLoadAction, SUPERUSER, USER_ADMIN, USER_OWNER, USER_CREATE); - verifyDenied(bulkLoadAction, USER_RW, USER_NONE, USER_RO); + return null; + } + }; - // Reinit after the bulk upload - TEST_UTIL.getHBaseAdmin().disableTable(TEST_TABLE.getTableName()); - TEST_UTIL.getHBaseAdmin().enableTable(TEST_TABLE.getTableName()); + // User performing bulk loads must have privilege to read table metadata + // (ADMIN or CREATE) + verifyAllowed(bulkLoadAction, SUPERUSER, USER_ADMIN, USER_OWNER, USER_CREATE); + verifyDenied(bulkLoadAction, USER_RW, USER_NONE, USER_RO); + } finally { + // Reinit after the bulk upload + TEST_UTIL.getHBaseAdmin().disableTable(TEST_TABLE); + TEST_UTIL.getHBaseAdmin().enableTable(TEST_TABLE); + } } public class BulkLoadHelper { @@ -933,7 +935,7 @@ public class TestAccessController extends SecureTestUtil { Append append = new Append(row); append.add(TEST_FAMILY, qualifier, Bytes.toBytes(2)); try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(TEST_TABLE.getTableName())) { + Table t = conn.getTable(TEST_TABLE)) { t.put(put); t.append(append); } @@ -952,11 +954,11 @@ public class TestAccessController extends SecureTestUtil { public Object run() throws Exception { try(Connection conn = ConnectionFactory.createConnection(conf); Table acl = conn.getTable(AccessControlLists.ACL_TABLE_NAME)) { - BlockingRpcChannel service = acl.coprocessorService(TEST_TABLE.getTableName().getName()); + BlockingRpcChannel service = acl.coprocessorService(TEST_TABLE.getName()); AccessControlService.BlockingInterface protocol = AccessControlService.newBlockingStub(service); - ProtobufUtil.grant(protocol, USER_RO.getShortName(), TEST_TABLE.getTableName(), - TEST_FAMILY, null, Action.READ); + ProtobufUtil.grant(protocol, USER_RO.getShortName(), TEST_TABLE, TEST_FAMILY, null, + Action.READ); } return null; } @@ -967,11 +969,11 @@ public class TestAccessController extends SecureTestUtil { public Object run() throws Exception { try(Connection conn = ConnectionFactory.createConnection(conf); Table acl = conn.getTable(AccessControlLists.ACL_TABLE_NAME)) { - BlockingRpcChannel service = acl.coprocessorService(TEST_TABLE.getTableName().getName()); + BlockingRpcChannel service = acl.coprocessorService(TEST_TABLE.getName()); AccessControlService.BlockingInterface protocol = AccessControlService.newBlockingStub(service); - ProtobufUtil.revoke(protocol, USER_RO.getShortName(), TEST_TABLE.getTableName(), - TEST_FAMILY, null, Action.READ); + ProtobufUtil.revoke(protocol, USER_RO.getShortName(), TEST_TABLE, TEST_FAMILY, null, + Action.READ); } return null; } @@ -981,11 +983,11 @@ public class TestAccessController extends SecureTestUtil { @Override public Object run() throws Exception { try(Connection conn = ConnectionFactory.createConnection(conf); - Table acl = conn.getTable(AccessControlLists.ACL_TABLE_NAME)) { - BlockingRpcChannel service = acl.coprocessorService(TEST_TABLE.getTableName().getName()); + Table acl = conn.getTable(AccessControlLists.ACL_TABLE_NAME)){ + BlockingRpcChannel service = acl.coprocessorService(TEST_TABLE.getName()); AccessControlService.BlockingInterface protocol = - AccessControlService.newBlockingStub(service); - ProtobufUtil.getUserPermissions(protocol, TEST_TABLE.getTableName()); + AccessControlService.newBlockingStub(service); + ProtobufUtil.getUserPermissions(protocol, TEST_TABLE); } return null; } @@ -1007,16 +1009,21 @@ public class TestAccessController extends SecureTestUtil { verifyAllowed(grantAction, SUPERUSER, USER_ADMIN, USER_OWNER); verifyDenied(grantAction, USER_CREATE, USER_RW, USER_RO, USER_NONE); + try { + verifyAllowed(revokeAction, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(revokeAction, USER_CREATE, USER_RW, USER_RO, USER_NONE); - verifyAllowed(revokeAction, SUPERUSER, USER_ADMIN, USER_OWNER); - verifyDenied(revokeAction, USER_CREATE, USER_RW, USER_RO, USER_NONE); - - verifyAllowed(getTablePermissionsAction, SUPERUSER, USER_ADMIN, USER_OWNER); - verifyDenied(getTablePermissionsAction, USER_CREATE, USER_RW, USER_RO, USER_NONE); + verifyAllowed(getTablePermissionsAction, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(getTablePermissionsAction, USER_CREATE, USER_RW, USER_RO, USER_NONE); - verifyAllowed(getGlobalPermissionsAction, SUPERUSER, USER_ADMIN); - verifyDenied(getGlobalPermissionsAction, USER_CREATE, - USER_OWNER, USER_RW, USER_RO, USER_NONE); + verifyAllowed(getGlobalPermissionsAction, SUPERUSER, USER_ADMIN); + verifyDenied(getGlobalPermissionsAction, USER_CREATE, USER_OWNER, USER_RW, USER_RO, + USER_NONE); + } finally { + // Cleanup, Grant the revoked permission back to the user + grantOnTable(TEST_UTIL, USER_RO.getShortName(), TEST_TABLE, TEST_FAMILY, null, + Permission.Action.READ); + } } @Test @@ -1036,237 +1043,231 @@ public class TestAccessController extends SecureTestUtil { htd.addFamily(new HColumnDescriptor(family1)); htd.addFamily(new HColumnDescriptor(family2)); createTable(TEST_UTIL, htd); - - // create temp users - User tblUser = User - .createUserForTesting(TEST_UTIL.getConfiguration(), "tbluser", new String[0]); - User gblUser = User - .createUserForTesting(TEST_UTIL.getConfiguration(), "gbluser", new String[0]); - - // prepare actions: - AccessTestAction putActionAll = new AccessTestAction() { - @Override - public Object run() throws Exception { - Put p = new Put(Bytes.toBytes("a")); - p.add(family1, qualifier, Bytes.toBytes("v1")); - p.add(family2, qualifier, Bytes.toBytes("v2")); - try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(tableName)) { - t.put(p); + try { + // create temp users + User tblUser = + User.createUserForTesting(TEST_UTIL.getConfiguration(), "tbluser", new String[0]); + User gblUser = + User.createUserForTesting(TEST_UTIL.getConfiguration(), "gbluser", new String[0]); + + // prepare actions: + AccessTestAction putActionAll = new AccessTestAction() { + @Override + public Object run() throws Exception { + Put p = new Put(Bytes.toBytes("a")); + p.add(family1, qualifier, Bytes.toBytes("v1")); + p.add(family2, qualifier, Bytes.toBytes("v2")); + try (Connection conn = ConnectionFactory.createConnection(conf); + Table t = conn.getTable(tableName);) { + t.put(p); + } + return null; } - return null; - } - }; + }; - AccessTestAction putAction1 = new AccessTestAction() { - @Override - public Object run() throws Exception { - Put p = new Put(Bytes.toBytes("a")); - p.add(family1, qualifier, Bytes.toBytes("v1")); - try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(tableName)) { - t.put(p); - } - return null; - } - }; + AccessTestAction putAction1 = new AccessTestAction() { + @Override + public Object run() throws Exception { + Put p = new Put(Bytes.toBytes("a")); + p.add(family1, qualifier, Bytes.toBytes("v1")); - AccessTestAction putAction2 = new AccessTestAction() { - @Override - public Object run() throws Exception { - Put p = new Put(Bytes.toBytes("a")); - p.add(family2, qualifier, Bytes.toBytes("v2")); - try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(tableName)) { - t.put(p); + try (Connection conn = ConnectionFactory.createConnection(conf); + Table t = conn.getTable(tableName)) { + t.put(p); + } + return null; } - return null; - } - }; + }; - AccessTestAction getActionAll = new AccessTestAction() { - @Override - public Object run() throws Exception { - Get g = new Get(TEST_ROW); - g.addFamily(family1); - g.addFamily(family2); - try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(tableName)) { - t.get(g); + AccessTestAction putAction2 = new AccessTestAction() { + @Override + public Object run() throws Exception { + Put p = new Put(Bytes.toBytes("a")); + p.add(family2, qualifier, Bytes.toBytes("v2")); + try (Connection conn = ConnectionFactory.createConnection(conf); + Table t = conn.getTable(tableName);) { + t.put(p); + } + return null; } - return null; - } - }; + }; - AccessTestAction getAction1 = new AccessTestAction() { - @Override - public Object run() throws Exception { - Get g = new Get(TEST_ROW); - g.addFamily(family1); - try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(tableName)) { - t.get(g); + AccessTestAction getActionAll = new AccessTestAction() { + @Override + public Object run() throws Exception { + Get g = new Get(TEST_ROW); + g.addFamily(family1); + g.addFamily(family2); + try (Connection conn = ConnectionFactory.createConnection(conf); + Table t = conn.getTable(tableName);) { + t.get(g); + } + return null; } - return null; - } - }; + }; - AccessTestAction getAction2 = new AccessTestAction() { - @Override - public Object run() throws Exception { - Get g = new Get(TEST_ROW); - g.addFamily(family2); - try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(tableName)) { - t.get(g); + AccessTestAction getAction1 = new AccessTestAction() { + @Override + public Object run() throws Exception { + Get g = new Get(TEST_ROW); + g.addFamily(family1); + try (Connection conn = ConnectionFactory.createConnection(conf); + Table t = conn.getTable(tableName)) { + t.get(g); + } + return null; } - return null; - } - }; + }; - AccessTestAction deleteActionAll = new AccessTestAction() { - @Override - public Object run() throws Exception { - Delete d = new Delete(TEST_ROW); - d.deleteFamily(family1); - d.deleteFamily(family2); - try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(tableName)) { - t.delete(d); + AccessTestAction getAction2 = new AccessTestAction() { + @Override + public Object run() throws Exception { + Get g = new Get(TEST_ROW); + g.addFamily(family2); + try (Connection conn = ConnectionFactory.createConnection(conf); + Table t = conn.getTable(tableName)) { + t.get(g); + } + return null; } - return null; - } - }; + }; - AccessTestAction deleteAction1 = new AccessTestAction() { - @Override - public Object run() throws Exception { - Delete d = new Delete(TEST_ROW); - d.deleteFamily(family1); - try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(tableName)) { - t.delete(d); + AccessTestAction deleteActionAll = new AccessTestAction() { + @Override + public Object run() throws Exception { + Delete d = new Delete(TEST_ROW); + d.deleteFamily(family1); + d.deleteFamily(family2); + try (Connection conn = ConnectionFactory.createConnection(conf); + Table t = conn.getTable(tableName)) { + t.delete(d); + } + return null; } - return null; - } - }; + }; - AccessTestAction deleteAction2 = new AccessTestAction() { - @Override - public Object run() throws Exception { - Delete d = new Delete(TEST_ROW); - d.deleteFamily(family2); - try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(tableName)) { - t.delete(d); + AccessTestAction deleteAction1 = new AccessTestAction() { + @Override + public Object run() throws Exception { + Delete d = new Delete(TEST_ROW); + d.deleteFamily(family1); + try (Connection conn = ConnectionFactory.createConnection(conf); + Table t = conn.getTable(tableName)) { + t.delete(d); + } + return null; } - return null; - } - }; - - // initial check: - verifyDenied(tblUser, getActionAll, getAction1, getAction2); - verifyDenied(tblUser, putActionAll, putAction1, putAction2); - verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); - - verifyDenied(gblUser, getActionAll, getAction1, getAction2); - verifyDenied(gblUser, putActionAll, putAction1, putAction2); - verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); - - // grant table read permission - grantGlobal(TEST_UTIL, gblUser.getShortName(), - Permission.Action.READ); - grantOnTable(TEST_UTIL, tblUser.getShortName(), - tableName, null, null, - Permission.Action.READ); - - // check - verifyAllowed(tblUser, getActionAll, getAction1, getAction2); - verifyDenied(tblUser, putActionAll, putAction1, putAction2); - verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); + }; - verifyAllowed(gblUser, getActionAll, getAction1, getAction2); - verifyDenied(gblUser, putActionAll, putAction1, putAction2); - verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); + AccessTestAction deleteAction2 = new AccessTestAction() { + @Override + public Object run() throws Exception { + Delete d = new Delete(TEST_ROW); + d.deleteFamily(family2); + try (Connection conn = ConnectionFactory.createConnection(conf); + Table t = conn.getTable(tableName)) { + t.delete(d); + } + return null; + } + }; - // grant table write permission while revoking read permissions - grantGlobal(TEST_UTIL, gblUser.getShortName(), - Permission.Action.WRITE); - grantOnTable(TEST_UTIL, tblUser.getShortName(), - tableName, null, null, - Permission.Action.WRITE); + // initial check: + verifyDenied(tblUser, getActionAll, getAction1, getAction2); + verifyDenied(tblUser, putActionAll, putAction1, putAction2); + verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); - verifyDenied(tblUser, getActionAll, getAction1, getAction2); - verifyAllowed(tblUser, putActionAll, putAction1, putAction2); - verifyAllowed(tblUser, deleteActionAll, deleteAction1, deleteAction2); + verifyDenied(gblUser, getActionAll, getAction1, getAction2); + verifyDenied(gblUser, putActionAll, putAction1, putAction2); + verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); - verifyDenied(gblUser, getActionAll, getAction1, getAction2); - verifyAllowed(gblUser, putActionAll, putAction1, putAction2); - verifyAllowed(gblUser, deleteActionAll, deleteAction1, deleteAction2); + // grant table read permission + grantGlobal(TEST_UTIL, gblUser.getShortName(), Permission.Action.READ); + grantOnTable(TEST_UTIL, tblUser.getShortName(), tableName, null, null, Permission.Action.READ); - // revoke table permissions - revokeGlobal(TEST_UTIL, gblUser.getShortName()); - revokeFromTable(TEST_UTIL, tblUser.getShortName(), - tableName, null, null); + // check + verifyAllowed(tblUser, getActionAll, getAction1, getAction2); + verifyDenied(tblUser, putActionAll, putAction1, putAction2); + verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); - verifyDenied(tblUser, getActionAll, getAction1, getAction2); - verifyDenied(tblUser, putActionAll, putAction1, putAction2); - verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); + verifyAllowed(gblUser, getActionAll, getAction1, getAction2); + verifyDenied(gblUser, putActionAll, putAction1, putAction2); + verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); - verifyDenied(gblUser, getActionAll, getAction1, getAction2); - verifyDenied(gblUser, putActionAll, putAction1, putAction2); - verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); + // grant table write permission while revoking read permissions + grantGlobal(TEST_UTIL, gblUser.getShortName(), Permission.Action.WRITE); + grantOnTable(TEST_UTIL, tblUser.getShortName(), tableName, null, null, + Permission.Action.WRITE); - // grant column family read permission - grantGlobal(TEST_UTIL, gblUser.getShortName(), - Permission.Action.READ); - grantOnTable(TEST_UTIL, tblUser.getShortName(), - tableName, family1, null, Permission.Action.READ); + verifyDenied(tblUser, getActionAll, getAction1, getAction2); + verifyAllowed(tblUser, putActionAll, putAction1, putAction2); + verifyAllowed(tblUser, deleteActionAll, deleteAction1, deleteAction2); - // Access should be denied for family2 - verifyAllowed(tblUser, getActionAll, getAction1); - verifyDenied(tblUser, getAction2); - verifyDenied(tblUser, putActionAll, putAction1, putAction2); - verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); + verifyDenied(gblUser, getActionAll, getAction1, getAction2); + verifyAllowed(gblUser, putActionAll, putAction1, putAction2); + verifyAllowed(gblUser, deleteActionAll, deleteAction1, deleteAction2); - verifyAllowed(gblUser, getActionAll, getAction1, getAction2); - verifyDenied(gblUser, putActionAll, putAction1, putAction2); - verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); + // revoke table permissions + revokeGlobal(TEST_UTIL, gblUser.getShortName()); + revokeFromTable(TEST_UTIL, tblUser.getShortName(), tableName, null, null); - // grant column family write permission - grantGlobal(TEST_UTIL, gblUser.getShortName(), - Permission.Action.WRITE); - grantOnTable(TEST_UTIL, tblUser.getShortName(), - tableName, family2, null, Permission.Action.WRITE); + verifyDenied(tblUser, getActionAll, getAction1, getAction2); + verifyDenied(tblUser, putActionAll, putAction1, putAction2); + verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); - // READ from family1, WRITE to family2 are allowed - verifyAllowed(tblUser, getActionAll, getAction1); - verifyAllowed(tblUser, putAction2, deleteAction2); - verifyDenied(tblUser, getAction2); - verifyDenied(tblUser, putActionAll, putAction1); - verifyDenied(tblUser, deleteActionAll, deleteAction1); + verifyDenied(gblUser, getActionAll, getAction1, getAction2); + verifyDenied(gblUser, putActionAll, putAction1, putAction2); + verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); - verifyDenied(gblUser, getActionAll, getAction1, getAction2); - verifyAllowed(gblUser, putActionAll, putAction1, putAction2); - verifyAllowed(gblUser, deleteActionAll, deleteAction1, deleteAction2); + // grant column family read permission + grantGlobal(TEST_UTIL, gblUser.getShortName(), Permission.Action.READ); + grantOnTable(TEST_UTIL, tblUser.getShortName(), tableName, family1, null, + Permission.Action.READ); - // revoke column family permission - revokeGlobal(TEST_UTIL, gblUser.getShortName()); - revokeFromTable(TEST_UTIL, tblUser.getShortName(), tableName, family2, null); + // Access should be denied for family2 + verifyAllowed(tblUser, getActionAll, getAction1); + verifyDenied(tblUser, getAction2); + verifyDenied(tblUser, putActionAll, putAction1, putAction2); + verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); - // Revoke on family2 should not have impact on family1 permissions - verifyAllowed(tblUser, getActionAll, getAction1); - verifyDenied(tblUser, getAction2); - verifyDenied(tblUser, putActionAll, putAction1, putAction2); - verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); + verifyAllowed(gblUser, getActionAll, getAction1, getAction2); + verifyDenied(gblUser, putActionAll, putAction1, putAction2); + verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); - // Should not have access as global permissions are completely revoked - verifyDenied(gblUser, getActionAll, getAction1, getAction2); - verifyDenied(gblUser, putActionAll, putAction1, putAction2); - verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); + // grant column family write permission + grantGlobal(TEST_UTIL, gblUser.getShortName(), Permission.Action.WRITE); + grantOnTable(TEST_UTIL, tblUser.getShortName(), tableName, family2, null, + Permission.Action.WRITE); - // delete table - deleteTable(TEST_UTIL, tableName); + // READ from family1, WRITE to family2 are allowed + verifyAllowed(tblUser, getActionAll, getAction1); + verifyAllowed(tblUser, putAction2, deleteAction2); + verifyDenied(tblUser, getAction2); + verifyDenied(tblUser, putActionAll, putAction1); + verifyDenied(tblUser, deleteActionAll, deleteAction1); + + verifyDenied(gblUser, getActionAll, getAction1, getAction2); + verifyAllowed(gblUser, putActionAll, putAction1, putAction2); + verifyAllowed(gblUser, deleteActionAll, deleteAction1, deleteAction2); + + // revoke column family permission + revokeGlobal(TEST_UTIL, gblUser.getShortName()); + revokeFromTable(TEST_UTIL, tblUser.getShortName(), tableName, family2, null); + + // Revoke on family2 should not have impact on family1 permissions + verifyAllowed(tblUser, getActionAll, getAction1); + verifyDenied(tblUser, getAction2); + verifyDenied(tblUser, putActionAll, putAction1, putAction2); + verifyDenied(tblUser, deleteActionAll, deleteAction1, deleteAction2); + + // Should not have access as global permissions are completely revoked + verifyDenied(gblUser, getActionAll, getAction1, getAction2); + verifyDenied(gblUser, putActionAll, putAction1, putAction2); + verifyDenied(gblUser, deleteActionAll, deleteAction1, deleteAction2); + } finally { + // delete table + deleteTable(TEST_UTIL, tableName); + } } private boolean hasFoundUserPermission(UserPermission userPermission, List perms) { @@ -1291,92 +1292,90 @@ public class TestAccessController extends SecureTestUtil { htd.addFamily(new HColumnDescriptor(family2)); createTable(TEST_UTIL, htd); - // create temp users - User user = User.createUserForTesting(TEST_UTIL.getConfiguration(), "user", new String[0]); + try { + // create temp users + User user = User.createUserForTesting(TEST_UTIL.getConfiguration(), "user", new String[0]); - AccessTestAction getQualifierAction = new AccessTestAction() { - @Override - public Object run() throws Exception { - Get g = new Get(TEST_ROW); - g.addColumn(family1, qualifier); - try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(tableName)) { - t.get(g); + AccessTestAction getQualifierAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + Get g = new Get(TEST_ROW); + g.addColumn(family1, qualifier); + try (Connection conn = ConnectionFactory.createConnection(conf); + Table t = conn.getTable(tableName)) { + t.get(g); + } + return null; } - return null; - } - }; + }; - AccessTestAction putQualifierAction = new AccessTestAction() { - @Override - public Object run() throws Exception { - Put p = new Put(TEST_ROW); - p.add(family1, qualifier, Bytes.toBytes("v1")); - try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(tableName)) { - t.put(p); + AccessTestAction putQualifierAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + Put p = new Put(TEST_ROW); + p.add(family1, qualifier, Bytes.toBytes("v1")); + try (Connection conn = ConnectionFactory.createConnection(conf); + Table t = conn.getTable(tableName)) { + t.put(p); + } + return null; } - return null; - } - }; + }; - AccessTestAction deleteQualifierAction = new AccessTestAction() { - @Override - public Object run() throws Exception { - Delete d = new Delete(TEST_ROW); - d.deleteColumn(family1, qualifier); - // d.deleteFamily(family1); - try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(tableName)) { - t.delete(d); + AccessTestAction deleteQualifierAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + Delete d = new Delete(TEST_ROW); + d.deleteColumn(family1, qualifier); + // d.deleteFamily(family1); + try (Connection conn = ConnectionFactory.createConnection(conf); + Table t = conn.getTable(tableName)) { + t.delete(d); + } + return null; } - return null; - } - }; - - revokeFromTable(TEST_UTIL, user.getShortName(), tableName, family1, null); + }; - verifyDenied(user, getQualifierAction); - verifyDenied(user, putQualifierAction); - verifyDenied(user, deleteQualifierAction); + revokeFromTable(TEST_UTIL, user.getShortName(), tableName, family1, null); - grantOnTable(TEST_UTIL, user.getShortName(), - tableName, family1, qualifier, - Permission.Action.READ); + verifyDenied(user, getQualifierAction); + verifyDenied(user, putQualifierAction); + verifyDenied(user, deleteQualifierAction); - verifyAllowed(user, getQualifierAction); - verifyDenied(user, putQualifierAction); - verifyDenied(user, deleteQualifierAction); + grantOnTable(TEST_UTIL, user.getShortName(), tableName, family1, qualifier, + Permission.Action.READ); - // only grant write permission - // TODO: comment this portion after HBASE-3583 - grantOnTable(TEST_UTIL, user.getShortName(), - tableName, family1, qualifier, - Permission.Action.WRITE); + verifyAllowed(user, getQualifierAction); + verifyDenied(user, putQualifierAction); + verifyDenied(user, deleteQualifierAction); - verifyDenied(user, getQualifierAction); - verifyAllowed(user, putQualifierAction); - verifyAllowed(user, deleteQualifierAction); + // only grant write permission + // TODO: comment this portion after HBASE-3583 + grantOnTable(TEST_UTIL, user.getShortName(), tableName, family1, qualifier, + Permission.Action.WRITE); - // grant both read and write permission - grantOnTable(TEST_UTIL, user.getShortName(), - tableName, family1, qualifier, - Permission.Action.READ, Permission.Action.WRITE); + verifyDenied(user, getQualifierAction); + verifyAllowed(user, putQualifierAction); + verifyAllowed(user, deleteQualifierAction); - verifyAllowed(user, getQualifierAction); - verifyAllowed(user, putQualifierAction); - verifyAllowed(user, deleteQualifierAction); + // grant both read and write permission + grantOnTable(TEST_UTIL, user.getShortName(), tableName, family1, qualifier, + Permission.Action.READ, Permission.Action.WRITE); - // revoke family level permission won't impact column level - revokeFromTable(TEST_UTIL, user.getShortName(), - tableName, family1, qualifier); + verifyAllowed(user, getQualifierAction); + verifyAllowed(user, putQualifierAction); + verifyAllowed(user, deleteQualifierAction); - verifyDenied(user, getQualifierAction); - verifyDenied(user, putQualifierAction); - verifyDenied(user, deleteQualifierAction); + // revoke family level permission won't impact column level + revokeFromTable(TEST_UTIL, user.getShortName(), tableName, family1, qualifier); - // delete table - deleteTable(TEST_UTIL, tableName); + verifyDenied(user, getQualifierAction); + verifyDenied(user, putQualifierAction); + verifyDenied(user, deleteQualifierAction); + } finally { + // delete table + deleteTable(TEST_UTIL, tableName); + } } @Test @@ -1397,117 +1396,117 @@ public class TestAccessController extends SecureTestUtil { htd.addFamily(new HColumnDescriptor(family2)); htd.setOwner(USER_OWNER); createTable(TEST_UTIL, htd); - - List perms; - - Table acl = systemUserConnection.getTable(AccessControlLists.ACL_TABLE_NAME); try { - BlockingRpcChannel service = acl.coprocessorService(tableName.getName()); - AccessControlService.BlockingInterface protocol = - AccessControlService.newBlockingStub(service); - perms = ProtobufUtil.getUserPermissions(protocol, tableName); - } finally { - acl.close(); - } + List perms; + Table acl = systemUserConnection.getTable(AccessControlLists.ACL_TABLE_NAME); + try { + BlockingRpcChannel service = acl.coprocessorService(tableName.getName()); + AccessControlService.BlockingInterface protocol = + AccessControlService.newBlockingStub(service); + perms = ProtobufUtil.getUserPermissions(protocol, tableName); + } finally { + acl.close(); + } - UserPermission ownerperm = new UserPermission( - Bytes.toBytes(USER_OWNER.getName()), tableName, null, Action.values()); - assertTrue("Owner should have all permissions on table", + UserPermission ownerperm = + new UserPermission(Bytes.toBytes(USER_OWNER.getName()), tableName, null, Action.values()); + assertTrue("Owner should have all permissions on table", hasFoundUserPermission(ownerperm, perms)); - User user = User.createUserForTesting(TEST_UTIL.getConfiguration(), "user", new String[0]); - byte[] userName = Bytes.toBytes(user.getShortName()); + User user = User.createUserForTesting(TEST_UTIL.getConfiguration(), "user", new String[0]); + byte[] userName = Bytes.toBytes(user.getShortName()); - UserPermission up = new UserPermission(userName, - tableName, family1, qualifier, Permission.Action.READ); - assertFalse("User should not be granted permission: " + up.toString(), - hasFoundUserPermission(up, perms)); + UserPermission up = + new UserPermission(userName, tableName, family1, qualifier, Permission.Action.READ); + assertFalse("User should not be granted permission: " + up.toString(), + hasFoundUserPermission(up, perms)); - // grant read permission - grantOnTable(TEST_UTIL, user.getShortName(), - tableName, family1, qualifier, Permission.Action.READ); + // grant read permission + grantOnTable(TEST_UTIL, user.getShortName(), tableName, family1, qualifier, + Permission.Action.READ); - acl = systemUserConnection.getTable(AccessControlLists.ACL_TABLE_NAME); - try { - BlockingRpcChannel service = acl.coprocessorService(tableName.getName()); - AccessControlService.BlockingInterface protocol = - AccessControlService.newBlockingStub(service); - perms = ProtobufUtil.getUserPermissions(protocol, tableName); - } finally { - acl.close(); - } + acl = systemUserConnection.getTable(AccessControlLists.ACL_TABLE_NAME); + try { + BlockingRpcChannel service = acl.coprocessorService(tableName.getName()); + AccessControlService.BlockingInterface protocol = + AccessControlService.newBlockingStub(service); + perms = ProtobufUtil.getUserPermissions(protocol, tableName); + } finally { + acl.close(); + } - UserPermission upToVerify = new UserPermission( - userName, tableName, family1, qualifier, Permission.Action.READ); - assertTrue("User should be granted permission: " + upToVerify.toString(), - hasFoundUserPermission(upToVerify, perms)); + UserPermission upToVerify = + new UserPermission(userName, tableName, family1, qualifier, Permission.Action.READ); + assertTrue("User should be granted permission: " + upToVerify.toString(), + hasFoundUserPermission(upToVerify, perms)); - upToVerify = new UserPermission( - userName, tableName, family1, qualifier, Permission.Action.WRITE); - assertFalse("User should not be granted permission: " + upToVerify.toString(), - hasFoundUserPermission(upToVerify, perms)); + upToVerify = + new UserPermission(userName, tableName, family1, qualifier, Permission.Action.WRITE); + assertFalse("User should not be granted permission: " + upToVerify.toString(), + hasFoundUserPermission(upToVerify, perms)); - // grant read+write - grantOnTable(TEST_UTIL, user.getShortName(), - tableName, family1, qualifier, - Permission.Action.WRITE, Permission.Action.READ); + // grant read+write + grantOnTable(TEST_UTIL, user.getShortName(), tableName, family1, qualifier, + Permission.Action.WRITE, Permission.Action.READ); - acl = systemUserConnection.getTable(AccessControlLists.ACL_TABLE_NAME); - try { - BlockingRpcChannel service = acl.coprocessorService(tableName.getName()); - AccessControlService.BlockingInterface protocol = - AccessControlService.newBlockingStub(service); - perms = ProtobufUtil.getUserPermissions(protocol, tableName); - } finally { - acl.close(); - } + acl = systemUserConnection.getTable(AccessControlLists.ACL_TABLE_NAME); + try { + BlockingRpcChannel service = acl.coprocessorService(tableName.getName()); + AccessControlService.BlockingInterface protocol = + AccessControlService.newBlockingStub(service); + perms = ProtobufUtil.getUserPermissions(protocol, tableName); + } finally { + acl.close(); + } - upToVerify = new UserPermission(userName, tableName, family1, - qualifier, Permission.Action.WRITE, Permission.Action.READ); - assertTrue("User should be granted permission: " + upToVerify.toString(), - hasFoundUserPermission(upToVerify, perms)); + upToVerify = + new UserPermission(userName, tableName, family1, qualifier, Permission.Action.WRITE, + Permission.Action.READ); + assertTrue("User should be granted permission: " + upToVerify.toString(), + hasFoundUserPermission(upToVerify, perms)); - // revoke - revokeFromTable(TEST_UTIL, user.getShortName(), tableName, family1, qualifier, - Permission.Action.WRITE, Permission.Action.READ); + // revoke + revokeFromTable(TEST_UTIL, user.getShortName(), tableName, family1, qualifier, + Permission.Action.WRITE, Permission.Action.READ); - acl = systemUserConnection.getTable(AccessControlLists.ACL_TABLE_NAME); - try { - BlockingRpcChannel service = acl.coprocessorService(tableName.getName()); - AccessControlService.BlockingInterface protocol = - AccessControlService.newBlockingStub(service); - perms = ProtobufUtil.getUserPermissions(protocol, tableName); - } finally { - acl.close(); - } + acl = systemUserConnection.getTable(AccessControlLists.ACL_TABLE_NAME); + try { + BlockingRpcChannel service = acl.coprocessorService(tableName.getName()); + AccessControlService.BlockingInterface protocol = + AccessControlService.newBlockingStub(service); + perms = ProtobufUtil.getUserPermissions(protocol, tableName); + } finally { + acl.close(); + } - assertFalse("User should not be granted permission: " + upToVerify.toString(), + assertFalse("User should not be granted permission: " + upToVerify.toString(), hasFoundUserPermission(upToVerify, perms)); - // disable table before modification - admin.disableTable(tableName); + // disable table before modification + admin.disableTable(tableName); - User newOwner = User.createUserForTesting(conf, "new_owner", new String[] {}); - htd.setOwner(newOwner); - admin.modifyTable(tableName, htd); + User newOwner = User.createUserForTesting(conf, "new_owner", new String[] {}); + htd.setOwner(newOwner); + admin.modifyTable(tableName, htd); - acl = systemUserConnection.getTable(AccessControlLists.ACL_TABLE_NAME); - try { - BlockingRpcChannel service = acl.coprocessorService(tableName.getName()); - AccessControlService.BlockingInterface protocol = - AccessControlService.newBlockingStub(service); - perms = ProtobufUtil.getUserPermissions(protocol, tableName); - } finally { - acl.close(); - } + acl = systemUserConnection.getTable(AccessControlLists.ACL_TABLE_NAME); + try { + BlockingRpcChannel service = acl.coprocessorService(tableName.getName()); + AccessControlService.BlockingInterface protocol = + AccessControlService.newBlockingStub(service); + perms = ProtobufUtil.getUserPermissions(protocol, tableName); + } finally { + acl.close(); + } - UserPermission newOwnerperm = new UserPermission( - Bytes.toBytes(newOwner.getName()), tableName, null, Action.values()); - assertTrue("New owner should have all permissions on table", + UserPermission newOwnerperm = + new UserPermission(Bytes.toBytes(newOwner.getName()), tableName, null, Action.values()); + assertTrue("New owner should have all permissions on table", hasFoundUserPermission(newOwnerperm, perms)); - - // delete table - deleteTable(TEST_UTIL, tableName); + } finally { + // delete table + deleteTable(TEST_UTIL, tableName); + } } @Test @@ -1525,7 +1524,7 @@ public class TestAccessController extends SecureTestUtil { UserPermission adminPerm = new UserPermission(Bytes.toBytes(USER_ADMIN.getShortName()), AccessControlLists.ACL_TABLE_NAME, null, null, Bytes.toBytes("ACRW")); assertTrue("Only user admin has permission on table _acl_ per setup", - perms.size() == 1 && hasFoundUserPermission(adminPerm, perms)); + perms.size() == 1 && hasFoundUserPermission(adminPerm, perms)); } /** global operations */ @@ -1571,127 +1570,134 @@ public class TestAccessController extends SecureTestUtil { User userQualifier = User.createUserForTesting(conf, "user_check_perms_q", new String[0]); grantOnTable(TEST_UTIL, userTable.getShortName(), - TEST_TABLE.getTableName(), null, null, + TEST_TABLE, null, null, Permission.Action.READ); grantOnTable(TEST_UTIL, userColumn.getShortName(), - TEST_TABLE.getTableName(), TEST_FAMILY, null, + TEST_TABLE, TEST_FAMILY, null, Permission.Action.READ); grantOnTable(TEST_UTIL, userQualifier.getShortName(), - TEST_TABLE.getTableName(), TEST_FAMILY, TEST_Q1, + TEST_TABLE, TEST_FAMILY, TEST_Q1, Permission.Action.READ); - AccessTestAction tableRead = new AccessTestAction() { - @Override - public Void run() throws Exception { - checkTablePerms(TEST_UTIL, TEST_TABLE.getTableName(), null, null, - Permission.Action.READ); - return null; - } - }; - - AccessTestAction columnRead = new AccessTestAction() { - @Override - public Void run() throws Exception { - checkTablePerms(TEST_UTIL, TEST_TABLE.getTableName(), TEST_FAMILY, null, - Permission.Action.READ); - return null; - } - }; + try { + AccessTestAction tableRead = new AccessTestAction() { + @Override + public Void run() throws Exception { + checkTablePerms(TEST_UTIL, TEST_TABLE, null, null, Permission.Action.READ); + return null; + } + }; - AccessTestAction qualifierRead = new AccessTestAction() { - @Override - public Void run() throws Exception { - checkTablePerms(TEST_UTIL, TEST_TABLE.getTableName(), TEST_FAMILY, TEST_Q1, - Permission.Action.READ); - return null; - } - }; + AccessTestAction columnRead = new AccessTestAction() { + @Override + public Void run() throws Exception { + checkTablePerms(TEST_UTIL, TEST_TABLE, TEST_FAMILY, null, Permission.Action.READ); + return null; + } + }; - AccessTestAction multiQualifierRead = new AccessTestAction() { - @Override - public Void run() throws Exception { - checkTablePerms(TEST_UTIL, TEST_TABLE.getTableName(), new Permission[] { - new TablePermission(TEST_TABLE.getTableName(), TEST_FAMILY, TEST_Q1, - Permission.Action.READ), - new TablePermission(TEST_TABLE.getTableName(), TEST_FAMILY, TEST_Q2, - Permission.Action.READ), }); - return null; - } - }; + AccessTestAction qualifierRead = new AccessTestAction() { + @Override + public Void run() throws Exception { + checkTablePerms(TEST_UTIL, TEST_TABLE, TEST_FAMILY, TEST_Q1, Permission.Action.READ); + return null; + } + }; - AccessTestAction globalAndTableRead = new AccessTestAction() { - @Override - public Void run() throws Exception { - checkTablePerms(TEST_UTIL, TEST_TABLE.getTableName(), - new Permission[] { new Permission(Permission.Action.READ), - new TablePermission(TEST_TABLE.getTableName(), null, (byte[]) null, - Permission.Action.READ), }); - return null; - } - }; + AccessTestAction multiQualifierRead = new AccessTestAction() { + @Override + public Void run() throws Exception { + checkTablePerms(TEST_UTIL, TEST_TABLE, new Permission[] { + new TablePermission(TEST_TABLE, TEST_FAMILY, TEST_Q1, Permission.Action.READ), + new TablePermission(TEST_TABLE, TEST_FAMILY, TEST_Q2, Permission.Action.READ), }); + return null; + } + }; - AccessTestAction noCheck = new AccessTestAction() { - @Override - public Void run() throws Exception { - checkTablePerms(TEST_UTIL, TEST_TABLE.getTableName(), new Permission[0]); - return null; - } - }; + AccessTestAction globalAndTableRead = new AccessTestAction() { + @Override + public Void run() throws Exception { + checkTablePerms(TEST_UTIL, TEST_TABLE, new Permission[] { + new Permission(Permission.Action.READ), + new TablePermission(TEST_TABLE, null, (byte[]) null, Permission.Action.READ), }); + return null; + } + }; - verifyAllowed(tableRead, SUPERUSER, userTable); - verifyDenied(tableRead, userColumn, userQualifier); + AccessTestAction noCheck = new AccessTestAction() { + @Override + public Void run() throws Exception { + checkTablePerms(TEST_UTIL, TEST_TABLE, new Permission[0]); + return null; + } + }; - verifyAllowed(columnRead, SUPERUSER, userTable, userColumn); - verifyDenied(columnRead, userQualifier); + verifyAllowed(tableRead, SUPERUSER, userTable); + verifyDenied(tableRead, userColumn, userQualifier); - verifyAllowed(qualifierRead, SUPERUSER, userTable, userColumn, userQualifier); + verifyAllowed(columnRead, SUPERUSER, userTable, userColumn); + verifyDenied(columnRead, userQualifier); - verifyAllowed(multiQualifierRead, SUPERUSER, userTable, userColumn); - verifyDenied(multiQualifierRead, userQualifier); + verifyAllowed(qualifierRead, SUPERUSER, userTable, userColumn, userQualifier); - verifyAllowed(globalAndTableRead, SUPERUSER); - verifyDenied(globalAndTableRead, userTable, userColumn, userQualifier); + verifyAllowed(multiQualifierRead, SUPERUSER, userTable, userColumn); + verifyDenied(multiQualifierRead, userQualifier); - verifyAllowed(noCheck, SUPERUSER, userTable, userColumn, userQualifier); + verifyAllowed(globalAndTableRead, SUPERUSER); + verifyDenied(globalAndTableRead, userTable, userColumn, userQualifier); - // -------------------------------------- - // test family level multiple permissions - AccessTestAction familyReadWrite = new AccessTestAction() { - @Override - public Void run() throws Exception { - checkTablePerms(TEST_UTIL, TEST_TABLE.getTableName(), TEST_FAMILY, null, - Permission.Action.READ, Permission.Action.WRITE); - return null; - } - }; + verifyAllowed(noCheck, SUPERUSER, userTable, userColumn, userQualifier); - verifyAllowed(familyReadWrite, SUPERUSER, USER_OWNER, USER_CREATE, USER_RW); - verifyDenied(familyReadWrite, USER_NONE, USER_RO); + // -------------------------------------- + // test family level multiple permissions + AccessTestAction familyReadWrite = new AccessTestAction() { + @Override + public Void run() throws Exception { + checkTablePerms(TEST_UTIL, TEST_TABLE, TEST_FAMILY, null, Permission.Action.READ, + Permission.Action.WRITE); + return null; + } + }; - // -------------------------------------- - // check for wrong table region - CheckPermissionsRequest checkRequest = CheckPermissionsRequest.newBuilder() - .addPermission(AccessControlProtos.Permission.newBuilder() - .setType(AccessControlProtos.Permission.Type.Table) - .setTablePermission( - AccessControlProtos.TablePermission.newBuilder() - .setTableName(ProtobufUtil.toProtoTableName(TEST_TABLE.getTableName())) - .addAction(AccessControlProtos.Permission.Action.CREATE)) - ).build(); - Table acl = systemUserConnection.getTable(AccessControlLists.ACL_TABLE_NAME); - try { - BlockingRpcChannel channel = acl.coprocessorService(new byte[0]); - AccessControlService.BlockingInterface protocol = - AccessControlService.newBlockingStub(channel); + verifyAllowed(familyReadWrite, SUPERUSER, USER_OWNER, USER_CREATE, USER_RW); + verifyDenied(familyReadWrite, USER_NONE, USER_RO); + + // -------------------------------------- + // check for wrong table region + CheckPermissionsRequest checkRequest = + CheckPermissionsRequest + .newBuilder() + .addPermission( + AccessControlProtos.Permission + .newBuilder() + .setType(AccessControlProtos.Permission.Type.Table) + .setTablePermission( + AccessControlProtos.TablePermission.newBuilder() + .setTableName(ProtobufUtil.toProtoTableName(TEST_TABLE)) + .addAction(AccessControlProtos.Permission.Action.CREATE))).build(); + Table acl = systemUserConnection.getTable(AccessControlLists.ACL_TABLE_NAME); try { - // but ask for TablePermissions for TEST_TABLE - protocol.checkPermissions(null, checkRequest); - fail("this should have thrown CoprocessorException"); - } catch (ServiceException ex) { - // expected + BlockingRpcChannel channel = acl.coprocessorService(new byte[0]); + AccessControlService.BlockingInterface protocol = + AccessControlService.newBlockingStub(channel); + try { + // but ask for TablePermissions for TEST_TABLE + protocol.checkPermissions(null, checkRequest); + fail("this should have thrown CoprocessorException"); + } catch (ServiceException ex) { + // expected + } + } finally { + acl.close(); } + } finally { - acl.close(); + revokeFromTable(TEST_UTIL, userTable.getShortName(), TEST_TABLE, null, null, + Permission.Action.READ); + revokeFromTable(TEST_UTIL, userColumn.getShortName(), TEST_TABLE, TEST_FAMILY, null, + Permission.Action.READ); + revokeFromTable(TEST_UTIL, userQualifier.getShortName(), TEST_TABLE, TEST_FAMILY, TEST_Q1, + Permission.Action.READ); } } @@ -1754,10 +1760,10 @@ public class TestAccessController extends SecureTestUtil { @Test public void testSnapshot() throws Exception { Admin admin = TEST_UTIL.getHBaseAdmin(); - final HTableDescriptor htd = admin.getTableDescriptor(TEST_TABLE.getTableName()); + final HTableDescriptor htd = admin.getTableDescriptor(TEST_TABLE); SnapshotDescription.Builder builder = SnapshotDescription.newBuilder(); - builder.setName(TEST_TABLE.getTableName().getNameAsString() + "-snapshot"); - builder.setTable(TEST_TABLE.getTableName().getNameAsString()); + builder.setName(TEST_TABLE.getNameAsString() + "-snapshot"); + builder.setTable(TEST_TABLE.getNameAsString()); final SnapshotDescription snapshot = builder.build(); AccessTestAction snapshotAction = new AccessTestAction() { @Override @@ -1811,10 +1817,10 @@ public class TestAccessController extends SecureTestUtil { @Test public void testSnapshotWithOwner() throws Exception { Admin admin = TEST_UTIL.getHBaseAdmin(); - final HTableDescriptor htd = admin.getTableDescriptor(TEST_TABLE.getTableName()); + final HTableDescriptor htd = admin.getTableDescriptor(TEST_TABLE); SnapshotDescription.Builder builder = SnapshotDescription.newBuilder(); - builder.setName(TEST_TABLE.getTableName().getNameAsString() + "-snapshot"); - builder.setTable(TEST_TABLE.getTableName().getNameAsString()); + builder.setName(TEST_TABLE.getNameAsString() + "-snapshot"); + builder.setTable(TEST_TABLE.getNameAsString()); builder.setOwner(USER_OWNER.getName()); final SnapshotDescription snapshot = builder.build(); AccessTestAction snapshotAction = new AccessTestAction() { @@ -1869,15 +1875,6 @@ public class TestAccessController extends SecureTestUtil { LOG.debug("Test for global authorization for a new registered RegionServer."); MiniHBaseCluster hbaseCluster = TEST_UTIL.getHBaseCluster(); - // Since each RegionServer running on different user, add global - // permissions for the new user. - String currentUser = User.getCurrent().getShortName(); - String activeUserForNewRs = currentUser + ".hfs." + - hbaseCluster.getLiveRegionServerThreads().size(); - grantGlobal(TEST_UTIL, activeUserForNewRs, - Permission.Action.ADMIN, Permission.Action.CREATE, Permission.Action.READ, - Permission.Action.WRITE); - final Admin admin = TEST_UTIL.getHBaseAdmin(); HTableDescriptor htd = new HTableDescriptor(TEST_TABLE2); htd.addFamily(new HColumnDescriptor(TEST_FAMILY)); @@ -1941,35 +1938,40 @@ public class TestAccessController extends SecureTestUtil { User TABLE_ADMIN = User.createUserForTesting(conf, "UserA", new String[0]); // Grant TABLE ADMIN privs - grantOnTable(TEST_UTIL, TABLE_ADMIN.getShortName(), - TEST_TABLE.getTableName(), null, null, + grantOnTable(TEST_UTIL, TABLE_ADMIN.getShortName(), TEST_TABLE, null, null, Permission.Action.ADMIN); - - AccessTestAction listTablesAction = new AccessTestAction() { - @Override - public Object run() throws Exception { - try(Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); - Admin admin = conn.getAdmin()) { - return Arrays.asList(admin.listTables()); + try { + AccessTestAction listTablesAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); + Admin admin = conn.getAdmin()) { + return Arrays.asList(admin.listTables()); + } } - } - }; + }; - AccessTestAction getTableDescAction = new AccessTestAction() { - @Override - public Object run() throws Exception { - try(Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); - Admin admin = conn.getAdmin();) { - return admin.getTableDescriptor(TEST_TABLE.getTableName()); + AccessTestAction getTableDescAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); + Admin admin = conn.getAdmin();) { + return admin.getTableDescriptor(TEST_TABLE); + } } - } - }; + }; - verifyAllowed(listTablesAction, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER, TABLE_ADMIN); - verifyIfEmptyList(listTablesAction, USER_RW, USER_RO, USER_NONE); + verifyAllowed(listTablesAction, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER, TABLE_ADMIN); + verifyIfEmptyList(listTablesAction, USER_RW, USER_RO, USER_NONE); - verifyAllowed(getTableDescAction, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER, TABLE_ADMIN); - verifyDenied(getTableDescAction, USER_RW, USER_RO, USER_NONE); + verifyAllowed(getTableDescAction, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER, + TABLE_ADMIN); + verifyDenied(getTableDescAction, USER_RW, USER_RO, USER_NONE); + } finally { + // Cleanup, revoke TABLE ADMIN privs + revokeFromTable(TEST_UTIL, TABLE_ADMIN.getShortName(), TEST_TABLE, null, null, + Permission.Action.ADMIN); + } } @Test @@ -1997,19 +1999,20 @@ public class TestAccessController extends SecureTestUtil { @Test public void testTableDeletion() throws Exception { User TABLE_ADMIN = User.createUserForTesting(conf, "TestUser", new String[0]); + final TableName tname = TableName.valueOf("testTableDeletion"); + createTestTable(tname); // Grant TABLE ADMIN privs - grantOnTable(TEST_UTIL, TABLE_ADMIN.getShortName(), - TEST_TABLE.getTableName(), null, null, - Permission.Action.ADMIN); + grantOnTable(TEST_UTIL, TABLE_ADMIN.getShortName(), tname, null, null, Permission.Action.ADMIN); AccessTestAction deleteTableAction = new AccessTestAction() { @Override public Object run() throws Exception { - Connection unmanagedConnection = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); + Connection unmanagedConnection = + ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); Admin admin = unmanagedConnection.getAdmin(); try { - deleteTable(TEST_UTIL, admin, TEST_TABLE.getTableName()); + deleteTable(TEST_UTIL, admin, tname); } finally { admin.close(); unmanagedConnection.close(); @@ -2022,19 +2025,28 @@ public class TestAccessController extends SecureTestUtil { verifyAllowed(deleteTableAction, TABLE_ADMIN); } + private void createTestTable(TableName tname) throws Exception { + HTableDescriptor htd = new HTableDescriptor(tname); + HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY); + hcd.setMaxVersions(100); + htd.addFamily(hcd); + htd.setOwner(USER_OWNER); + createTable(TEST_UTIL, htd, new byte[][]{Bytes.toBytes("s")}); + } + @Test public void testNamespaceUserGrant() throws Exception { AccessTestAction getAction = new AccessTestAction() { @Override public Object run() throws Exception { try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(TEST_TABLE.getTableName());) { + Table t = conn.getTable(TEST_TABLE);) { return t.get(new Get(TEST_ROW)); } } }; - String namespace = TEST_TABLE.getTableName().getNamespaceAsString(); + String namespace = TEST_TABLE.getNamespaceAsString(); // Grant namespace READ to USER_NONE, this should supersede any table permissions grantOnNamespace(TEST_UTIL, USER_NONE.getShortName(), namespace, Permission.Action.READ); @@ -2054,7 +2066,7 @@ public class TestAccessController extends SecureTestUtil { @Override public Object run() throws Exception { try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(TEST_TABLE.getTableName())) { + Table t = conn.getTable(TEST_TABLE);) { return t.get(new Get(TEST_ROW)); } } @@ -2064,8 +2076,8 @@ public class TestAccessController extends SecureTestUtil { // Grant table READ permissions to testGrantRevoke. try { - grantOnTableUsingAccessControlClient(TEST_UTIL, systemUserConnection, testGrantRevoke.getShortName(), - TEST_TABLE.getTableName(), null, null, Permission.Action.READ); + grantOnTableUsingAccessControlClient(TEST_UTIL, systemUserConnection, + testGrantRevoke.getShortName(), TEST_TABLE, null, null, Permission.Action.READ); } catch (Throwable e) { LOG.error("error during call of AccessControlClient.grant. ", e); } @@ -2075,8 +2087,8 @@ public class TestAccessController extends SecureTestUtil { // Revoke table READ permission to testGrantRevoke. try { - revokeFromTableUsingAccessControlClient(TEST_UTIL, systemUserConnection, testGrantRevoke.getShortName(), - TEST_TABLE.getTableName(), null, null, Permission.Action.READ); + revokeFromTableUsingAccessControlClient(TEST_UTIL, systemUserConnection, + testGrantRevoke.getShortName(), TEST_TABLE, null, null, Permission.Action.READ); } catch (Throwable e) { LOG.error("error during call of AccessControlClient.revoke ", e); } @@ -2094,7 +2106,7 @@ public class TestAccessController extends SecureTestUtil { @Override public Object run() throws Exception { try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(TEST_TABLE.getTableName())) { + Table t = conn.getTable(TEST_TABLE)) { return t.get(new Get(TEST_ROW)); } } @@ -2103,26 +2115,30 @@ public class TestAccessController extends SecureTestUtil { verifyDenied(getAction, testGlobalGrantRevoke); // Grant table READ permissions to testGlobalGrantRevoke. + String userName = testGlobalGrantRevoke.getShortName(); try { grantGlobalUsingAccessControlClient(TEST_UTIL, systemUserConnection, - testGlobalGrantRevoke.getShortName(), Permission.Action.READ); + userName, Permission.Action.READ); } catch (Throwable e) { LOG.error("error during call of AccessControlClient.grant. ", e); } + try { + // Now testGlobalGrantRevoke should be able to read also + verifyAllowed(getAction, testGlobalGrantRevoke); - // Now testGlobalGrantRevoke should be able to read also - verifyAllowed(getAction, testGlobalGrantRevoke); + // Revoke table READ permission to testGlobalGrantRevoke. + try { + revokeGlobalUsingAccessControlClient(TEST_UTIL, systemUserConnection, + userName, Permission.Action.READ); + } catch (Throwable e) { + LOG.error("error during call of AccessControlClient.revoke ", e); + } - // Revoke table READ permission to testGlobalGrantRevoke. - try { - revokeGlobalUsingAccessControlClient(TEST_UTIL, systemUserConnection, - testGlobalGrantRevoke.getShortName(), Permission.Action.READ); - } catch (Throwable e) { - LOG.error("error during call of AccessControlClient.revoke ", e); + // Now testGlobalGrantRevoke shouldn't be able read + verifyDenied(getAction, testGlobalGrantRevoke); + } finally { + revokeGlobal(TEST_UTIL, userName, Permission.Action.READ); } - - // Now testGlobalGrantRevoke shouldn't be able read - verifyDenied(getAction, testGlobalGrantRevoke); } @Test @@ -2133,7 +2149,7 @@ public class TestAccessController extends SecureTestUtil { @Override public Object run() throws Exception { try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(TEST_TABLE.getTableName())) { + Table t = conn.getTable(TEST_TABLE);) { return t.get(new Get(TEST_ROW)); } } @@ -2141,27 +2157,32 @@ public class TestAccessController extends SecureTestUtil { verifyDenied(getAction, testNS); + String userName = testNS.getShortName(); + String namespace = TEST_TABLE.getNamespaceAsString(); // Grant namespace READ to testNS, this should supersede any table permissions try { - grantOnNamespaceUsingAccessControlClient(TEST_UTIL, systemUserConnection, testNS.getShortName(), - TEST_TABLE.getTableName().getNamespaceAsString(), Permission.Action.READ); + grantOnNamespaceUsingAccessControlClient(TEST_UTIL, systemUserConnection, userName, + namespace, Permission.Action.READ); } catch (Throwable e) { LOG.error("error during call of AccessControlClient.grant. ", e); } + try { + // Now testNS should be able to read also + verifyAllowed(getAction, testNS); - // Now testNS should be able to read also - verifyAllowed(getAction, testNS); + // Revoke namespace READ to testNS, this should supersede any table permissions + try { + revokeFromNamespaceUsingAccessControlClient(TEST_UTIL, systemUserConnection, userName, + namespace, Permission.Action.READ); + } catch (Throwable e) { + LOG.error("error during call of AccessControlClient.revoke ", e); + } - // Revoke namespace READ to testNS, this should supersede any table permissions - try { - revokeFromNamespaceUsingAccessControlClient(TEST_UTIL, systemUserConnection, testNS.getShortName(), - TEST_TABLE.getTableName().getNamespaceAsString(), Permission.Action.READ); - } catch (Throwable e) { - LOG.error("error during call of AccessControlClient.revoke ", e); + // Now testNS shouldn't be able read + verifyDenied(getAction, testNS); + } finally { + revokeFromNamespace(TEST_UTIL, userName, namespace, Permission.Action.READ); } - - // Now testNS shouldn't be able read - verifyDenied(getAction, testNS); } @@ -2216,7 +2237,7 @@ public class TestAccessController extends SecureTestUtil { for (JVMClusterUtil.RegionServerThread thread: TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads()) { HRegionServer rs = thread.getRegionServer(); - for (Region region: rs.getOnlineRegions(TEST_TABLE.getTableName())) { + for (Region region: rs.getOnlineRegions(TEST_TABLE)) { region.getCoprocessorHost().load(PingCoprocessor.class, Coprocessor.PRIORITY_USER, conf); } @@ -2228,32 +2249,37 @@ public class TestAccessController extends SecureTestUtil { User userB = User.createUserForTesting(conf, "UserB", new String[0]); grantOnTable(TEST_UTIL, userA.getShortName(), - TEST_TABLE.getTableName(), null, null, + TEST_TABLE, null, null, Permission.Action.EXEC); - - // Create an action for invoking our test endpoint - AccessTestAction execEndpointAction = new AccessTestAction() { - @Override - public Object run() throws Exception { - try(Connection conn = ConnectionFactory.createConnection(conf); - Table t = conn.getTable(TEST_TABLE.getTableName());) { - BlockingRpcChannel service = t.coprocessorService(HConstants.EMPTY_BYTE_ARRAY); - PingCoprocessor.newBlockingStub(service).noop(null, NoopRequest.newBuilder().build()); + try { + // Create an action for invoking our test endpoint + AccessTestAction execEndpointAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + try (Connection conn = ConnectionFactory.createConnection(conf); + Table t = conn.getTable(TEST_TABLE);) { + BlockingRpcChannel service = t.coprocessorService(HConstants.EMPTY_BYTE_ARRAY); + PingCoprocessor.newBlockingStub(service).noop(null, NoopRequest.newBuilder().build()); + } + return null; } - return null; - } - }; + }; - String namespace = TEST_TABLE.getTableName().getNamespaceAsString(); - // Now grant EXEC to the entire namespace to user B - grantOnNamespace(TEST_UTIL, userB.getShortName(), namespace, Permission.Action.EXEC); - // User B should now be allowed also - verifyAllowed(execEndpointAction, userA, userB); + String namespace = TEST_TABLE.getNamespaceAsString(); + // Now grant EXEC to the entire namespace to user B + grantOnNamespace(TEST_UTIL, userB.getShortName(), namespace, Permission.Action.EXEC); + // User B should now be allowed also + verifyAllowed(execEndpointAction, userA, userB); - revokeFromNamespace(TEST_UTIL, userB.getShortName(), namespace, Permission.Action.EXEC); - // Verify that EXEC permission is checked correctly - verifyDenied(execEndpointAction, userB); - verifyAllowed(execEndpointAction, userA); + revokeFromNamespace(TEST_UTIL, userB.getShortName(), namespace, Permission.Action.EXEC); + // Verify that EXEC permission is checked correctly + verifyDenied(execEndpointAction, userB); + verifyAllowed(execEndpointAction, userA); + } finally { + // Cleanup, revoke the userA privileges + revokeFromTable(TEST_UTIL, userA.getShortName(), TEST_TABLE, null, null, + Permission.Action.EXEC); + } } @Test @@ -2261,16 +2287,14 @@ public class TestAccessController extends SecureTestUtil { AccessTestAction putWithReservedTag = new AccessTestAction() { @Override public Object run() throws Exception { - Table t = new HTable(conf, TEST_TABLE.getTableName()); - try { + try(Connection conn = ConnectionFactory.createConnection(conf); + Table t = conn.getTable(TEST_TABLE);) { KeyValue kv = new KeyValue(TEST_ROW, TEST_FAMILY, TEST_QUALIFIER, HConstants.LATEST_TIMESTAMP, HConstants.EMPTY_BYTE_ARRAY, new Tag[] { new Tag(AccessControlLists.ACL_TAG_TYPE, ProtobufUtil.toUsersAndPermissions(USER_OWNER.getShortName(), new Permission(Permission.Action.READ)).toByteArray()) }); t.put(new Put(TEST_ROW).add(kv)); - } finally { - t.close(); } return null; } @@ -2282,38 +2306,104 @@ public class TestAccessController extends SecureTestUtil { verifyDenied(putWithReservedTag, USER_OWNER, USER_ADMIN, USER_CREATE, USER_RW, USER_RO); } - @Test - public void testGetNamespacePermission() throws Exception { - String namespace = "testGetNamespacePermission"; - NamespaceDescriptor desc = NamespaceDescriptor.create(namespace).build(); - createNamespace(TEST_UTIL, desc); - grantOnNamespace(TEST_UTIL, USER_NONE.getShortName(), namespace, Permission.Action.READ); - try { - List namespacePermissions = AccessControlClient.getUserPermissions( - systemUserConnection, AccessControlLists.toNamespaceEntry(namespace)); - assertTrue(namespacePermissions != null); - assertTrue(namespacePermissions.size() == 1); - } catch (Throwable thw) { - throw new HBaseException(thw); - } - deleteNamespace(TEST_UTIL, namespace); - } + @Test + public void testSetQuota() throws Exception { + AccessTestAction setUserQuotaAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preSetUserQuota(ObserverContext.createAndPrepare(CP_ENV, null), + null, null); + return null; + } + }; + + AccessTestAction setUserTableQuotaAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preSetUserQuota(ObserverContext.createAndPrepare(CP_ENV, null), null, + TEST_TABLE, null); + return null; + } + }; + + AccessTestAction setUserNamespaceQuotaAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preSetUserQuota(ObserverContext.createAndPrepare(CP_ENV, null), + null, (String)null, null); + return null; + } + }; + + AccessTestAction setTableQuotaAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preSetTableQuota(ObserverContext.createAndPrepare(CP_ENV, null), + TEST_TABLE, null); + return null; + } + }; + + AccessTestAction setNamespaceQuotaAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + ACCESS_CONTROLLER.preSetNamespaceQuota(ObserverContext.createAndPrepare(CP_ENV, null), + null, null); + return null; + } + }; + + verifyAllowed(setUserQuotaAction, SUPERUSER, USER_ADMIN); + verifyDenied(setUserQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + + verifyAllowed(setUserTableQuotaAction, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(setUserTableQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE); + + verifyAllowed(setUserNamespaceQuotaAction, SUPERUSER, USER_ADMIN); + verifyDenied(setUserNamespaceQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, + USER_OWNER); + + verifyAllowed(setTableQuotaAction, SUPERUSER, USER_ADMIN, USER_OWNER); + verifyDenied(setTableQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE); + + verifyAllowed(setNamespaceQuotaAction, SUPERUSER, USER_ADMIN); + verifyDenied(setNamespaceQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); + } + + @Test + public void testGetNamespacePermission() throws Exception { + String namespace = "testGetNamespacePermission"; + NamespaceDescriptor desc = NamespaceDescriptor.create(namespace).build(); + createNamespace(TEST_UTIL, desc); + grantOnNamespace(TEST_UTIL, USER_NONE.getShortName(), namespace, Permission.Action.READ); + try { + List namespacePermissions = AccessControlClient.getUserPermissions( + systemUserConnection, AccessControlLists.toNamespaceEntry(namespace)); + assertTrue(namespacePermissions != null); + assertTrue(namespacePermissions.size() == 1); + } catch (Throwable thw) { + throw new HBaseException(thw); + } + deleteNamespace(TEST_UTIL, namespace); + } @Test - public void testTruncatePerms() throws Throwable { - List existingPerms = - AccessControlClient.getUserPermissions(systemUserConnection, - TEST_TABLE.getTableName().getNameAsString()); - assertTrue(existingPerms != null); - assertTrue(existingPerms.size() > 1); - try (Admin admin = systemUserConnection.getAdmin()) { - admin.disableTable(TEST_TABLE.getTableName()); - admin.truncateTable(TEST_TABLE.getTableName(), true); + public void testTruncatePerms() throws Exception { + try { + List existingPerms = AccessControlClient.getUserPermissions( + systemUserConnection, TEST_TABLE.getNameAsString()); + assertTrue(existingPerms != null); + assertTrue(existingPerms.size() > 1); + TEST_UTIL.getHBaseAdmin().disableTable(TEST_TABLE); + TEST_UTIL.truncateTable(TEST_TABLE); + TEST_UTIL.waitTableAvailable(TEST_TABLE); + List perms = AccessControlClient.getUserPermissions( + systemUserConnection, TEST_TABLE.getNameAsString()); + assertTrue(perms != null); + assertEquals(existingPerms.size(), perms.size()); + } catch (Throwable e) { + throw new HBaseIOException(e); } - List perms = AccessControlClient.getUserPermissions(systemUserConnection, - TEST_TABLE.getTableName().getNameAsString()); - assertTrue(perms != null); - assertEquals(existingPerms.size(), perms.size()); } private PrivilegedAction> getPrivilegedAction(final String regex) { @@ -2332,20 +2422,21 @@ public class TestAccessController extends SecureTestUtil { @Test public void testAccessControlClientUserPerms() throws Exception { - // adding default prefix explicitly as it is not included in the table name. - assertEquals(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR, - TEST_TABLE.getTableName().getNamespaceAsString()); - final String regex = NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR + - TableName.NAMESPACE_DELIM + TEST_TABLE.getTableName().getNameAsString(); - User testUserPerms = User.createUserForTesting(conf, "testUserPerms", new String[0]); - assertEquals(0, testUserPerms.runAs(getPrivilegedAction(regex)).size()); - // Grant TABLE ADMIN privs to testUserPerms - grantOnTable(TEST_UTIL, testUserPerms.getShortName(), TEST_TABLE.getTableName(), null, - null, Action.ADMIN); - List perms = testUserPerms.runAs(getPrivilegedAction(regex)); - assertNotNull(perms); - // USER_ADMIN, USER_CREATE, USER_RW, USER_RO, testUserPerms, USER_ADMIN_CF has row each. - assertEquals(6, perms.size()); + TableName tname = TableName.valueOf("testAccessControlClientUserPerms"); + createTestTable(tname); + try { + final String regex = tname.getNameWithNamespaceInclAsString(); + User testUserPerms = User.createUserForTesting(conf, "testUserPerms", new String[0]); + assertEquals(0, testUserPerms.runAs(getPrivilegedAction(regex)).size()); + // Grant TABLE ADMIN privs to testUserPerms + grantOnTable(TEST_UTIL, testUserPerms.getShortName(), tname, null, null, Action.ADMIN); + List perms = testUserPerms.runAs(getPrivilegedAction(regex)); + assertNotNull(perms); + // Superuser, testUserPerms + assertEquals(2, perms.size()); + } finally { + deleteTable(TEST_UTIL, tname); + } } @Test @@ -2408,14 +2499,16 @@ public class TestAccessController extends SecureTestUtil { AccessTestAction prepareBulkLoadAction = new AccessTestAction() { @Override public Object run() throws Exception { - ACCESS_CONTROLLER.prePrepareBulkLoad(ObserverContext.createAndPrepare(RCP_ENV, null), null); + ACCESS_CONTROLLER.prePrepareBulkLoad(ObserverContext.createAndPrepare(RCP_ENV, null), + null); return null; } }; AccessTestAction cleanupBulkLoadAction = new AccessTestAction() { @Override public Object run() throws Exception { - ACCESS_CONTROLLER.preCleanupBulkLoad(ObserverContext.createAndPrepare(RCP_ENV, null), null); + ACCESS_CONTROLLER.preCleanupBulkLoad(ObserverContext.createAndPrepare(RCP_ENV, null), + null); return null; } }; @@ -2439,67 +2532,4 @@ public class TestAccessController extends SecureTestUtil { verifyAllowed(replicateLogEntriesAction, SUPERUSER, USER_ADMIN); verifyDenied(replicateLogEntriesAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); } - - @Test - public void testSetQuota() throws Exception { - AccessTestAction setUserQuotaAction = new AccessTestAction() { - @Override - public Object run() throws Exception { - ACCESS_CONTROLLER.preSetUserQuota(ObserverContext.createAndPrepare(CP_ENV, null), - null, null); - return null; - } - }; - - AccessTestAction setUserTableQuotaAction = new AccessTestAction() { - @Override - public Object run() throws Exception { - ACCESS_CONTROLLER.preSetUserQuota(ObserverContext.createAndPrepare(CP_ENV, null), - null, TEST_TABLE.getTableName(), null); - return null; - } - }; - - AccessTestAction setUserNamespaceQuotaAction = new AccessTestAction() { - @Override - public Object run() throws Exception { - ACCESS_CONTROLLER.preSetUserQuota(ObserverContext.createAndPrepare(CP_ENV, null), - null, (String)null, null); - return null; - } - }; - - AccessTestAction setTableQuotaAction = new AccessTestAction() { - @Override - public Object run() throws Exception { - ACCESS_CONTROLLER.preSetTableQuota(ObserverContext.createAndPrepare(CP_ENV, null), - TEST_TABLE.getTableName(), null); - return null; - } - }; - - AccessTestAction setNamespaceQuotaAction = new AccessTestAction() { - @Override - public Object run() throws Exception { - ACCESS_CONTROLLER.preSetNamespaceQuota(ObserverContext.createAndPrepare(CP_ENV, null), - null, null); - return null; - } - }; - - verifyAllowed(setUserQuotaAction, SUPERUSER, USER_ADMIN); - verifyDenied(setUserQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); - - verifyAllowed(setUserTableQuotaAction, SUPERUSER, USER_ADMIN, USER_OWNER); - verifyDenied(setUserTableQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE); - - verifyAllowed(setUserNamespaceQuotaAction, SUPERUSER, USER_ADMIN); - verifyDenied(setUserNamespaceQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); - - verifyAllowed(setTableQuotaAction, SUPERUSER, USER_ADMIN, USER_OWNER); - verifyDenied(setTableQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE); - - verifyAllowed(setNamespaceQuotaAction, SUPERUSER, USER_ADMIN); - verifyDenied(setNamespaceQuotaAction, USER_CREATE, USER_RW, USER_RO, USER_NONE, USER_OWNER); - } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController2.java index f352136..01a45bc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController2.java @@ -196,6 +196,27 @@ public class TestAccessController2 extends SecureTestUtil { } @Test + public void testCreateTableWithGroupPermissions() throws Exception { + grantGlobal(TEST_UTIL, convertToGroup(TESTGROUP_1), Action.CREATE); + AccessTestAction createAction = new AccessTestAction() { + @Override + public Object run() throws Exception { + HTableDescriptor desc = new HTableDescriptor(TEST_TABLE.getTableName()); + desc.addFamily(new HColumnDescriptor(TEST_FAMILY)); + try (Connection connection = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) { + try (Admin admin = connection.getAdmin()) { + admin.createTable(desc); + } + } + return null; + } + }; + verifyAllowed(createAction, TESTGROUP1_USER1); + verifyDenied(createAction, TESTGROUP2_USER1); + revokeGlobal(TEST_UTIL, convertToGroup(TESTGROUP_1), Action.CREATE); + } + + @Test public void testACLTableAccess() throws Exception { final Configuration conf = TEST_UTIL.getConfiguration(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java index 74a1754..89888de 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.Properties; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.LocalHBaseCluster; @@ -47,6 +48,7 @@ import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos.WhoAmIRes import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.security.HBaseKerberosUtils; import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.minikdc.MiniKdc; @@ -121,6 +123,8 @@ public class TestGenerateDelegationToken { TEST_UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, TokenProvider.class.getName()); TEST_UTIL.startMiniDFSCluster(1); + Path rootdir = TEST_UTIL.getDataTestDirOnTestFS("TestGenerateDelegationToken"); + FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootdir); CLUSTER = new LocalHBaseCluster(TEST_UTIL.getConfiguration(), 1); CLUSTER.startup(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java index 85e947b..185893a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java @@ -30,6 +30,7 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -38,6 +39,7 @@ import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; @@ -56,8 +58,11 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResul import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsResponse; import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.Region; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; @@ -805,6 +810,39 @@ public abstract class TestVisibilityLabels { } } + @Test + public void testFlushedFileWithVisibilityTags() throws Exception { + final byte[] qual2 = Bytes.toBytes("qual2"); + TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTableDescriptor desc = new HTableDescriptor(tableName); + HColumnDescriptor col = new HColumnDescriptor(fam); + desc.addFamily(col); + TEST_UTIL.getHBaseAdmin().createTable(desc); + try (Table table = TEST_UTIL.getConnection().getTable(tableName)) { + Put p1 = new Put(row1); + p1.add(fam, qual, value); + p1.setCellVisibility(new CellVisibility(CONFIDENTIAL)); + + Put p2 = new Put(row1); + p2.add(fam, qual2, value); + p2.setCellVisibility(new CellVisibility(SECRET)); + + RowMutations rm = new RowMutations(row1); + rm.add(p1); + rm.add(p2); + + table.mutateRow(rm); + } + TEST_UTIL.getHBaseAdmin().flush(tableName); + List regions = TEST_UTIL.getHBaseCluster().getRegions(tableName); + Store store = regions.get(0).getStore(fam); + Collection storefiles = store.getStorefiles(); + assertTrue(storefiles.size() > 0); + for (StoreFile storeFile : storefiles) { + assertTrue(storeFile.getReader().getHFileReader().getFileContext().isIncludesTags()); + } + } + static Table createTableAndWriteDataWithLabels(TableName tableName, String... labelExps) throws Exception { List puts = new ArrayList(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java index 5168b85..8ebeb97 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSnapshotClientRetries.java @@ -18,6 +18,8 @@ */ package org.apache.hadoop.hbase.snapshot; +import static org.junit.Assert.assertEquals; + import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; @@ -31,21 +33,14 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; -import org.apache.hadoop.hbase.snapshot.SnapshotExistsException; -import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.TestTableName; - import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - @Category({ MediumTests.class }) public class TestSnapshotClientRetries { private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestConfigurationUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestConfigurationUtil.java new file mode 100644 index 0000000..a9ecf9e --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestConfigurationUtil.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.util; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +@Category({ SmallTests.class }) +public class TestConfigurationUtil { + + private Configuration conf; + private Map keyValues; + private String key; + + @Before + public void setUp() throws Exception { + this.conf = new Configuration(); + this.keyValues = ImmutableMap.of("k1", "v1", "k2", "v2"); + this.key = "my_conf_key"; + } + + public void callSetKeyValues() { + ConfigurationUtil.setKeyValues(conf, key, keyValues.entrySet()); + } + + public List> callGetKeyValues() { + return ConfigurationUtil.getKeyValues(conf, key); + } + + @Test + public void testGetAndSetKeyValuesWithValues() throws Exception { + callSetKeyValues(); + assertEquals(Lists.newArrayList(this.keyValues.entrySet()), callGetKeyValues()); + } + + @Test + public void testGetKeyValuesWithUnsetKey() throws Exception { + assertNull(callGetKeyValues()); + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java index b5ee300..4058534 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java @@ -583,6 +583,9 @@ public class TestHBaseFsck { public HBaseFsck call(){ Configuration c = new Configuration(conf); c.setInt("hbase.hbck.lockfile.attempts", 1); + // HBASE-13574 found that in HADOOP-2.6 and later, the create file would internally retry. + // To avoid flakiness of the test, set low max wait time. + c.setInt("hbase.hbck.lockfile.maxwaittime", 3); try{ return doFsck(c, false); } catch(Exception e){ @@ -614,7 +617,7 @@ public class TestHBaseFsck { } /** - * This test makes sure that with 10 retries both parallel instances + * This test makes sure that with enough retries both parallel instances * of hbck will be completed successfully. * * @throws Exception @@ -624,22 +627,37 @@ public class TestHBaseFsck { final ExecutorService service; final Future hbck1,hbck2; + // With the ExponentialBackoffPolicyWithLimit (starting with 200 milliseconds sleep time, and + // max sleep time of 5 seconds), we can retry around 15 times within 80 seconds before bail out. + // + // Note: the reason to use 80 seconds is that in HADOOP-2.6 and later, the create file would + // retry up to HdfsConstants.LEASE_SOFTLIMIT_PERIOD (60 seconds). See HBASE-13574 for more + // details. + final int timeoutInSeconds = 80; + final int sleepIntervalInMilliseconds = 200; + final int maxSleepTimeInMilliseconds = 6000; + final int maxRetryAttempts = 15; + class RunHbck implements Callable{ @Override public HBaseFsck call() throws Exception { // Increase retry attempts to make sure the non-active hbck doesn't get starved Configuration c = new Configuration(conf); - c.setInt("hbase.hbck.lockfile.attempts", 10); + c.setInt("hbase.hbck.lockfile.maxwaittime", timeoutInSeconds); + c.setInt("hbase.hbck.lockfile.attempt.sleep.interval", sleepIntervalInMilliseconds); + c.setInt("hbase.hbck.lockfile.attempt.maxsleeptime", maxSleepTimeInMilliseconds); + c.setInt("hbase.hbck.lockfile.attempts", maxRetryAttempts); return doFsck(c, false); } } + service = Executors.newFixedThreadPool(2); hbck1 = service.submit(new RunHbck()); hbck2 = service.submit(new RunHbck()); service.shutdown(); - //wait for 15 seconds, for both hbck calls finish - service.awaitTermination(25, TimeUnit.SECONDS); + //wait for some time, for both hbck calls finish + service.awaitTermination(timeoutInSeconds * 2, TimeUnit.SECONDS); HBaseFsck h1 = hbck1.get(); HBaseFsck h2 = hbck2.get(); // Both should be successful diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java index 56b2d67..467c63d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java @@ -57,7 +57,7 @@ import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hdfs.protocol.FSConstants; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -383,7 +383,7 @@ public class TestWALFactory { // Stop the cluster. (ensure restart since we're sharing MiniDFSCluster) try { DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem(); - dfs.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_ENTER); + dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER); TEST_UTIL.shutdownMiniDFSCluster(); try { // wal.writer.close() will throw an exception, diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml index 8c8312c..34a1b20 100644 --- a/hbase-server/src/test/resources/hbase-site.xml +++ b/hbase-server/src/test/resources/hbase-site.xml @@ -147,4 +147,11 @@ Skip sanity checks in tests + + hbase.procedure.fail.on.corruption + true + + Enable replay sanity checks on procedure tests. + + diff --git a/hbase-shaded/hbase-shaded-client/pom.xml b/hbase-shaded/hbase-shaded-client/pom.xml index 7c8150d..34ac7a1 100644 --- a/hbase-shaded/hbase-shaded-client/pom.xml +++ b/hbase-shaded/hbase-shaded-client/pom.xml @@ -39,10 +39,6 @@ - org.apache.maven.plugins - maven-shade-plugin - - maven-assembly-plugin @@ -58,4 +54,17 @@ + + + release + + + + org.apache.maven.plugins + maven-shade-plugin + + + + + \ No newline at end of file diff --git a/hbase-shaded/hbase-shaded-server/pom.xml b/hbase-shaded/hbase-shaded-server/pom.xml index 7bce10c..d75b723 100644 --- a/hbase-shaded/hbase-shaded-server/pom.xml +++ b/hbase-shaded/hbase-shaded-server/pom.xml @@ -39,10 +39,6 @@ - org.apache.maven.plugins - maven-shade-plugin - - maven-assembly-plugin @@ -58,4 +54,17 @@ + + + release + + + + org.apache.maven.plugins + maven-shade-plugin + + + + + \ No newline at end of file diff --git a/hbase-shell/src/main/ruby/hbase/table.rb b/hbase-shell/src/main/ruby/hbase/table.rb index 4293ace..0946eb2 100644 --- a/hbase-shell/src/main/ruby/hbase/table.rb +++ b/hbase-shell/src/main/ruby/hbase/table.rb @@ -271,6 +271,7 @@ EOF org.apache.hadoop.hbase.util.Bytes::toStringBinary(row.getRow)) end + scanner.close() # Return the counter return count end @@ -525,8 +526,8 @@ EOF break end end - scanner.close() + scanner.close() return ((block_given?) ? count : res) end diff --git a/pom.xml b/pom.xml index a518dc0..b335db8 100644 --- a/pom.xml +++ b/pom.xml @@ -775,6 +775,13 @@ org.apache.maven.plugins maven-enforcer-plugin + + + org.codehaus.mojo + extra-enforcer-rules + ${extra.enforcer.version} + + @@ -1193,6 +1200,7 @@ true 0.6.2.201302030002 + 1.0-beta-3 @@ -1809,6 +1817,24 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + + ${compileSource} + HBase has unsupported dependencies. + HBase requires that all dependencies be compiled with version ${compileSource} or earlier + of the JDK to properly build from source. You appear to be using a newer dependency. You can use + either "mvn -version" or "mvn enforcer:display-info" to verify what version is active. + Non-release builds can temporarily build with a newer JDK version by setting the + 'compileSource' property (eg. mvn -DcompileSource=1.8 clean package). + + + + +