diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java index 75a4b55..a865eca 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java @@ -201,7 +201,8 @@ public class KeyValueUtil { * @return cell if it is an instance of {@link KeyValue} else we will return a * new {@link KeyValue} instance made from cell */ - public static Cell ensureKeyValue(final Cell cell) { - return cell instanceof KeyValue? cell: copyToNewKeyValue(cell); + public static KeyValue ensureKeyValue(final Cell cell) { + if (cell == null) return null; + return cell instanceof KeyValue? (KeyValue)cell: copyToNewKeyValue(cell); } } diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index 2e09df1..773460a 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -57,8 +57,8 @@ - org.apache.maven.plugins @@ -69,8 +69,8 @@ org/apache/hadoop/hbase/mapreduce/Driver - org/apache/jute/** @@ -211,8 +211,9 @@ - + org.eclipse.m2e lifecycle-mapping @@ -542,11 +543,13 @@ - - + - + @@ -560,8 +563,8 @@ - hadoop-1.0 diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Action.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Action.java index 1a5152f..ba14702 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Action.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Action.java @@ -29,33 +29,16 @@ import org.apache.hadoop.classification.InterfaceStability; @InterfaceAudience.Public @InterfaceStability.Stable public class Action implements Comparable { - + // TODO: This class should not be visible outside of the client package. private Row action; private int originalIndex; private R result; - /** - * This constructor is replaced by {@link #Action(Row, int)} - */ - @Deprecated - public Action(byte[] regionName, Row action, int originalIndex) { - this(action, originalIndex); - } - public Action(Row action, int originalIndex) { super(); this.action = action; this.originalIndex = originalIndex; } - - @Deprecated - public byte[] getRegionName() { - return null; - } - - @Deprecated - public void setRegionName(byte[] regionName) { - } public R getResult() { return result; @@ -73,6 +56,7 @@ public class Action implements Comparable { return originalIndex; } + @SuppressWarnings("rawtypes") @Override public int compareTo(Object o) { return action.compareTo(((Action) o).getAction()); @@ -85,4 +69,4 @@ public class Action implements Comparable { Action other = (Action) obj; return compareTo(other) == 0; } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Append.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Append.java index ba1e085..5e847cd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Append.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Append.java @@ -24,7 +24,9 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hbase.Cell; /** * Performs Append operations on a single row. @@ -78,13 +80,25 @@ public class Append extends Mutation { * @return this */ public Append add(byte [] family, byte [] qualifier, byte [] value) { - List list = familyMap.get(family); - if(list == null) { - list = new ArrayList(); + KeyValue kv = new KeyValue(this.row, family, qualifier, this.ts, KeyValue.Type.Put, value); + return add(kv); + } + + /** + * Add column and value to this Append operation. + * @param cell + * @return This instance + */ + public Append add(final Cell cell) { + // Presume it is KeyValue for now. + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); + byte [] family = kv.getFamily(); + List list = this.familyMap.get(family); + if (list == null) { + list = new ArrayList(); } - list.add(new KeyValue( - this.row, family, qualifier, this.ts, KeyValue.Type.Put, value)); - familyMap.put(family, list); + list.add(kv); + this.familyMap.put(family, list); return this; } } \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java index 8a4999a..2a6f291 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java @@ -38,7 +38,6 @@ import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.MapReduceProtos; import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.DataOutputBuffer; /** * Implements the scanner interface for the HBase client. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Delete.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Delete.java index 4103b77..4f17785 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Delete.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Delete.java @@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hbase.Cell; import java.io.IOException; import java.util.ArrayList; @@ -112,6 +113,7 @@ public class Delete extends Mutation implements Comparable { * @throws IOException */ public Delete addDeleteMarker(KeyValue kv) throws IOException { + // TODO: Deprecate and rename 'add' so it matches how we add KVs to Puts. if (!kv.isDelete()) { throw new IOException("The recently added KeyValue is not of type " + "delete. Rowkey: " + Bytes.toStringBinary(this.row)); @@ -124,9 +126,9 @@ public class Delete extends Mutation implements Comparable { + Bytes.toStringBinary(this.row)); } byte [] family = kv.getFamily(); - List list = familyMap.get(family); + List list = familyMap.get(family); if (list == null) { - list = new ArrayList(); + list = new ArrayList(); } list.add(kv); familyMap.put(family, list); @@ -157,9 +159,9 @@ public class Delete extends Mutation implements Comparable { * @return this for invocation chaining */ public Delete deleteFamily(byte [] family, long timestamp) { - List list = familyMap.get(family); + List list = familyMap.get(family); if(list == null) { - list = new ArrayList(); + list = new ArrayList(); } else if(!list.isEmpty()) { list.clear(); } @@ -188,9 +190,9 @@ public class Delete extends Mutation implements Comparable { * @return this for invocation chaining */ public Delete deleteColumns(byte [] family, byte [] qualifier, long timestamp) { - List list = familyMap.get(family); + List list = familyMap.get(family); if (list == null) { - list = new ArrayList(); + list = new ArrayList(); } list.add(new KeyValue(this.row, family, qualifier, timestamp, KeyValue.Type.DeleteColumn)); @@ -220,9 +222,9 @@ public class Delete extends Mutation implements Comparable { * @return this for invocation chaining */ public Delete deleteColumn(byte [] family, byte [] qualifier, long timestamp) { - List list = familyMap.get(family); + List list = familyMap.get(family); if(list == null) { - list = new ArrayList(); + list = new ArrayList(); } list.add(new KeyValue( this.row, family, qualifier, timestamp, KeyValue.Type.Delete)); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Get.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Get.java index b3b1adc..0b6636d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Get.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Get.java @@ -41,8 +41,7 @@ import java.util.TreeSet; * Used to perform Get operations on a single row. *

* To get everything for a row, instantiate a Get object with the row to get. - * To further define the scope of what to get, perform additional methods as - * outlined below. + * To further narrow the scope of what to Get, use the methods below. *

* To get all columns from specific families, execute {@link #addFamily(byte[]) addFamily} * for each family to retrieve. @@ -59,7 +58,7 @@ import java.util.TreeSet; * To limit the number of versions of each column to be returned, execute * {@link #setMaxVersions(int) setMaxVersions}. *

- * To add a filter, execute {@link #setFilter(Filter) setFilter}. + * To add a filter, call {@link #setFilter(Filter) setFilter}. */ @InterfaceAudience.Public @InterfaceStability.Stable diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HTable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HTable.java index 9a9128e..de9a485 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HTable.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HTable.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable; import org.apache.hadoop.hbase.client.coprocessor.Batch; @@ -67,6 +68,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.CompareType; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Threads; +import org.apache.hbase.Cell; import com.google.protobuf.Service; import com.google.protobuf.ServiceException; @@ -1099,8 +1101,10 @@ public class HTable implements HTableInterface { throw new IllegalArgumentException("No columns to insert"); } if (maxKeyValueSize > 0) { - for (List list : put.getFamilyMap().values()) { - for (KeyValue kv : list) { + for (List list : put.getFamilyMap().values()) { + for (Cell cell : list) { + // KeyValue v1 expectation. Cast for now. + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); if (kv.getLength() > maxKeyValueSize) { throw new IllegalArgumentException("KeyValue size too large"); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Increment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Increment.java index 8e59bd6..892d76d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Increment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Increment.java @@ -19,15 +19,19 @@ package org.apache.hadoop.hbase.client; import java.io.IOException; +import java.util.Arrays; +import java.util.List; import java.util.Map; -import java.util.NavigableMap; -import java.util.Set; -import java.util.TreeMap; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hbase.Cell; /** * Used to perform Increment operations on a single row. @@ -43,15 +47,8 @@ import org.apache.hadoop.hbase.util.Bytes; */ @InterfaceAudience.Public @InterfaceStability.Stable -public class Increment implements Row { - private byte [] row = null; - private boolean writeToWAL = true; +public class Increment extends Mutation implements HeapSize, Comparable { private TimeRange tr = new TimeRange(); - private Map> familyMap = - new TreeMap>(Bytes.BYTES_COMPARATOR); - - /** Constructor for Writable. DO NOT USE */ - public Increment() {} /** * Create a Increment operation for the specified row, using an existing row @@ -61,10 +58,10 @@ public class Increment implements Row { * @param row row key */ public Increment(byte [] row) { - if (row == null) { - throw new IllegalArgumentException("Cannot increment a null row"); + if (row == null || row.length > HConstants.MAX_ROW_LENGTH) { + throw new IllegalArgumentException("Row key is invalid"); } - this.row = row; + this.row = Arrays.copyOf(row, row.length); } /** @@ -84,40 +81,10 @@ public class Increment implements Row { if (qualifier == null) { throw new IllegalArgumentException("qualifier cannot be null"); } - NavigableMap set = familyMap.get(family); - if(set == null) { - set = new TreeMap(Bytes.BYTES_COMPARATOR); - } - set.put(qualifier, amount); - familyMap.put(family, set); - return this; - } - - /* Accessors */ - - /** - * Method for retrieving the increment's row - * @return row - */ - public byte [] getRow() { - return this.row; - } - - /** - * Method for retrieving whether WAL will be written to or not - * @return true if WAL should be used, false if not - */ - public boolean getWriteToWAL() { - return this.writeToWAL; - } - - /** - * Sets whether this operation should write to the WAL or not. - * @param writeToWAL true if WAL should be used, false if not - * @return this increment operation - */ - public Increment setWriteToWAL(boolean writeToWAL) { - this.writeToWAL = writeToWAL; + List list = getCellList(family); + KeyValue kv = createPutKeyValue(family, qualifier, ts, Bytes.toBytes(amount)); + list.add(kv); + familyMap.put(kv.getFamily(), list); return this; } @@ -150,14 +117,6 @@ public class Increment implements Row { } /** - * Method for retrieving the keys in the familyMap - * @return keys in the current familyMap - */ - public Set familySet() { - return this.familyMap.keySet(); - } - - /** * Method for retrieving the number of families to increment from * @return number of families */ @@ -166,19 +125,6 @@ public class Increment implements Row { } /** - * Method for retrieving the number of columns to increment - * @return number of columns across all families - */ - public int numColumns() { - if (!hasFamilies()) return 0; - int num = 0; - for (NavigableMap family : familyMap.values()) { - num += family.size(); - } - return num; - } - - /** * Method for checking if any families have been inserted into this Increment * @return true if familyMap is non empty false otherwise */ @@ -187,14 +133,6 @@ public class Increment implements Row { } /** - * Method for retrieving the increment's familyMap - * @return familyMap - */ - public Map> getFamilyMap() { - return this.familyMap; - } - - /** * @return String */ @Override @@ -208,8 +146,7 @@ public class Increment implements Row { } sb.append(", families="); boolean moreThanOne = false; - for(Map.Entry> entry : - this.familyMap.entrySet()) { + for(Map.Entry> entry: this.familyMap.entrySet()) { if(moreThanOne) { sb.append("), "); } else { @@ -224,13 +161,14 @@ public class Increment implements Row { } else { sb.append("{"); boolean moreThanOneB = false; - for(Map.Entry column : entry.getValue().entrySet()) { + for(Cell cell : entry.getValue()) { if(moreThanOneB) { sb.append(", "); } else { moreThanOneB = true; } - sb.append(Bytes.toStringBinary(column.getKey()) + "+=" + column.getValue()); + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); + sb.append(Bytes.toStringBinary(kv.getKey()) + "+=" + Bytes.toLong(kv.getValue())); } sb.append("}"); } @@ -255,4 +193,10 @@ public class Increment implements Row { Row other = (Row) obj; return compareTo(other) == 0; } -} + + @Override + public long heapSize() { + // TODO + return 0; + } +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java index 5605013..eedf0f0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java @@ -35,9 +35,11 @@ import org.apache.hadoop.hbase.util.Bytes; @InterfaceAudience.Public @InterfaceStability.Evolving public final class MultiAction { + // TODO: This class should not be visible outside of the client package. // map of regions to lists of puts/gets/deletes for that region. - public Map>> actions = new TreeMap>>(Bytes.BYTES_COMPARATOR); + public Map>> actions = + new TreeMap>>(Bytes.BYTES_COMPARATOR); public MultiAction() { super(); @@ -87,4 +89,4 @@ public final class MultiAction { } return res; } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Mutation.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Mutation.java index dd883a5..850d59d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Mutation.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Mutation.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.TreeMap; import java.util.UUID; @@ -30,19 +31,54 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hbase.Cell; +import org.apache.hbase.CellScannable; +import org.apache.hbase.CellScanner; +import org.apache.hbase.CellUtil; @InterfaceAudience.Public @InterfaceStability.Evolving -public abstract class Mutation extends OperationWithAttributes implements Row { +public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable { // Attribute used in Mutations to indicate the originating cluster. private static final String CLUSTER_ID_ATTR = "_c.id_"; protected byte [] row = null; protected long ts = HConstants.LATEST_TIMESTAMP; protected boolean writeToWAL = true; - protected Map> familyMap = - new TreeMap>(Bytes.BYTES_COMPARATOR); + // A Map sorted by column family. + protected NavigableMap> familyMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); + + @Override + public CellScanner cellScanner() { + return CellUtil.createCellScanner(getFamilyMap()); + } + + /** + * Creates an empty list if one doesn't exist for the given column family + * or else it returns the associated list of Cell objects. + * + * @param family column family + * @return a list of Cell objects, returns an empty list if one doesn't exist. + */ + List getCellList(byte[] family) { + List list = (List)this.familyMap.get(family); + if (list == null) { + list = new ArrayList(0); + } + return list; + } + + /* + * Create a KeyValue with this objects row key and the Put identifier. + * + * @return a KeyValue with this objects row key and the Put identifier. + */ + KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) { + return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value); + } /** * Compile the column family (i.e. schema) information @@ -57,7 +93,7 @@ public abstract class Mutation extends OperationWithAttributes implements Row { // ideally, we would also include table information, but that information // is not stored in each Operation instance. map.put("families", families); - for (Map.Entry> entry : this.familyMap.entrySet()) { + for (Map.Entry> entry : this.familyMap.entrySet()) { families.add(Bytes.toStringBinary(entry.getKey())); } return map; @@ -82,20 +118,21 @@ public abstract class Mutation extends OperationWithAttributes implements Row { map.put("row", Bytes.toStringBinary(this.row)); int colCount = 0; // iterate through all column families affected - for (Map.Entry> entry : this.familyMap.entrySet()) { - // map from this family to details for each kv affected within the family - List> qualifierDetails = - new ArrayList>(); + for (Map.Entry> entry : this.familyMap.entrySet()) { + // map from this family to details for each cell affected within the family + List> qualifierDetails = new ArrayList>(); columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails); colCount += entry.getValue().size(); if (maxCols <= 0) { continue; } - // add details for each kv - for (KeyValue kv : entry.getValue()) { + // add details for each cell + for (Cell cell: entry.getValue()) { if (--maxCols <= 0 ) { continue; } + // KeyValue v1 expectation. Cast for now until we go all Cell all the time. + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); Map kvMap = kv.toStringMap(); // row and family information are already available in the bigger map kvMap.remove("row"); @@ -131,14 +168,16 @@ public abstract class Mutation extends OperationWithAttributes implements Row { * Method for retrieving the put's familyMap * @return familyMap */ - public Map> getFamilyMap() { + public NavigableMap> getFamilyMap() { return this.familyMap; } /** * Method for setting the put's familyMap */ - public void setFamilyMap(Map> map) { + public void setFamilyMap(NavigableMap> map) { + // TODO: Shut this down or move it up to be a Constructor. Get new object rather than change + // this internal data member. this.familyMap = map; } @@ -199,8 +238,8 @@ public abstract class Mutation extends OperationWithAttributes implements Row { */ public int size() { int size = 0; - for(List kvList : this.familyMap.values()) { - size += kvList.size(); + for (List cells : this.familyMap.values()) { + size += cells.size(); } return size; } @@ -211,4 +250,4 @@ public abstract class Mutation extends OperationWithAttributes implements Row { public int numFamilies() { return familyMap.size(); } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Operation.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Operation.java index 07e9c19..d0fa3ef 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Operation.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Operation.java @@ -34,6 +34,7 @@ import org.codehaus.jackson.map.ObjectMapper; @InterfaceStability.Evolving public abstract class Operation { // TODO make this configurable + // TODO Do we need this anymore now we have protobuffed it all? private static final int DEFAULT_MAX_COLS = 5; /** @@ -109,5 +110,4 @@ public abstract class Operation { public String toString() { return toString(DEFAULT_MAX_COLS); } -} - +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Put.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Put.java index 805f53c..0d9d542 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Put.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Put.java @@ -19,20 +19,22 @@ package org.apache.hadoop.hbase.client; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; +import org.apache.hbase.Cell; /** * Used to perform Put operations for a single row. @@ -77,10 +79,8 @@ public class Put extends Mutation implements HeapSize, Comparable { */ public Put(Put putToCopy) { this(putToCopy.getRow(), putToCopy.ts); - this.familyMap = - new TreeMap>(Bytes.BYTES_COMPARATOR); - for(Map.Entry> entry : - putToCopy.getFamilyMap().entrySet()) { + this.familyMap = new TreeMap>(Bytes.BYTES_COMPARATOR); + for(Map.Entry> entry: putToCopy.getFamilyMap().entrySet()) { this.familyMap.put(entry.getKey(), entry.getValue()); } this.writeToWAL = putToCopy.writeToWAL; @@ -107,7 +107,7 @@ public class Put extends Mutation implements HeapSize, Comparable { * @return this */ public Put add(byte [] family, byte [] qualifier, long ts, byte [] value) { - List list = getKeyValueList(family); + List list = getCellList(family); KeyValue kv = createPutKeyValue(family, qualifier, ts, value); list.add(kv); familyMap.put(kv.getFamily(), list); @@ -124,7 +124,7 @@ public class Put extends Mutation implements HeapSize, Comparable { */ public Put add(KeyValue kv) throws IOException{ byte [] family = kv.getFamily(); - List list = getKeyValueList(family); + List list = getCellList(family); //Checking that the row of the kv is the same as the put int res = Bytes.compareTo(this.row, 0, row.length, kv.getBuffer(), kv.getRowOffset(), kv.getRowLength()); @@ -139,17 +139,6 @@ public class Put extends Mutation implements HeapSize, Comparable { return this; } - /* - * Create a KeyValue with this objects row key and the Put identifier. - * - * @return a KeyValue with this objects row key and the Put identifier. - */ - private KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, - byte[] value) { - return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, - value); - } - /** * A convenience method to determine if this object's familyMap contains * a value assigned to the given family & qualifier. @@ -226,7 +215,7 @@ public class Put extends Mutation implements HeapSize, Comparable { */ private boolean has(byte[] family, byte[] qualifier, long ts, byte[] value, boolean ignoreTS, boolean ignoreValue) { - List list = getKeyValueList(family); + List list = getCellList(family); if (list.size() == 0) { return false; } @@ -236,7 +225,8 @@ public class Put extends Mutation implements HeapSize, Comparable { // F T => 2 // F F => 1 if (!ignoreTS && !ignoreValue) { - for (KeyValue kv : list) { + for (Cell cell : list) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); if (Arrays.equals(kv.getFamily(), family) && Arrays.equals(kv.getQualifier(), qualifier) && Arrays.equals(kv.getValue(), value) && @@ -245,21 +235,24 @@ public class Put extends Mutation implements HeapSize, Comparable { } } } else if (ignoreValue && !ignoreTS) { - for (KeyValue kv : list) { + for (Cell cell : list) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); if (Arrays.equals(kv.getFamily(), family) && Arrays.equals(kv.getQualifier(), qualifier) && kv.getTimestamp() == ts) { return true; } } } else if (!ignoreValue && ignoreTS) { - for (KeyValue kv : list) { + for (Cell cell : list) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); if (Arrays.equals(kv.getFamily(), family) && Arrays.equals(kv.getQualifier(), qualifier) && Arrays.equals(kv.getValue(), value)) { return true; } } } else { - for (KeyValue kv : list) { + for (Cell cell : list) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); if (Arrays.equals(kv.getFamily(), family) && Arrays.equals(kv.getQualifier(), qualifier)) { return true; @@ -279,7 +272,8 @@ public class Put extends Mutation implements HeapSize, Comparable { */ public List get(byte[] family, byte[] qualifier) { List filteredList = new ArrayList(); - for (KeyValue kv: getKeyValueList(family)) { + for (Cell cell: getCellList(family)) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); if (Arrays.equals(kv.getQualifier(), qualifier)) { filteredList.add(kv); } @@ -287,21 +281,6 @@ public class Put extends Mutation implements HeapSize, Comparable { return filteredList; } - /** - * Creates an empty list if one doesnt exist for the given column family - * or else it returns the associated list of KeyValue objects. - * - * @param family column family - * @return a list of KeyValue objects, returns an empty list if one doesnt exist. - */ - private List getKeyValueList(byte[] family) { - List list = familyMap.get(family); - if(list == null) { - list = new ArrayList(0); - } - return list; - } - //HeapSize public long heapSize() { long heapsize = OVERHEAD; @@ -311,7 +290,7 @@ public class Put extends Mutation implements HeapSize, Comparable { //Adding map overhead heapsize += ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY); - for(Map.Entry> entry : this.familyMap.entrySet()) { + for(Map.Entry> entry : this.familyMap.entrySet()) { //Adding key overhead heapsize += ClassSize.align(ClassSize.ARRAY + entry.getKey().length); @@ -324,7 +303,8 @@ public class Put extends Mutation implements HeapSize, Comparable { heapsize += ClassSize.align(ClassSize.ARRAY + size * ClassSize.REFERENCE); - for(KeyValue kv : entry.getValue()) { + for(Cell cell : entry.getValue()) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); heapsize += kv.heapSize(); } } @@ -332,4 +312,4 @@ public class Put extends Mutation implements HeapSize, Comparable { return ClassSize.align((int)heapsize); } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Result.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Result.java index 9e15bbb..308392a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Result.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Result.java @@ -34,11 +34,15 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.SplitKeyValue; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hbase.Cell; +import org.apache.hbase.CellScannable; +import org.apache.hbase.CellScanner; +import org.apache.hbase.CellUtil; /** * Single row result of a {@link Get} or {@link Scan} query.

* - * This class is NOT THREAD SAFE.

+ * This class is NOT THREAD SAFE.

* * Convenience methods are available that return various {@link Map} * structures and values directly.

@@ -67,7 +71,7 @@ import org.apache.hadoop.hbase.util.Bytes; */ @InterfaceAudience.Public @InterfaceStability.Stable -public class Result { +public class Result implements CellScannable { private KeyValue [] kvs; // We're not using java serialization. Transient here is just a marker to say // that this is where we cache row if we're ever asked for it. @@ -78,6 +82,7 @@ public class Result { // never use directly private static byte [] buffer = null; private static final int PAD_WIDTH = 128; + public static final Result EMPTY_RESULT = new Result(); /** * Creates an empty Result w/ no KeyValue payload; returns null if you call {@link #raw()}. @@ -105,7 +110,8 @@ public class Result { * are already sorted * @param kvs List of KeyValues */ - public Result(List kvs) { + public Result(List kvs) { + // TODO: Here we presume the passed in Cells are KVs. One day this won't always be so. this(kvs.toArray(new KeyValue[kvs.size()])); } @@ -706,4 +712,9 @@ public class Result { this.familyMap = null; this.kvs = other.kvs; } + + @Override + public CellScanner cellScanner() { + return CellUtil.createCellScanner(this.kvs); + } } \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Row.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Row.java index 63dca08..06fe5e4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Row.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/Row.java @@ -31,4 +31,4 @@ public interface Row extends Comparable { * @return The row. */ public byte [] getRow(); -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/RowMutations.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/RowMutations.java index 8a6e5a7..2ee29ff 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/RowMutations.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/RowMutations.java @@ -38,9 +38,8 @@ import org.apache.hadoop.hbase.util.Bytes; @InterfaceAudience.Public @InterfaceStability.Evolving public class RowMutations implements Row { - private List mutations = new ArrayList(); + private final List mutations = new ArrayList(); private byte [] row; - private static final byte VERSION = (byte)0; /** Constructor for Writable. DO NOT USE */ public RowMutations() {} @@ -100,4 +99,4 @@ public class RowMutations implements Row { public List getMutations() { return Collections.unmodifiableList(mutations); } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java index 85be8e6..1af0d9b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java @@ -43,6 +43,7 @@ import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.net.DNS; import com.google.protobuf.ServiceException; +import com.google.protobuf.TextFormat; /** * Retries scanner operations such as create, next, etc. @@ -55,7 +56,7 @@ public class ScannerCallable extends ServerCallable { = "hbase.client.log.scanner.latency.cutoff"; public static final String LOG_SCANNER_ACTIVITY = "hbase.client.log.scanner.activity"; - private static final Log LOG = LogFactory.getLog(ScannerCallable.class); + public static final Log LOG = LogFactory.getLog(ScannerCallable.class); private long scannerId = -1L; private boolean instantiated = false; private boolean closed = false; @@ -136,9 +137,10 @@ public class ScannerCallable extends ServerCallable { this.scannerId = openScanner(); } else { Result [] rrs = null; + ScanRequest request = null; try { incRPCcallsMetrics(); - ScanRequest request = + request = RequestConverter.buildScanRequest(scannerId, caching, false, nextCallSeq); ScanResponse response = null; try { @@ -175,8 +177,7 @@ public class ScannerCallable extends ServerCallable { updateResultsMetrics(response); } catch (IOException e) { if (logScannerActivity) { - LOG.info("Got exception in fetching from scanner=" - + scannerId, e); + LOG.info("Got exception making request " + TextFormat.shortDebugString(request), e); } IOException ioe = e; if (e instanceof RemoteException) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java index bc2a208..798da62 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java @@ -246,4 +246,4 @@ public abstract class ServerCallable implements Callable { } return t; } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java index 3a7000f..235b49f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java @@ -25,10 +25,12 @@ import java.util.TreeSet; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.util.StringUtils; +import org.apache.hbase.Cell; /** * Emits sorted Puts. @@ -61,8 +63,9 @@ public class PutSortReducer extends // stop at the end or the RAM threshold while (iter.hasNext() && curSize < threshold) { Put p = iter.next(); - for (List kvs : p.getFamilyMap().values()) { - for (KeyValue kv : kvs) { + for (List cells: p.getFamilyMap().values()) { + for (Cell cell: cells) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); map.add(kv); curSize += kv.getLength(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index ae3be09..3d3b4ac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -32,7 +32,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.NavigableMap; import java.util.NavigableSet; import java.util.TreeMap; @@ -42,6 +41,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.MasterAdminProtocol; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.Action; @@ -757,16 +757,15 @@ public final class ProtobufUtil { } ColumnValue.Builder columnBuilder = ColumnValue.newBuilder(); QualifierValue.Builder valueBuilder = QualifierValue.newBuilder(); - for (Map.Entry> - family: increment.getFamilyMap().entrySet()) { + for (Map.Entry> family: increment.getFamilyMap().entrySet()) { columnBuilder.setFamily(ByteString.copyFrom(family.getKey())); columnBuilder.clearQualifierValue(); - NavigableMap values = family.getValue(); + List values = family.getValue(); if (values != null && values.size() > 0) { - for (Map.Entry value: values.entrySet()) { - valueBuilder.setQualifier(ByteString.copyFrom(value.getKey())); - valueBuilder.setValue(ByteString.copyFrom( - Bytes.toBytes(value.getValue().longValue()))); + for (Cell cell: values) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); + valueBuilder.setQualifier(ByteString.copyFrom(kv.getQualifier())); + valueBuilder.setValue(ByteString.copyFrom(kv.getValue())); columnBuilder.addQualifierValue(valueBuilder.build()); } } @@ -801,16 +800,17 @@ public final class ProtobufUtil { } ColumnValue.Builder columnBuilder = ColumnValue.newBuilder(); QualifierValue.Builder valueBuilder = QualifierValue.newBuilder(); - for (Map.Entry> + for (Map.Entry> family: mutation.getFamilyMap().entrySet()) { columnBuilder.setFamily(ByteString.copyFrom(family.getKey())); columnBuilder.clearQualifierValue(); - for (KeyValue value: family.getValue()) { - valueBuilder.setQualifier(ByteString.copyFrom(value.getQualifier())); - valueBuilder.setValue(ByteString.copyFrom(value.getValue())); - valueBuilder.setTimestamp(value.getTimestamp()); + for (Cell cell: family.getValue()) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); + valueBuilder.setQualifier(ByteString.copyFrom(kv.getQualifier())); + valueBuilder.setValue(ByteString.copyFrom(kv.getValue())); + valueBuilder.setTimestamp(kv.getTimestamp()); if (mutateType == MutateType.DELETE) { - KeyValue.Type keyValueType = KeyValue.Type.codeToType(value.getType()); + KeyValue.Type keyValueType = KeyValue.Type.codeToType(kv.getType()); valueBuilder.setDeleteType(toDeleteType(keyValueType)); } columnBuilder.addQualifierValue(valueBuilder.build()); @@ -1832,8 +1832,8 @@ public final class ProtobufUtil { * @throws IOException */ @SuppressWarnings("unchecked") - public static - T getParsedGenericInstance(Class runtimeClass, int position, ByteString b) + public static + T getParsedGenericInstance(Class runtimeClass, int position, ByteString b) throws IOException { Type type = runtimeClass.getGenericSuperclass(); Type argType = ((ParameterizedType)type).getActualTypeArguments()[position]; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 5ae8bc1..d2be9d7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -77,6 +77,7 @@ import org.apache.hadoop.hbase.FailedSanityCheckException; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.HConstants.OperationStatusCode; import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.HRegionInfo; @@ -134,6 +135,7 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.io.MultipleIOException; import org.apache.hadoop.util.StringUtils; +import org.apache.hbase.Cell; import org.cliffc.high_scale_lib.Counter; import com.google.common.base.Preconditions; @@ -1821,7 +1823,7 @@ public class HRegion implements HeapSize { // , Writable{ * @param writeToWAL * @throws IOException */ - void delete(Map> familyMap, UUID clusterId, + void delete(NavigableMap> familyMap, UUID clusterId, boolean writeToWAL) throws IOException { Delete delete = new Delete(HConstants.EMPTY_BYTE_ARRAY); delete.setFamilyMap(familyMap); @@ -1837,15 +1839,16 @@ public class HRegion implements HeapSize { // , Writable{ * @param byteNow * @throws IOException */ - void prepareDeleteTimestamps(Map> familyMap, byte[] byteNow) + void prepareDeleteTimestamps(Map> familyMap, byte[] byteNow) throws IOException { - for (Map.Entry> e : familyMap.entrySet()) { + for (Map.Entry> e : familyMap.entrySet()) { byte[] family = e.getKey(); - List kvs = e.getValue(); + List cells = e.getValue(); Map kvCount = new TreeMap(Bytes.BYTES_COMPARATOR); - for (KeyValue kv: kvs) { + for (Cell cell: cells) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); // Check if time is LATEST, change to time of most recent addition if so // This is expensive. if (kv.isLatestTimestamp() && kv.isDeleteType()) { @@ -2059,7 +2062,7 @@ public class HRegion implements HeapSize { // , Writable{ /** Keep track of the locks we hold so we can release them in finally clause */ List acquiredLocks = Lists.newArrayListWithCapacity(batchOp.operations.length); // reference family maps directly so coprocessors can mutate them if desired - Map>[] familyMaps = new Map[batchOp.operations.length]; + Map>[] familyMaps = new Map[batchOp.operations.length]; // We try to set up a batch in the range [firstIndex,lastIndexExclusive) int firstIndex = batchOp.nextIndexToProcess; int lastIndexExclusive = firstIndex; @@ -2078,7 +2081,7 @@ public class HRegion implements HeapSize { // , Writable{ boolean isPutMutation = mutation instanceof Put; Integer providedLockId = nextPair.getSecond(); - Map> familyMap = mutation.getFamilyMap(); + Map> familyMap = mutation.getFamilyMap(); // store the family map reference to allow for mutations familyMaps[lastIndexExclusive] = familyMap; @@ -2515,15 +2518,14 @@ public class HRegion implements HeapSize { // , Writable{ } /** - * Replaces any KV timestamps set to {@link HConstants#LATEST_TIMESTAMP} with the provided current - * timestamp. + * Replaces any KV timestamps set to {@link HConstants#LATEST_TIMESTAMP} with the + * provided current timestamp. */ - void updateKVTimestamps( - final Iterable> keyLists, final byte[] now) { - for (List keys: keyLists) { - if (keys == null) continue; - for (KeyValue key : keys) { - key.updateLatestStamp(now); + void updateKVTimestamps(final Iterable> keyLists, final byte[] now) { + for (List cells: keyLists) { + if (cells == null) continue; + for (Cell key : cells) { + ((KeyValue)key).updateLatestStamp(now); } } } @@ -2611,10 +2613,10 @@ public class HRegion implements HeapSize { // , Writable{ * @praram now * @throws IOException */ - private void put(final byte [] row, byte [] family, List edits) + private void put(final byte [] row, byte [] family, List edits) throws IOException { - Map> familyMap; - familyMap = new HashMap>(); + NavigableMap> familyMap; + familyMap = new TreeMap>(Bytes.BYTES_COMPARATOR); familyMap.put(family, edits); Put p = new Put(row); @@ -2636,7 +2638,7 @@ public class HRegion implements HeapSize { // , Writable{ * @return the additional memory usage of the memstore caused by the * new entries. */ - private long applyFamilyMapToMemstore(Map> familyMap, + private long applyFamilyMapToMemstore(Map> familyMap, MultiVersionConsistencyControl.WriteEntry localizedWriteEntry) { long size = 0; boolean freemvcc = false; @@ -2647,12 +2649,13 @@ public class HRegion implements HeapSize { // , Writable{ freemvcc = true; } - for (Map.Entry> e : familyMap.entrySet()) { + for (Map.Entry> e : familyMap.entrySet()) { byte[] family = e.getKey(); - List edits = e.getValue(); + List cells = e.getValue(); Store store = getStore(family); - for (KeyValue kv: edits) { + for (Cell cell: cells) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); kv.setMemstoreTS(localizedWriteEntry.getWriteNumber()); size += store.add(kv); } @@ -2672,7 +2675,7 @@ public class HRegion implements HeapSize { // , Writable{ * the wal. This method is then invoked to rollback the memstore. */ private void rollbackMemstore(BatchOperationInProgress> batchOp, - Map>[] familyMaps, + Map>[] familyMaps, int start, int end) { int kvsRolledback = 0; for (int i = start; i < end; i++) { @@ -2683,17 +2686,17 @@ public class HRegion implements HeapSize { // , Writable{ } // Rollback all the kvs for this row. - Map> familyMap = familyMaps[i]; - for (Map.Entry> e : familyMap.entrySet()) { + Map> familyMap = familyMaps[i]; + for (Map.Entry> e : familyMap.entrySet()) { byte[] family = e.getKey(); - List edits = e.getValue(); + List cells = e.getValue(); // Remove those keys from the memstore that matches our // key's (row, cf, cq, timestamp, memstoreTS). The interesting part is // that even the memstoreTS has to match for keys that will be rolleded-back. Store store = getStore(family); - for (KeyValue kv: edits) { - store.rollback(kv); + for (Cell cell: cells) { + store.rollback(KeyValueUtil.ensureKeyValue(cell)); kvsRolledback++; } } @@ -2713,18 +2716,18 @@ public class HRegion implements HeapSize { // , Writable{ } } - void checkTimestamps(final Map> familyMap, + void checkTimestamps(final Map> familyMap, long now) throws FailedSanityCheckException { if (timestampSlop == HConstants.LATEST_TIMESTAMP) { return; } long maxTs = now + timestampSlop; - for (List kvs : familyMap.values()) { - for (KeyValue kv : kvs) { + for (List kvs : familyMap.values()) { + for (Cell cell : kvs) { // see if the user-side TS is out of range. latest = server-side - if (!kv.isLatestTimestamp() && kv.getTimestamp() > maxTs) { + if (KeyValueUtil.ensureKeyValue(cell).isLatestTimestamp() && cell.getTimestamp() > maxTs) { throw new FailedSanityCheckException("Timestamp for KV out of range " - + kv + " (too.new=" + timestampSlop + ")"); + + cell + " (too.new=" + timestampSlop + ")"); } } } @@ -2736,11 +2739,10 @@ public class HRegion implements HeapSize { // , Writable{ * @param familyMap map of family->edits * @param walEdit the destination entry to append into */ - private void addFamilyMapToWALEdit(Map> familyMap, - WALEdit walEdit) { - for (List edits : familyMap.values()) { - for (KeyValue kv : edits) { - walEdit.add(kv); + private void addFamilyMapToWALEdit(Map> familyMap, WALEdit walEdit) { + for (List edits : familyMap.values()) { + for (Cell cell : edits) { + walEdit.add(KeyValueUtil.ensureKeyValue(cell)); } } } @@ -3445,7 +3447,7 @@ public class HRegion implements HeapSize { // , Writable{ public HRegionInfo getRegionInfo() { return regionInfo; } - + RegionScannerImpl(Scan scan, List additionalScanners, HRegion region) throws IOException { // DebugPrint.println("HRegionScanner."); @@ -3595,11 +3597,11 @@ public class HRegion implements HeapSize { // , Writable{ return next(outResults, batch, metric); } - private void populateFromJoinedHeap(List results, int limit, String metric) + private void populateFromJoinedHeap(List results, int limit, String metric) throws IOException { assert joinedContinuationRow != null; - KeyValue kv = populateResult(results, this.joinedHeap, limit, - joinedContinuationRow.getBuffer(), joinedContinuationRow.getRowOffset(), + KeyValue kv = populateResult(results, this.joinedHeap, limit, + joinedContinuationRow.getBuffer(), joinedContinuationRow.getRowOffset(), joinedContinuationRow.getRowLength(), metric); if (kv != KV_LIMIT) { // We are done with this row, reset the continuation. @@ -3621,7 +3623,7 @@ public class HRegion implements HeapSize { // , Writable{ * @param metric Metric key to be passed into KeyValueHeap::next(). * @return KV_LIMIT if limit reached, next KeyValue otherwise. */ - private KeyValue populateResult(List results, KeyValueHeap heap, int limit, + private KeyValue populateResult(List results, KeyValueHeap heap, int limit, byte[] currentRow, int offset, short length, String metric) throws IOException { KeyValue nextKv; do { @@ -4209,15 +4211,15 @@ public class HRegion implements HeapSize { // , Writable{ // The row key is the region name byte[] row = r.getRegionName(); final long now = EnvironmentEdgeManager.currentTimeMillis(); - final List edits = new ArrayList(2); - edits.add(new KeyValue(row, HConstants.CATALOG_FAMILY, + final List cells = new ArrayList(2); + cells.add(new KeyValue(row, HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, now, r.getRegionInfo().toByteArray())); // Set into the root table the version of the meta table. - edits.add(new KeyValue(row, HConstants.CATALOG_FAMILY, + cells.add(new KeyValue(row, HConstants.CATALOG_FAMILY, HConstants.META_VERSION_QUALIFIER, now, Bytes.toBytes(HConstants.META_VERSION))); - meta.put(row, HConstants.CATALOG_FAMILY, edits); + meta.put(row, HConstants.CATALOG_FAMILY, cells); } /** @@ -4792,8 +4794,8 @@ public class HRegion implements HeapSize { // , Writable{ checkRow(row, "append"); boolean flush = false; WALEdit walEdits = null; - List allKVs = new ArrayList(append.size()); - Map> tempMemstore = new HashMap>(); + List allKVs = new ArrayList(append.size()); + Map> tempMemstore = new HashMap>(); long size = 0; long txid = 0; @@ -4814,15 +4816,15 @@ public class HRegion implements HeapSize { // , Writable{ try { long now = EnvironmentEdgeManager.currentTimeMillis(); // Process each family - for (Map.Entry> family : append.getFamilyMap() - .entrySet()) { + for (Map.Entry> family : append.getFamilyMap().entrySet()) { Store store = stores.get(family.getKey()); - List kvs = new ArrayList(family.getValue().size()); + List kvs = new ArrayList(family.getValue().size()); // Get previous values for all columns in this family Get get = new Get(row); - for (KeyValue kv : family.getValue()) { + for (Cell cell : family.getValue()) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); get.addColumn(family.getKey(), kv.getQualifier()); } List results = get(get, false); @@ -4834,7 +4836,8 @@ public class HRegion implements HeapSize { // , Writable{ // once. // Would be nice if KeyValue had scatter/gather logic int idx = 0; - for (KeyValue kv : family.getValue()) { + for (Cell cell : family.getValue()) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); KeyValue newKV; if (idx < results.size() && results.get(idx).matchingQualifier(kv.getBuffer(), @@ -4901,14 +4904,15 @@ public class HRegion implements HeapSize { // , Writable{ } //Actually write to Memstore now - for (Map.Entry> entry : tempMemstore.entrySet()) { + for (Map.Entry> entry : tempMemstore.entrySet()) { Store store = entry.getKey(); if (store.getFamily().getMaxVersions() == 1) { // upsert if VERSIONS for this CF == 1 size += store.upsert(entry.getValue(), getSmallestReadPoint()); } else { // otherwise keep older versions around - for (KeyValue kv : entry.getValue()) { + for (Cell cell: entry.getValue()) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); size += store.add(kv); } } @@ -4957,8 +4961,8 @@ public class HRegion implements HeapSize { // , Writable{ TimeRange tr = increment.getTimeRange(); boolean flush = false; WALEdit walEdits = null; - List allKVs = new ArrayList(increment.numColumns()); - Map> tempMemstore = new HashMap>(); + List allKVs = new ArrayList(increment.size()); + Map> tempMemstore = new HashMap>(); long size = 0; long txid = 0; @@ -4979,16 +4983,16 @@ public class HRegion implements HeapSize { // , Writable{ try { long now = EnvironmentEdgeManager.currentTimeMillis(); // Process each family - for (Map.Entry> family : - increment.getFamilyMap().entrySet()) { + for (Map.Entry> family: increment.getFamilyMap().entrySet()) { Store store = stores.get(family.getKey()); - List kvs = new ArrayList(family.getValue().size()); + List kvs = new ArrayList(family.getValue().size()); // Get previous values for all columns in this family Get get = new Get(row); - for (Map.Entry column : family.getValue().entrySet()) { - get.addColumn(family.getKey(), column.getKey()); + for (Cell cell: family.getValue()) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); + get.addColumn(family.getKey(), kv.getQualifier()); } get.setTimeRange(tr.getMin(), tr.getMax()); List results = get(get, false); @@ -4996,11 +5000,12 @@ public class HRegion implements HeapSize { // , Writable{ // Iterate the input columns and update existing values if they were // found, otherwise add new column initialized to the increment amount int idx = 0; - for (Map.Entry column : family.getValue().entrySet()) { - long amount = column.getValue(); - if (idx < results.size() && - results.get(idx).matchingQualifier(column.getKey())) { - KeyValue kv = results.get(idx); + for (Cell cell: family.getValue()) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); + long amount = Bytes.toLong(kv.getValue()); + byte [] qualifier = kv.getQualifier(); + if (idx < results.size() && results.get(idx).matchingQualifier(qualifier)) { + kv = results.get(idx); if(kv.getValueLength() == Bytes.SIZEOF_LONG) { amount += Bytes.toLong(kv.getBuffer(), kv.getValueOffset(), Bytes.SIZEOF_LONG); } else { @@ -5012,8 +5017,8 @@ public class HRegion implements HeapSize { // , Writable{ } // Append new incremented KeyValue to list - KeyValue newKV = new KeyValue(row, family.getKey(), column.getKey(), - now, Bytes.toBytes(amount)); + KeyValue newKV = + new KeyValue(row, family.getKey(), qualifier, now, Bytes.toBytes(amount)); newKV.setMemstoreTS(w.getWriteNumber()); kvs.add(newKV); @@ -5041,14 +5046,15 @@ public class HRegion implements HeapSize { // , Writable{ } //Actually write to Memstore now - for (Map.Entry> entry : tempMemstore.entrySet()) { + for (Map.Entry> entry : tempMemstore.entrySet()) { Store store = entry.getKey(); if (store.getFamily().getMaxVersions() == 1) { // upsert if VERSIONS for this CF == 1 size += store.upsert(entry.getValue(), getSmallestReadPoint()); } else { // otherwise keep older versions around - for (KeyValue kv : entry.getValue()) { + for (Cell cell : entry.getValue()) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); size += store.add(kv); } } @@ -5439,7 +5445,7 @@ public class HRegion implements HeapSize { // , Writable{ * Update counters for numer of puts without wal and the size of possible data loss. * These information are exposed by the region server metrics. */ - private void recordPutWithoutWal(final Map> familyMap) { + private void recordPutWithoutWal(final Map> familyMap) { numPutsWithoutWAL.increment(); if (numPutsWithoutWAL.get() <= 1) { LOG.info("writing data to region " + this + @@ -5447,8 +5453,9 @@ public class HRegion implements HeapSize { // , Writable{ } long putSize = 0; - for (List edits : familyMap.values()) { - for (KeyValue kv : edits) { + for (List cells: familyMap.values()) { + for (Cell cell : cells) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); putSize += kv.getKeyLength() + kv.getValueLength(); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index b3a2cfd..5a42377 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -50,7 +50,6 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.RemoteExceptionHandler; import org.apache.hadoop.hbase.backup.HFileArchiver; import org.apache.hadoop.hbase.client.Scan; @@ -78,6 +77,7 @@ import org.apache.hadoop.hbase.util.CollectionBackedScanner; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.util.StringUtils; +import org.apache.hbase.Cell; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableCollection; @@ -1820,10 +1820,10 @@ public class HStore implements Store { } @Override - public long upsert(Iterable kvs, long readpoint) throws IOException { + public long upsert(Iterable cells, long readpoint) throws IOException { this.lock.readLock().lock(); try { - return this.memstore.upsert(kvs, readpoint); + return this.memstore.upsert(cells, readpoint); } finally { this.lock.readLock().unlock(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java index b7b8df4..26e4731 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java @@ -22,7 +22,7 @@ package org.apache.hadoop.hbase.regionserver; import java.lang.management.ManagementFactory; import java.lang.management.RuntimeMXBean; import java.rmi.UnexpectedException; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -38,11 +38,13 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.regionserver.MemStoreLAB.Allocation; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; +import org.apache.hbase.Cell; /** * The MemStore holds in-memory modifications to the Store. Modifications @@ -498,9 +500,9 @@ public class MemStore implements HeapSize { // create or update (upsert) a new KeyValue with // 'now' and a 0 memstoreTS == immediately visible - return upsert(Arrays.asList( - new KeyValue(row, family, qualifier, now, Bytes.toBytes(newValue))), 1L - ); + List cells = new ArrayList(1); + cells.add(new KeyValue(row, family, qualifier, now, Bytes.toBytes(newValue))); + return upsert(cells, 1L); } finally { this.lock.readLock().unlock(); } @@ -520,16 +522,16 @@ public class MemStore implements HeapSize { * This is called under row lock, so Get operations will still see updates * atomically. Scans will only see each KeyValue update as atomic. * - * @param kvs + * @param cells * @param readpoint readpoint below which we can safely remove duplicate KVs * @return change in memstore size */ - public long upsert(Iterable kvs, long readpoint) { + public long upsert(Iterable cells, long readpoint) { this.lock.readLock().lock(); try { long size = 0; - for (KeyValue kv : kvs) { - size += upsert(kv, readpoint); + for (Cell cell : cells) { + size += upsert(cell, readpoint); } return size; } finally { @@ -548,16 +550,17 @@ public class MemStore implements HeapSize { *

* Callers must hold the read lock. * - * @param kv + * @param cell * @return change in size of MemStore */ - private long upsert(KeyValue kv, long readpoint) { + private long upsert(Cell cell, long readpoint) { // Add the KeyValue to the MemStore // Use the internalAdd method here since we (a) already have a lock // and (b) cannot safely use the MSLAB here without potentially // hitting OOME - see TestMemStore.testUpsertMSLAB for a // test that triggers the pathological case if we don't avoid MSLAB // here. + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); long addedSize = internalAdd(kv); // Get the KeyValues for the row/family/qualifier regardless of timestamp. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MultiRowMutationProcessor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MultiRowMutationProcessor.java index 4bea011..4af1bc5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MultiRowMutationProcessor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MultiRowMutationProcessor.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; @@ -31,6 +32,7 @@ import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProcessorProto import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProcessorProtos.MultiRowMutationProcessorResponse; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hbase.Cell; /** * A MultiRowProcessor that performs multiple puts and deletes. @@ -70,7 +72,7 @@ MultiRowMutationProcessorResponse> { // Check mutations and apply edits to a single WALEdit for (Mutation m : mutations) { if (m instanceof Put) { - Map> familyMap = m.getFamilyMap(); + Map> familyMap = m.getFamilyMap(); region.checkFamilies(familyMap.keySet()); region.checkTimestamps(familyMap, now); region.updateKVTimestamps(familyMap.values(), byteNow); @@ -83,9 +85,10 @@ MultiRowMutationProcessorResponse> { "Action must be Put or Delete. But was: " + m.getClass().getName()); } - for (List edits : m.getFamilyMap().values()) { + for (List cells: m.getFamilyMap().values()) { boolean writeToWAL = m.getWriteToWAL(); - for (KeyValue kv : edits) { + for (Cell cell : cells) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); mutationKvs.add(kv); if (writeToWAL) { walEdit.add(kv); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java index d3a8edc..ee9cd9b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hbase.Cell; import com.google.common.collect.ImmutableList; @@ -102,12 +103,12 @@ public interface Store extends HeapSize, StoreConfigInformation { *

* This operation is atomic on each KeyValue (row/family/qualifier) but not necessarily atomic * across all of them. - * @param kvs + * @param cells * @param readpoint readpoint below which we can safely remove duplicate KVs * @return memstore size delta * @throws IOException */ - public long upsert(Iterable kvs, long readpoint) throws IOException; + public long upsert(Iterable cells, long readpoint) throws IOException; /** * Adds a value to the memstore diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java index 63914de..0e4e9ed 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; @@ -63,6 +64,7 @@ import org.apache.hadoop.hbase.rest.model.RowModel; import org.apache.hadoop.hbase.rest.model.ScannerModel; import org.apache.hadoop.hbase.rest.model.TableSchemaModel; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hbase.Cell; /** * HTable interface to remote tables accessed via REST gateway @@ -183,8 +185,9 @@ public class RemoteHTable implements HTableInterface { protected CellSetModel buildModelFromPut(Put put) { RowModel row = new RowModel(put.getRow()); long ts = put.getTimeStamp(); - for (List kvs: put.getFamilyMap().values()) { - for (KeyValue kv: kvs) { + for (List cells: put.getFamilyMap().values()) { + for (Cell cell: cells) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); row.addCell(new CellModel(kv.getFamily(), kv.getQualifier(), ts != HConstants.LATEST_TIMESTAMP ? ts : kv.getTimestamp(), kv.getValue())); @@ -388,25 +391,26 @@ public class RemoteHTable implements HTableInterface { // ignores the row specification in the URI // separate puts by row - TreeMap> map = - new TreeMap>(Bytes.BYTES_COMPARATOR); + TreeMap> map = + new TreeMap>(Bytes.BYTES_COMPARATOR); for (Put put: puts) { byte[] row = put.getRow(); - List kvs = map.get(row); - if (kvs == null) { - kvs = new ArrayList(); - map.put(row, kvs); + List cells = map.get(row); + if (cells == null) { + cells = new ArrayList(); + map.put(row, cells); } - for (List l: put.getFamilyMap().values()) { - kvs.addAll(l); + for (List l: put.getFamilyMap().values()) { + cells.addAll(l); } } // build the cell set CellSetModel model = new CellSetModel(); - for (Map.Entry> e: map.entrySet()) { + for (Map.Entry> e: map.entrySet()) { RowModel row = new RowModel(e.getKey()); - for (KeyValue kv: e.getValue()) { + for (Cell cell: e.getValue()) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); row.addCell(new CellModel(kv)); } model.addRow(row); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java index 84f6777..4f5d5bd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -37,6 +38,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.Delete; @@ -68,6 +70,7 @@ import org.apache.hadoop.hbase.security.access.Permission.Action; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.hbase.util.Pair; +import org.apache.hbase.Cell; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ListMultimap; @@ -153,11 +156,12 @@ public class AccessController extends BaseRegionObserver * table updates. */ void updateACL(RegionCoprocessorEnvironment e, - final Map> familyMap) { + final Map> familyMap) { Set tableSet = new TreeSet(Bytes.BYTES_COMPARATOR); - for (Map.Entry> f : familyMap.entrySet()) { - List kvs = f.getValue(); - for (KeyValue kv: kvs) { + for (Map.Entry> f : familyMap.entrySet()) { + List cells = f.getValue(); + for (Cell cell: cells) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); if (Bytes.equals(kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(), AccessControlLists.ACL_LIST_FAMILY, 0, AccessControlLists.ACL_LIST_FAMILY.length)) { @@ -963,9 +967,15 @@ public class AccessController extends BaseRegionObserver public Result preIncrement(final ObserverContext c, final Increment increment) throws IOException { + // Create a map of family to qualifiers. Map> familyMap = Maps.newHashMap(); - for (Map.Entry> entry : increment.getFamilyMap().entrySet()) { - familyMap.put(entry.getKey(), entry.getValue().keySet()); + for (Map.Entry> entry: increment.getFamilyMap().entrySet()) { + Set qualifiers = new HashSet(entry.getValue().size()); + for (Cell cell: entry.getValue()) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); + qualifiers.add(kv.getQualifier()); + } + familyMap.put(entry.getKey(), qualifiers); } requirePermission("increment", Permission.Action.WRITE, c.getEnvironment(), familyMap); return null; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java index 0188dab..417839d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.thrift2; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.thrift2.generated.*; @@ -37,13 +38,13 @@ public class ThriftUtilities { /** * Creates a {@link Get} (HBase) from a {@link TGet} (Thrift). - * + * * This ignores any timestamps set on {@link TColumn} objects. - * + * * @param in the TGet to convert - * + * * @return Get object - * + * * @throws IOException if an invalid time range or max version parameter is given */ public static Get getFromThrift(TGet in) throws IOException { @@ -77,11 +78,11 @@ public class ThriftUtilities { /** * Converts multiple {@link TGet}s (Thrift) into a list of {@link Get}s (HBase). - * + * * @param in list of TGets to convert - * + * * @return list of Get objects - * + * * @throws IOException if an invalid time range or max version parameter is given * @see #getFromThrift(TGet) */ @@ -95,9 +96,9 @@ public class ThriftUtilities { /** * Creates a {@link TResult} (Thrift) from a {@link Result} (HBase). - * + * * @param in the Result to convert - * + * * @return converted result, returns an empty result if the input is null */ public static TResult resultFromHBase(Result in) { @@ -122,11 +123,11 @@ public class ThriftUtilities { /** * Converts multiple {@link Result}s (HBase) into a list of {@link TResult}s (Thrift). - * + * * @param in array of Results to convert - * + * * @return list of converted TResults - * + * * @see #resultFromHBase(Result) */ public static List resultsFromHBase(Result[] in) { @@ -139,9 +140,9 @@ public class ThriftUtilities { /** * Creates a {@link Put} (HBase) from a {@link TPut} (Thrift) - * + * * @param in the TPut to convert - * + * * @return converted Put */ public static Put putFromThrift(TPut in) { @@ -169,11 +170,11 @@ public class ThriftUtilities { /** * Converts multiple {@link TPut}s (Thrift) into a list of {@link Put}s (HBase). - * + * * @param in list of TPuts to convert - * + * * @return list of converted Puts - * + * * @see #putFromThrift(TPut) */ public static List putsFromThrift(List in) { @@ -186,9 +187,9 @@ public class ThriftUtilities { /** * Creates a {@link Delete} (HBase) from a {@link TDelete} (Thrift). - * + * * @param in the TDelete to convert - * + * * @return converted Delete */ public static Delete deleteFromThrift(TDelete in) { @@ -233,11 +234,11 @@ public class ThriftUtilities { /** * Converts multiple {@link TDelete}s (Thrift) into a list of {@link Delete}s (HBase). - * + * * @param in list of TDeletes to convert - * + * * @return list of converted Deletes - * + * * @see #deleteFromThrift(TDelete) */ @@ -259,12 +260,14 @@ public class ThriftUtilities { } // Map> - for (Map.Entry> familyEntry : in.getFamilyMap().entrySet()) { + for (Map.Entry> familyEntry: + in.getFamilyMap().entrySet()) { TColumn column = new TColumn(ByteBuffer.wrap(familyEntry.getKey())); - for (KeyValue keyValue : familyEntry.getValue()) { - byte[] family = keyValue.getFamily(); - byte[] qualifier = keyValue.getQualifier(); - long timestamp = keyValue.getTimestamp(); + for (org.apache.hbase.Cell cell: familyEntry.getValue()) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); + byte[] family = kv.getFamily(); + byte[] qualifier = kv.getQualifier(); + long timestamp = kv.getTimestamp(); if (family != null) { column.setFamily(family); } @@ -272,7 +275,7 @@ public class ThriftUtilities { column.setQualifier(qualifier); } if (timestamp != HConstants.LATEST_TIMESTAMP) { - column.setTimestamp(keyValue.getTimestamp()); + column.setTimestamp(kv.getTimestamp()); } } columns.add(column); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java index a7efa22..8a41930 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java @@ -3635,7 +3635,8 @@ public class TestFromClientSide { assertEquals(put.size(), 1); assertEquals(put.getFamilyMap().get(CONTENTS_FAMILY).size(), 1); - KeyValue kv = put.getFamilyMap().get(CONTENTS_FAMILY).get(0); + // KeyValue v1 expectation. Cast for now until we go all Cell all the time. TODO + KeyValue kv = (KeyValue)put.getFamilyMap().get(CONTENTS_FAMILY).get(0); assertTrue(Bytes.equals(kv.getFamily(), CONTENTS_FAMILY)); // will it return null or an empty byte array? @@ -4159,7 +4160,7 @@ public class TestFromClientSide { mrmBuilder.addMutationRequest(m2); MultiMutateRequest mrm = mrmBuilder.build(); CoprocessorRpcChannel channel = t.coprocessorService(ROW); - MultiRowMutationService.BlockingInterface service = + MultiRowMutationService.BlockingInterface service = MultiRowMutationService.newBlockingStub(channel); service.mutateRows(null, mrm); Get g = new Get(ROW); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java index f262948..b4d07d8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; +import org.apache.hbase.Cell; /** * A sample region observer that tests the RegionObserver interface. @@ -310,27 +311,30 @@ public class SimpleRegionObserver extends BaseRegionObserver { public void prePut(final ObserverContext c, final Put put, final WALEdit edit, final boolean writeToWAL) throws IOException { - Map> familyMap = put.getFamilyMap(); + Map> familyMap = put.getFamilyMap(); RegionCoprocessorEnvironment e = c.getEnvironment(); assertNotNull(e); assertNotNull(e.getRegion()); assertNotNull(familyMap); if (Arrays.equals(e.getRegion().getTableDesc().getName(), TestRegionObserverInterface.TEST_TABLE)) { - List kvs = familyMap.get(TestRegionObserverInterface.A); - assertNotNull(kvs); - assertNotNull(kvs.get(0)); - assertTrue(Bytes.equals(kvs.get(0).getQualifier(), + List cells = familyMap.get(TestRegionObserverInterface.A); + assertNotNull(cells); + assertNotNull(cells.get(0)); + KeyValue kv = (KeyValue)cells.get(0); + assertTrue(Bytes.equals(kv.getQualifier(), TestRegionObserverInterface.A)); - kvs = familyMap.get(TestRegionObserverInterface.B); - assertNotNull(kvs); - assertNotNull(kvs.get(0)); - assertTrue(Bytes.equals(kvs.get(0).getQualifier(), + cells = familyMap.get(TestRegionObserverInterface.B); + assertNotNull(cells); + assertNotNull(cells.get(0)); + kv = (KeyValue)cells.get(0); + assertTrue(Bytes.equals(kv.getQualifier(), TestRegionObserverInterface.B)); - kvs = familyMap.get(TestRegionObserverInterface.C); - assertNotNull(kvs); - assertNotNull(kvs.get(0)); - assertTrue(Bytes.equals(kvs.get(0).getQualifier(), + cells = familyMap.get(TestRegionObserverInterface.C); + assertNotNull(cells); + assertNotNull(cells.get(0)); + kv = (KeyValue)cells.get(0); + assertTrue(Bytes.equals(kv.getQualifier(), TestRegionObserverInterface.C)); } hadPrePut = true; @@ -340,28 +344,31 @@ public class SimpleRegionObserver extends BaseRegionObserver { public void postPut(final ObserverContext c, final Put put, final WALEdit edit, final boolean writeToWAL) throws IOException { - Map> familyMap = put.getFamilyMap(); + Map> familyMap = put.getFamilyMap(); RegionCoprocessorEnvironment e = c.getEnvironment(); assertNotNull(e); assertNotNull(e.getRegion()); assertNotNull(familyMap); - List kvs = familyMap.get(TestRegionObserverInterface.A); + List cells = familyMap.get(TestRegionObserverInterface.A); if (Arrays.equals(e.getRegion().getTableDesc().getName(), TestRegionObserverInterface.TEST_TABLE)) { - assertNotNull(kvs); - assertNotNull(kvs.get(0)); - assertTrue(Bytes.equals(kvs.get(0).getQualifier(), - TestRegionObserverInterface.A)); - kvs = familyMap.get(TestRegionObserverInterface.B); - assertNotNull(kvs); - assertNotNull(kvs.get(0)); - assertTrue(Bytes.equals(kvs.get(0).getQualifier(), - TestRegionObserverInterface.B)); - kvs = familyMap.get(TestRegionObserverInterface.C); - assertNotNull(kvs); - assertNotNull(kvs.get(0)); - assertTrue(Bytes.equals(kvs.get(0).getQualifier(), - TestRegionObserverInterface.C)); + assertNotNull(cells); + assertNotNull(cells.get(0)); + // KeyValue v1 expectation. Cast for now until we go all Cell all the time. TODO + KeyValue kv = (KeyValue)cells.get(0); + assertTrue(Bytes.equals(kv.getQualifier(), TestRegionObserverInterface.A)); + cells = familyMap.get(TestRegionObserverInterface.B); + assertNotNull(cells); + assertNotNull(cells.get(0)); + // KeyValue v1 expectation. Cast for now until we go all Cell all the time. TODO + kv = (KeyValue)cells.get(0); + assertTrue(Bytes.equals(kv.getQualifier(), TestRegionObserverInterface.B)); + cells = familyMap.get(TestRegionObserverInterface.C); + assertNotNull(cells); + assertNotNull(cells.get(0)); + // KeyValue v1 expectation. Cast for now until we go all Cell all the time. TODO + kv = (KeyValue)cells.get(0); + assertTrue(Bytes.equals(kv.getQualifier(), TestRegionObserverInterface.C)); } hadPostPut = true; } @@ -370,7 +377,7 @@ public class SimpleRegionObserver extends BaseRegionObserver { public void preDelete(final ObserverContext c, final Delete delete, final WALEdit edit, final boolean writeToWAL) throws IOException { - Map> familyMap = delete.getFamilyMap(); + Map> familyMap = delete.getFamilyMap(); RegionCoprocessorEnvironment e = c.getEnvironment(); assertNotNull(e); assertNotNull(e.getRegion()); @@ -384,7 +391,7 @@ public class SimpleRegionObserver extends BaseRegionObserver { public void postDelete(final ObserverContext c, final Delete delete, final WALEdit edit, final boolean writeToWAL) throws IOException { - Map> familyMap = delete.getFamilyMap(); + Map> familyMap = delete.getFamilyMap(); RegionCoprocessorEnvironment e = c.getEnvironment(); assertNotNull(e); assertNotNull(e.getRegion()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java index ef4efe5..58a7af9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper; import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge; +import org.apache.hbase.Cell; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -206,12 +207,10 @@ public class TestRegionObserverBypass { public void prePut(final ObserverContext e, final Put put, final WALEdit edit, final boolean writeToWAL) throws IOException { - Map> familyMap = put.getFamilyMap(); + Map> familyMap = put.getFamilyMap(); if (familyMap.containsKey(test)) { e.bypass(); } } } - -} - +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java index 8ec79b2..0e57959 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdge; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hbase.Cell; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -52,8 +53,9 @@ import java.util.Map; import static org.junit.Assert.*; /** - * Tests invocation of the {@link org.apache.hadoop.hbase.coprocessor.MasterObserver} - * interface hooks at all appropriate times during normal HMaster operations. + * Tests invocation of the + * {@link org.apache.hadoop.hbase.coprocessor.MasterObserver} interface hooks at + * all appropriate times during normal HMaster operations. */ @Category(MediumTests.class) public class TestWALObserver { @@ -62,17 +64,11 @@ public class TestWALObserver { private static byte[] TEST_TABLE = Bytes.toBytes("observedTable"); private static byte[][] TEST_FAMILY = { Bytes.toBytes("fam1"), - Bytes.toBytes("fam2"), - Bytes.toBytes("fam3"), - }; + Bytes.toBytes("fam2"), Bytes.toBytes("fam3"), }; private static byte[][] TEST_QUALIFIER = { Bytes.toBytes("q1"), - Bytes.toBytes("q2"), - Bytes.toBytes("q3"), - }; + Bytes.toBytes("q2"), Bytes.toBytes("q3"), }; private static byte[][] TEST_VALUE = { Bytes.toBytes("v1"), - Bytes.toBytes("v2"), - Bytes.toBytes("v3"), - }; + Bytes.toBytes("v2"), Bytes.toBytes("v3"), }; private static byte[] TEST_ROW = Bytes.toBytes("testRow"); private Configuration conf; @@ -94,8 +90,8 @@ public class TestWALObserver { conf.setInt("dfs.client.block.recovery.retries", 2); TEST_UTIL.startMiniCluster(1); - Path hbaseRootDir = - TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase")); + Path hbaseRootDir = TEST_UTIL.getDFSCluster().getFileSystem() + .makeQualified(new Path("/hbase")); LOG.info("hbase.rootdir=" + hbaseRootDir); conf.set(HConstants.HBASE_DIR, hbaseRootDir.toString()); } @@ -108,11 +104,12 @@ public class TestWALObserver { @Before public void setUp() throws Exception { this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration()); - //this.cluster = TEST_UTIL.getDFSCluster(); + // this.cluster = TEST_UTIL.getDFSCluster(); this.fs = TEST_UTIL.getDFSCluster().getFileSystem(); this.hbaseRootDir = new Path(conf.get(HConstants.HBASE_DIR)); this.dir = new Path(this.hbaseRootDir, TestWALObserver.class.getName()); - this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME); + this.oldLogDir = new Path(this.hbaseRootDir, + HConstants.HREGION_OLDLOGDIR_NAME); this.logDir = new Path(this.hbaseRootDir, HConstants.HREGION_LOGDIR_NAME); this.logName = HConstants.HREGION_LOGDIR_NAME; @@ -127,21 +124,22 @@ public class TestWALObserver { } /** - * Test WAL write behavior with WALObserver. The coprocessor monitors - * a WALEdit written to WAL, and ignore, modify, and add KeyValue's for the + * Test WAL write behavior with WALObserver. The coprocessor monitors a + * WALEdit written to WAL, and ignore, modify, and add KeyValue's for the * WALEdit. */ @Test public void testWALObserverWriteToWAL() throws Exception { HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(TEST_TABLE)); - final HTableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE)); + final HTableDescriptor htd = createBasic3FamilyHTD(Bytes + .toString(TEST_TABLE)); Path basedir = new Path(this.hbaseRootDir, Bytes.toString(TEST_TABLE)); deleteDir(basedir); fs.mkdirs(new Path(basedir, hri.getEncodedName())); - HLog log = HLogFactory.createHLog(this.fs, hbaseRootDir, + HLog log = HLogFactory.createHLog(this.fs, hbaseRootDir, TestWALObserver.class.getName(), this.conf); SampleRegionWALObserver cp = getCoprocessor(log); @@ -149,8 +147,7 @@ public class TestWALObserver { // TEST_FAMILY[1] value shall be changed. // TEST_FAMILY[2] shall be added to WALEdit, although it's not in the put. cp.setTestValues(TEST_TABLE, TEST_ROW, TEST_FAMILY[0], TEST_QUALIFIER[0], - TEST_FAMILY[1], TEST_QUALIFIER[1], - TEST_FAMILY[2], TEST_QUALIFIER[2]); + TEST_FAMILY[1], TEST_QUALIFIER[1], TEST_FAMILY[2], TEST_QUALIFIER[2]); assertFalse(cp.isPreWALWriteCalled()); assertFalse(cp.isPostWALWriteCalled()); @@ -160,7 +157,7 @@ public class TestWALObserver { // Use a Put to create familyMap. Put p = creatPutWith2Families(TEST_ROW); - Map> familyMap = p.getFamilyMap(); + Map> familyMap = p.getFamilyMap(); WALEdit edit = new WALEdit(); addFamilyMapToWALEdit(familyMap, edit); @@ -224,9 +221,12 @@ public class TestWALObserver { // WAL replay is handled at HRegion::replayRecoveredEdits(), which is // ultimately called by HRegion::initialize() byte[] tableName = Bytes.toBytes("testWALCoprocessorReplay"); - final HTableDescriptor htd = getBasic3FamilyHTableDescriptor(Bytes.toString(tableName)); - //final HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(tableName)); - //final HRegionInfo hri1 = createBasic3FamilyHRegionInfo(Bytes.toString(tableName)); + final HTableDescriptor htd = getBasic3FamilyHTableDescriptor(Bytes + .toString(tableName)); + // final HRegionInfo hri = + // createBasic3FamilyHRegionInfo(Bytes.toString(tableName)); + // final HRegionInfo hri1 = + // createBasic3FamilyHRegionInfo(Bytes.toString(tableName)); final HRegionInfo hri = new HRegionInfo(tableName, null, null); final Path basedir = new Path(this.hbaseRootDir, Bytes.toString(tableName)); @@ -235,19 +235,19 @@ public class TestWALObserver { final Configuration newConf = HBaseConfiguration.create(this.conf); - //HLog wal = new HLog(this.fs, this.dir, this.oldLogDir, this.conf); + // HLog wal = new HLog(this.fs, this.dir, this.oldLogDir, this.conf); HLog wal = createWAL(this.conf); - //Put p = creatPutWith2Families(TEST_ROW); + // Put p = creatPutWith2Families(TEST_ROW); WALEdit edit = new WALEdit(); long now = EnvironmentEdgeManager.currentTimeMillis(); - //addFamilyMapToWALEdit(p.getFamilyMap(), edit); + // addFamilyMapToWALEdit(p.getFamilyMap(), edit); final int countPerFamily = 1000; - //for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) { - for (HColumnDescriptor hcd: htd.getFamilies()) { - //addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily, - //EnvironmentEdgeManager.getDelegate(), wal); + // for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) { + for (HColumnDescriptor hcd : htd.getFamilies()) { + // addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily, + // EnvironmentEdgeManager.getDelegate(), wal); addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily, - EnvironmentEdgeManager.getDelegate(), wal, htd); + EnvironmentEdgeManager.getDelegate(), wal, htd); } wal.append(hri, tableName, edit, now, htd); // sync to fs. @@ -281,32 +281,34 @@ public class TestWALObserver { } /** - * Test to see CP loaded successfully or not. There is a duplication - * at TestHLog, but the purpose of that one is to see whether the loaded - * CP will impact existing HLog tests or not. + * Test to see CP loaded successfully or not. There is a duplication at + * TestHLog, but the purpose of that one is to see whether the loaded CP will + * impact existing HLog tests or not. */ @Test public void testWALObserverLoaded() throws Exception { HLog log = HLogFactory.createHLog(fs, hbaseRootDir, - TestWALObserver.class.getName(), conf); + TestWALObserver.class.getName(), conf); assertNotNull(getCoprocessor(log)); } private SampleRegionWALObserver getCoprocessor(HLog wal) throws Exception { WALCoprocessorHost host = wal.getCoprocessorHost(); - Coprocessor c = host.findCoprocessor(SampleRegionWALObserver.class.getName()); - return (SampleRegionWALObserver)c; + Coprocessor c = host.findCoprocessor(SampleRegionWALObserver.class + .getName()); + return (SampleRegionWALObserver) c; } /* * Creates an HRI around an HTD that has tableName and three * column families named. + * * @param tableName Name of table to use when we create HTableDescriptor. */ private HRegionInfo createBasic3FamilyHRegionInfo(final String tableName) { HTableDescriptor htd = new HTableDescriptor(tableName); - for (int i = 0; i < TEST_FAMILY.length; i++ ) { + for (int i = 0; i < TEST_FAMILY.length; i++) { HColumnDescriptor a = new HColumnDescriptor(TEST_FAMILY[i]); htd.addFamily(a); } @@ -326,27 +328,30 @@ public class TestWALObserver { private Put creatPutWith2Families(byte[] row) throws IOException { Put p = new Put(row); - for (int i = 0; i < TEST_FAMILY.length-1; i++ ) { - p.add(TEST_FAMILY[i], TEST_QUALIFIER[i], - TEST_VALUE[i]); + for (int i = 0; i < TEST_FAMILY.length - 1; i++) { + p.add(TEST_FAMILY[i], TEST_QUALIFIER[i], TEST_VALUE[i]); } return p; } /** * Copied from HRegion. - * - * @param familyMap map of family->edits - * @param walEdit the destination entry to append into + * + * @param familyMap + * map of family->edits + * @param walEdit + * the destination entry to append into */ - private void addFamilyMapToWALEdit(Map> familyMap, + private void addFamilyMapToWALEdit(Map> familyMap, WALEdit walEdit) { - for (List edits : familyMap.values()) { - for (KeyValue kv : edits) { - walEdit.add(kv); + for (List edits : familyMap.values()) { + for (Cell cell : edits) { + // KeyValue v1 expectation. Cast for now until we go all Cell all the time. TODO. + walEdit.add((KeyValue)cell); } } } + private Path runWALSplit(final Configuration c) throws IOException { FileSystem fs = FileSystem.get(c); HLogSplitter logSplitter = HLogSplitter.createLogSplitter(c, @@ -359,28 +364,31 @@ public class TestWALObserver { LOG.info("Split file=" + splits.get(0)); return splits.get(0); } + private HLog createWAL(final Configuration c) throws IOException { return HLogFactory.createHLog(FileSystem.get(c), hbaseRootDir, logName, c); } - private void addWALEdits (final byte [] tableName, final HRegionInfo hri, - final byte [] rowName, final byte [] family, - final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd) - throws IOException { + + private void addWALEdits(final byte[] tableName, final HRegionInfo hri, + final byte[] rowName, final byte[] family, final int count, + EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd) + throws IOException { String familyStr = Bytes.toString(family); for (int j = 0; j < count; j++) { byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j)); byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j)); WALEdit edit = new WALEdit(); - edit.add(new KeyValue(rowName, family, qualifierBytes, - ee.currentTimeMillis(), columnBytes)); + edit.add(new KeyValue(rowName, family, qualifierBytes, ee + .currentTimeMillis(), columnBytes)); wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd); } } + private HTableDescriptor getBasic3FamilyHTableDescriptor( final String tableName) { HTableDescriptor htd = new HTableDescriptor(tableName); - for (int i = 0; i < TEST_FAMILY.length; i++ ) { + for (int i = 0; i < TEST_FAMILY.length; i++) { HColumnDescriptor a = new HColumnDescriptor(TEST_FAMILY[i]); htd.addFamily(a); } @@ -398,7 +406,4 @@ public class TestWALObserver { return htd; } - } - - diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 0cdd6d1..0e38534 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -26,6 +26,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NavigableMap; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -92,6 +93,7 @@ import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.PairOfSameType; import org.apache.hadoop.hbase.util.Threads; import org.apache.hbase.CellComparator; +import org.apache.hbase.Cell; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -1135,14 +1137,15 @@ public class TestHRegion extends HBaseTestCase { String method = this.getName(); this.region = initHRegion(tableName, method, conf, fam1, fam2, fam3); try { - List kvs = new ArrayList(); + List kvs = new ArrayList(); kvs.add(new KeyValue(row1, fam4, null, null)); //testing existing family byte [] family = fam2; try { - Map> deleteMap = new HashMap>(); + NavigableMap> deleteMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); deleteMap.put(family, kvs); region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true); } catch (Exception e) { @@ -1153,7 +1156,8 @@ public class TestHRegion extends HBaseTestCase { boolean ok = false; family = fam4; try { - Map> deleteMap = new HashMap>(); + NavigableMap> deleteMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); deleteMap.put(family, kvs); region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true); } catch (Exception e) { @@ -1475,12 +1479,13 @@ public class TestHRegion extends HBaseTestCase { this.region = initHRegion(tableName, method, conf, fam1); try { //Building checkerList - List kvs = new ArrayList(); + List kvs = new ArrayList(); kvs.add(new KeyValue(row1, fam1, col1, null)); kvs.add(new KeyValue(row1, fam1, col2, null)); kvs.add(new KeyValue(row1, fam1, col3, null)); - Map> deleteMap = new HashMap>(); + NavigableMap> deleteMap = + new TreeMap>(Bytes.BYTES_COMPARATOR); deleteMap.put(fam1, kvs); region.delete(deleteMap, HConstants.DEFAULT_CLUSTER_ID, true); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java index 478c6e5..b841031 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.regionserver.ScanInfo; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hbase.Cell; import com.google.common.base.Joiner; import com.google.common.collect.Iterables; @@ -795,17 +796,17 @@ public class TestMemStore extends TestCase { //////////////////////////////////// //Test for upsert with MSLAB //////////////////////////////////// - + /** * Test a pathological pattern that shows why we can't currently * use the MSLAB for upsert workloads. This test inserts data * in the following pattern: - * + * * - row0001 through row1000 (fills up one 2M Chunk) * - row0002 through row1001 (fills up another 2M chunk, leaves one reference * to the first chunk * - row0003 through row1002 (another chunk, another dangling reference) - * + * * This causes OOME pretty quickly if we use MSLAB for upsert * since each 2M chunk is held onto by a single reference. */ @@ -813,17 +814,17 @@ public class TestMemStore extends TestCase { Configuration conf = HBaseConfiguration.create(); conf.setBoolean(MemStore.USEMSLAB_KEY, true); memstore = new MemStore(conf, KeyValue.COMPARATOR); - + int ROW_SIZE = 2048; byte[] qualifier = new byte[ROW_SIZE - 4]; - + MemoryMXBean bean = ManagementFactory.getMemoryMXBean(); for (int i = 0; i < 3; i++) { System.gc(); } long usageBefore = bean.getHeapMemoryUsage().getUsed(); - + long size = 0; long ts=0; - + for (int newValue = 0; newValue < 1000; newValue++) { for (int row = newValue; row < newValue + 1000; row++) { byte[] rowBytes = Bytes.toBytes(row); @@ -834,10 +835,10 @@ public class TestMemStore extends TestCase { for (int i = 0; i < 3; i++) { System.gc(); } long usageAfter = bean.getHeapMemoryUsage().getUsed(); System.out.println("Memory used: " + (usageAfter - usageBefore) - + " (heapsize: " + memstore.heapSize() + + + " (heapsize: " + memstore.heapSize() + " size: " + size + ")"); } - + ////////////////////////////////////////////////////////////////////////////// // Helpers ////////////////////////////////////////////////////////////////////////////// @@ -855,19 +856,19 @@ public class TestMemStore extends TestCase { Configuration conf = HBaseConfiguration.create(); memstore = new MemStore(conf, KeyValue.COMPARATOR); long oldSize = memstore.size.get(); - - List l = new ArrayList(); + + List l = new ArrayList(); KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v"); KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v"); KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v"); kv1.setMvccVersion(1); kv2.setMvccVersion(1);kv3.setMvccVersion(1); l.add(kv1); l.add(kv2); l.add(kv3); - + this.memstore.upsert(l, 2);// readpoint is 2 long newSize = this.memstore.size.get(); assert(newSize > oldSize); - + KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v"); kv4.setMvccVersion(1); l.clear(); l.add(kv4); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java index 8dd9f15..f31ec10 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java @@ -18,37 +18,33 @@ package org.apache.hadoop.hbase.regionserver.wal; -import java.util.Map; +import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Random; -import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.classification.InterfaceAudience; - +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.regionserver.HRegion; -import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry; -import org.apache.hadoop.hbase.regionserver.wal.HLogFactory; -import org.apache.hadoop.hbase.regionserver.wal.HLogUtil; -import org.apache.hadoop.hbase.regionserver.wal.WALEdit; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hbase.Cell; /** * This class runs performance benchmarks for {@link HLog}. @@ -330,9 +326,10 @@ public final class HLogPerformanceEvaluation extends Configured implements Tool return put; } - private void addFamilyMapToWALEdit(Map> familyMap, WALEdit walEdit) { - for (List edits : familyMap.values()) { - for (KeyValue kv : edits) { + private void addFamilyMapToWALEdit(Map> familyMap, WALEdit walEdit) { + for (List edits : familyMap.values()) { + for (Cell cell : edits) { + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); walEdit.add(kv); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java index a35b315..43ae8ee 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.MediumTests; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HTable; @@ -55,13 +56,12 @@ import org.apache.hadoop.hbase.regionserver.ScanInfo; import org.apache.hadoop.hbase.regionserver.StoreScanner; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hbase.Cell; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -import static org.junit.Assert.*; - import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; @@ -217,11 +217,13 @@ public class TestCoprocessorScanPolicy { public void prePut(final ObserverContext c, final Put put, final WALEdit edit, final boolean writeToWAL) throws IOException { if (put.getAttribute("ttl") != null) { - KeyValue kv = put.getFamilyMap().values().iterator().next().get(0); + Cell cell = put.getFamilyMap().values().iterator().next().get(0); + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); ttls.put(Bytes.toString(kv.getQualifier()), Bytes.toLong(kv.getValue())); c.bypass(); } else if (put.getAttribute("versions") != null) { - KeyValue kv = put.getFamilyMap().values().iterator().next().get(0); + Cell cell = put.getFamilyMap().values().iterator().next().get(0); + KeyValue kv = KeyValueUtil.ensureKeyValue(cell); versions.put(Bytes.toString(kv.getQualifier()), Bytes.toInt(kv.getValue())); c.bypass(); }