From 8db03add12a4ca169a79e999775e76420559a0ba Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Wed, 7 Aug 2013 16:30:43 -0700 Subject: [PATCH] HBASE-8693 Extensible data types API This patch introduces an extensible data types API for HBase. It is inspired by the following systems: - PostgreSQL. Postgres has a user-extensible data type API, which has been used to great effect by it's user community (ie, PostGIS). The desire is for HBase to expose an equally extensible data type API. One aspect of the Postgres data type is the ability to provide equivalence functions for index operations. This appears to be of critical performance utility for its execution engine. - Orderly. Orderly handles the issue of compound rowkeys by providing convenience classes for handling these kinds of data types. This influence is reflected in the Struct and Union family of classes. - Phoenix. The PDataType enum used in Phoenix provides type hints, similar Postgres's equivalence functions. These appear to be used during query execution for numerical type promotion. This patch introduces an interface, DataType, along with a number of data type implementations based on the Bytes encoding. Also included are Struct and Union types, demonstrating simple implementations of compound types. Helper classes around the Struct implementation are also provided. This patch does not address the type compatibility concerns expressed by Phoenix's PDataType API (ie, isComparableTo, isCoercibleTo); these will be addressed in HBASE-8863. This patch also provides DataType implementations based on the OrderedBytes encoding from HBASE-8201. --- .../org/apache/hadoop/hbase/types/DataType.java | 122 +++++++ .../hadoop/hbase/types/FixedLengthWrapper.java | 107 ++++++ .../org/apache/hadoop/hbase/types/OrderedBlob.java | 68 ++++ .../apache/hadoop/hbase/types/OrderedBlobVar.java | 64 ++++ .../hadoop/hbase/types/OrderedBytesBase.java | 56 ++++ .../apache/hadoop/hbase/types/OrderedFloat32.java | 73 +++++ .../apache/hadoop/hbase/types/OrderedFloat64.java | 73 +++++ .../apache/hadoop/hbase/types/OrderedInt32.java | 73 +++++ .../apache/hadoop/hbase/types/OrderedInt64.java | 73 +++++ .../apache/hadoop/hbase/types/OrderedNumeric.java | 108 ++++++ .../apache/hadoop/hbase/types/OrderedString.java | 57 ++++ .../org/apache/hadoop/hbase/types/RawBytes.java | 102 ++++++ .../hadoop/hbase/types/RawBytesFixedLength.java | 68 ++++ .../hadoop/hbase/types/RawBytesTerminated.java | 88 +++++ .../org/apache/hadoop/hbase/types/RawDouble.java | 88 +++++ .../org/apache/hadoop/hbase/types/RawFloat.java | 87 +++++ .../org/apache/hadoop/hbase/types/RawInteger.java | 87 +++++ .../org/apache/hadoop/hbase/types/RawLong.java | 87 +++++ .../org/apache/hadoop/hbase/types/RawString.java | 94 ++++++ .../hadoop/hbase/types/RawStringFixedLength.java | 51 +++ .../hadoop/hbase/types/RawStringTerminated.java | 72 ++++ .../java/org/apache/hadoop/hbase/types/Struct.java | 165 ++++++++++ .../apache/hadoop/hbase/types/StructBuilder.java | 54 +++ .../apache/hadoop/hbase/types/StructIterator.java | 92 ++++++ .../hadoop/hbase/types/TerminatedWrapper.java | 158 +++++++++ .../java/org/apache/hadoop/hbase/types/Union2.java | 84 +++++ .../java/org/apache/hadoop/hbase/types/Union3.java | 71 ++++ .../java/org/apache/hadoop/hbase/types/Union4.java | 70 ++++ .../apache/hadoop/hbase/types/package-info.java | 50 +++ .../hadoop/hbase/types/TestFixedLengthWrapper.java | 88 +++++ .../apache/hadoop/hbase/types/TestOrderedBlob.java | 51 +++ .../hadoop/hbase/types/TestOrderedBlobVar.java | 52 +++ .../hadoop/hbase/types/TestOrderedString.java | 48 +++ .../apache/hadoop/hbase/types/TestRawString.java | 57 ++++ .../org/apache/hadoop/hbase/types/TestStruct.java | 364 +++++++++++++++++++++ .../hadoop/hbase/types/TestTerminatedWrapper.java | 100 ++++++ .../org/apache/hadoop/hbase/types/TestUnion2.java | 139 ++++++++ 37 files changed, 3341 insertions(+) create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/DataType.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedLengthWrapper.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlob.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlobVar.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBytesBase.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat32.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat64.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt32.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt64.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedNumeric.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedString.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawBytes.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawBytesFixedLength.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawBytesTerminated.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawDouble.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawFloat.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawInteger.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawLong.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawString.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringFixedLength.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringTerminated.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructBuilder.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructIterator.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/TerminatedWrapper.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union2.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union3.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union4.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/package-info.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestFixedLengthWrapper.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlob.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlobVar.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedString.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestRawString.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestStruct.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestTerminatedWrapper.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestUnion2.java diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/DataType.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/DataType.java new file mode 100644 index 0000000..8b75373 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/DataType.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; +import org.apache.hadoop.io.Writable; + +/** + *

+ * {@code DataType} is the base class for all HBase data types. Data + * type implementations are designed to be serialized to and deserialized from + * byte[]. Serialized representations can retain the natural sort ordering of + * the source object, when a suitable encoding is supported by the underlying + * implementation. This is a desirable feature for use in rowkeys and column + * qualifiers. + *

+ *

+ * {@code DataType}s are different from Hadoop {@link Writable}s in two + * significant ways. First, {@code DataType} describes how to serialize a + * value, it does not encapsulate a serialized value. Second, {@code DataType} + * implementations provide hints to consumers about relationships between the + * POJOs they represent and richness of the encoded representation. + *

+ *

+ * Data type instances are designed to be stateless, thread-safe, and reused. + * Implementations should provide {@code static final} instances corresponding + * to each variation on configurable parameters. This is to encourage and + * simplify instance reuse. For instance, order-preserving types should provide + * static ASCENDING and DESCENDING instances. It is also encouraged for + * implementations operating on Java primitive types to provide primitive + * implementations of the {@code encode} and {@code decode} methods. This + * advice is a performance consideration to clients reading and writing values + * in tight loops. + *

+ */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public interface DataType { + + /** + * Indicates whether this instance writes encoded {@code byte[]}'s + * which preserve the natural sort order of the unencoded value. + * @return {@code true} when natural order is preserved, + * {@code false} otherwise. + */ + public boolean isOrderPreserving(); + + /** + * Retrieve the sort {@link Order} imposed by this data type, or null when + * natural ordering is not preserved. Value is either ascending or + * descending. Default is assumed to be {@link Order#ASCENDING}. + */ + public Order getOrder(); + + /** + * Indicates whether this instance supports encoding null values. This + * depends on the implementation details of the encoding format. All + * {@code DataType}s that support null should treat null as comparing + * less than any non-null value for default sort ordering purposes. + * @return {@code true} when null is supported, {@code false} otherwise. + */ + public boolean isNullable(); + + /** + * Indicates whether this instance is able to skip over it's encoded value. + * {@code DataType}s that are not skippable can only be used as the + * right-most field of a {@link Struct}. + */ + public boolean isSkippable(); + + /** + * Inform consumers how long the encoded {@code byte[]} will be. + * @param val The value to check. + * @return the number of bytes required to encode {@code val}.a + */ + public int encodedLength(T val); + + /** + * Inform consumers over what type this {@code DataType} operates. Useful + * when working with bare {@code DataType} instances. + */ + public Class encodedClass(); + + /** + * Skip {@code src}'s position forward over one encoded value. + * @param src the buffer containing the encoded value. + * @return number of bytes skipped. + */ + public int skip(PositionedByteRange src); + + /** + * Read an instance of {@code T} from the buffer {@code src}. + * @param src the buffer containing the encoded value. + */ + public T decode(PositionedByteRange src); + + /** + * Write instance {@code val} into buffer {@code dst}. + * @param dst the buffer containing the encoded value. + * @param val the value to encode onto {@code dst}. + * @return number of bytes written. + */ + public int encode(PositionedByteRange dst, T val); +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedLengthWrapper.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedLengthWrapper.java new file mode 100644 index 0000000..024f9ff --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedLengthWrapper.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; +import org.apache.hadoop.hbase.util.SimplePositionedByteRange; + +/** + * Wraps an existing {@link DataType} implementation as a fixed-length + * version of itself. This has the useful side-effect of turning an existing + * {@link DataType} which is not {@code skippable} into a {@code skippable} + * variant. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class FixedLengthWrapper implements DataType { + + protected final DataType base; + protected final int length; + + /** + * Create a fixed-length version of the {@code wrapped}. + * @param base the {@link DataType} to restrict to a fixed length. + * @param length the maximum length (in bytes) for encoded values. + */ + public FixedLengthWrapper(DataType base, int length) { + this.base = base; + this.length = length; + } + + /** + * Retrieve the maximum length (in bytes) of encoded values. + */ + public int getLength() { return length; } + + @Override + public boolean isOrderPreserving() { return base.isOrderPreserving(); } + + @Override + public Order getOrder() { return base.getOrder(); } + + @Override + public boolean isNullable() { return base.isNullable(); } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(T val) { return length; } + + @Override + public Class encodedClass() { return base.encodedClass(); } + + @Override + public int skip(PositionedByteRange src) { + src.setPosition(src.getPosition() + this.length); + return this.length; + } + + @Override + public T decode(PositionedByteRange src) { + if (src.getRemaining() < length) { + throw new IllegalArgumentException("Not enough buffer remaining. src.offset: " + + src.getOffset() + " src.length: " + src.getLength() + " src.position: " + + src.getPosition() + " max length: " + length); + } + // create a copy range limited to length bytes. boo. + PositionedByteRange b = new SimplePositionedByteRange(length); + src.get(b.getBytes()); + return base.decode(b); + } + + @Override + public int encode(PositionedByteRange dst, T val) { + if (dst.getRemaining() < length) { + throw new IllegalArgumentException("Not enough buffer remaining. dst.offset: " + + dst.getOffset() + " dst.length: " + dst.getLength() + " dst.position: " + + dst.getPosition() + " max length: " + length); + } + int written = base.encode(dst, val); + if (written > length) { + throw new IllegalArgumentException("Length of encoded value (" + written + + ") exceeds max length (" + length + ")."); + } + // TODO: is the zero-padding appropriate? + for (; written < length; written++) { dst.put((byte) 0x00); } + return written; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlob.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlob.java new file mode 100644 index 0000000..ddacea2 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlob.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * A {@code byte[]} of variable-length. Build on + * {@link OrderedBytes#encodeBlobCopy(PositionedByteRange, byte[], int, int, Order)}. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OrderedBlob extends OrderedBytesBase { + + public static final OrderedBlob ASCENDING = new OrderedBlob(Order.ASCENDING); + public static final OrderedBlob DESCENDING = new OrderedBlob(Order.DESCENDING); + + protected OrderedBlob(Order order) { super(order); } + + @Override + public boolean isSkippable() { return false; } + + @Override + public int encodedLength(byte[] val) { + return null == val ? + (Order.ASCENDING == order ? 1 : 2) : + (Order.ASCENDING == order ? val.length + 1 : val.length + 2); + } + + @Override + public Class encodedClass() { return byte[].class; } + + @Override + public byte[] decode(PositionedByteRange src) { + return OrderedBytes.decodeBlobCopy(src); + } + + @Override + public int encode(PositionedByteRange dst, byte[] val) { + return OrderedBytes.encodeBlobCopy(dst, val, order); + } + + /** + * Write a subset of {@code val} to {@code dst}. + */ + public int encode(PositionedByteRange dst, byte[] val, int voff, int vlen) { + return OrderedBytes.encodeBlobCopy(dst, val, voff, vlen, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlobVar.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlobVar.java new file mode 100644 index 0000000..45f1ae4 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlobVar.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * An alternative to {@link OrderedBlob} for use by {@link Struct} fields that + * do not terminate the fields list. Built on + * {@link OrderedBytes#encodeBlobVar(PositionedByteRange, byte[], int, int, Order)}. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class OrderedBlobVar extends OrderedBytesBase { + + public static final OrderedBlobVar ASCENDING = new OrderedBlobVar(Order.ASCENDING); + public static final OrderedBlobVar DESCENDING = new OrderedBlobVar(Order.DESCENDING); + + protected OrderedBlobVar(Order order) { super(order); } + + @Override + public int encodedLength(byte[] val) { + return null == val ? 1 : OrderedBytes.blobVarEncodedLength(val.length); + } + + @Override + public Class encodedClass() { return byte[].class; } + + @Override + public byte[] decode(PositionedByteRange src) { + return OrderedBytes.decodeBlobVar(src); + } + + @Override + public int encode(PositionedByteRange dst, byte[] val) { + return OrderedBytes.encodeBlobVar(dst, val, order); + } + + /** + * Write a subset of {@code val} to {@code buff}. + */ + public int encode(PositionedByteRange dst, byte[] val, int voff, int vlen) { + return OrderedBytes.encodeBlobVar(dst, val, voff, vlen, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBytesBase.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBytesBase.java new file mode 100644 index 0000000..d6eb498 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBytesBase.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * Base class for data types backed by the {@link OrderedBytes} encoding + * implementations. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class OrderedBytesBase implements DataType { + + protected final Order order; + + protected OrderedBytesBase(Order order) { this.order = order; } + + @Override + public boolean isOrderPreserving() { return true; } + + @Override + public Order getOrder() { return order; } + + // almost all OrderedBytes implementations are nullable. + @Override + public boolean isNullable() { return true; } + + // almost all OrderedBytes implementations are skippable. + @Override + public boolean isSkippable() { return true; } + + @Override + public int skip(PositionedByteRange src) { + return OrderedBytes.skip(src); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat32.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat32.java new file mode 100644 index 0000000..f0a8097 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat32.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; +import org.apache.hadoop.hbase.util.PositionedByteRange; + + +/** + * A {@code float} of 32-bits using a fixed-length encoding. Based on + * {@link OrderedBytes#encodeFloat32(PositionedByteRange, float, Order)}. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OrderedFloat32 extends OrderedBytesBase { + + public static final OrderedFloat32 ASCENDING = new OrderedFloat32(Order.ASCENDING); + public static final OrderedFloat32 DESCENDING = new OrderedFloat32(Order.DESCENDING); + + protected OrderedFloat32(Order order) { super(order); } + + @Override + public boolean isNullable() { return false; } + + @Override + public int encodedLength(Float val) { return 5; } + + @Override + public Class encodedClass() { return Float.class; } + + @Override + public Float decode(PositionedByteRange src) { + return OrderedBytes.decodeFloat32(src); + } + + @Override + public int encode(PositionedByteRange dst, Float val) { + if (null == val) throw new IllegalArgumentException("Null values not supported."); + return OrderedBytes.encodeFloat32(dst, val, order); + } + + /** + * Read a {@code float} value from the buffer {@code dst}. + */ + public float decodeFloat(PositionedByteRange dst) { + return OrderedBytes.decodeFloat32(dst); + } + + /** + * Write instance {@code val} into buffer {@code buff}. + */ + public int encodeFloat(PositionedByteRange dst, float val) { + return OrderedBytes.encodeFloat32(dst, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat64.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat64.java new file mode 100644 index 0000000..f60dbce --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat64.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; +import org.apache.hadoop.hbase.util.PositionedByteRange; + + +/** + * A {@code double} of 64-bits using a fixed-length encoding. Built on + * {@link OrderedBytes#encodeFloat64(PositionedByteRange, double, Order)}. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OrderedFloat64 extends OrderedBytesBase { + + public static final OrderedFloat64 ASCENDING = new OrderedFloat64(Order.ASCENDING); + public static final OrderedFloat64 DESCENDING = new OrderedFloat64(Order.DESCENDING); + + protected OrderedFloat64(Order order) { super(order); } + + @Override + public boolean isNullable() { return false; } + + @Override + public int encodedLength(Double val) { return 9; } + + @Override + public Class encodedClass() { return Double.class; } + + @Override + public Double decode(PositionedByteRange src) { + return OrderedBytes.decodeFloat64(src); + } + + @Override + public int encode(PositionedByteRange dst, Double val) { + if (null == val) throw new IllegalArgumentException("Null values not supported."); + return OrderedBytes.encodeFloat64(dst, val, order); + } + + /** + * Read a {@code double} value from the buffer {@code src}. + */ + public double decodeDouble(PositionedByteRange src) { + return OrderedBytes.decodeFloat64(src); + } + + /** + * Write instance {@code val} into buffer {@code dst}. + */ + public int encodeDouble(PositionedByteRange dst, double val) { + return OrderedBytes.encodeFloat64(dst, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt32.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt32.java new file mode 100644 index 0000000..95bfd8e --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt32.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; +import org.apache.hadoop.hbase.util.PositionedByteRange; + + +/** + * An {@code int} of 32-bits using a fixed-length encoding. Built on + * {@link OrderedBytes#encodeInt32(PositionedByteRange, int, Order)}. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OrderedInt32 extends OrderedBytesBase { + + public static final OrderedInt32 ASCENDING = new OrderedInt32(Order.ASCENDING); + public static final OrderedInt32 DESCENDING = new OrderedInt32(Order.DESCENDING); + + protected OrderedInt32(Order order) { super(order); } + + @Override + public boolean isNullable() { return false; } + + @Override + public int encodedLength(Integer val) { return 5; } + + @Override + public Class encodedClass() { return Integer.class; } + + @Override + public Integer decode(PositionedByteRange src) { + return OrderedBytes.decodeInt32(src); + } + + @Override + public int encode(PositionedByteRange dst, Integer val) { + if (null == val) throw new IllegalArgumentException("Null values not supported."); + return OrderedBytes.encodeInt32(dst, val, order); + } + + /** + * Read an {@code int} value from the buffer {@code src}. + */ + public int decodeInt(PositionedByteRange src) { + return OrderedBytes.decodeInt32(src); + } + + /** + * Write instance {@code val} into buffer {@code dst}. + */ + public int encodeInt(PositionedByteRange dst, int val) { + return OrderedBytes.encodeInt32(dst, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt64.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt64.java new file mode 100644 index 0000000..54e5ad8 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt64.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; +import org.apache.hadoop.hbase.util.PositionedByteRange; + + +/** + * A {@code long} of 64-bits using a fixed-length encoding. Built on + * {@link OrderedBytes#encodeInt64(PositionedByteRange, long, Order)}. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OrderedInt64 extends OrderedBytesBase { + + public static final OrderedInt64 ASCENDING = new OrderedInt64(Order.ASCENDING); + public static final OrderedInt64 DESCENDING = new OrderedInt64(Order.DESCENDING); + + protected OrderedInt64(Order order) { super(order); } + + @Override + public boolean isNullable() { return false; } + + @Override + public int encodedLength(Long val) { return 9; } + + @Override + public Class encodedClass() { return Long.class; } + + @Override + public Long decode(PositionedByteRange src) { + return OrderedBytes.decodeInt64(src); + } + + @Override + public int encode(PositionedByteRange dst, Long val) { + if (null == val) throw new IllegalArgumentException("Null values not supported."); + return OrderedBytes.encodeInt64(dst, val, order); + } + + /** + * Read a {@code long} value from the buffer {@code src}. + */ + public long decodeLong(PositionedByteRange src) { + return OrderedBytes.decodeInt64(src); + } + + /** + * Write instance {@code val} into buffer {@code dst}. + */ + public int encodeLong(PositionedByteRange dst, long val) { + return OrderedBytes.encodeInt64(dst, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedNumeric.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedNumeric.java new file mode 100644 index 0000000..42636f1 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedNumeric.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.math.BigDecimal; +import java.math.BigInteger; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; +import org.apache.hadoop.hbase.util.PositionedByteRange; +import org.apache.hadoop.hbase.util.SimplePositionedByteRange; + +/** + * An {@link Number} of arbitrary precision and variable-length encoding. The + * resulting length of encoded values is determined by the numerical (base + * 100) precision, not absolute value. Use this data type anywhere you would + * expect to use a {@code DECIMAL} type, a {@link BigDecimal}, a + * {@link BigInteger}, or any time you've parsed floating precision values + * from text. Built on {@link OrderedBytes#encodeNumeric(PositionedByteRange, BigDecimal, Order)}. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OrderedNumeric extends OrderedBytesBase { + + public static final OrderedNumeric ASCENDING = new OrderedNumeric(Order.ASCENDING); + public static final OrderedNumeric DESCENDING = new OrderedNumeric(Order.DESCENDING); + + protected OrderedNumeric(Order order) { super(order); } + + @Override + public int encodedLength(Number val) { + // TODO: this could be done better. + PositionedByteRange buff = new SimplePositionedByteRange(100); + return encode(buff, val); + } + + @Override + public Class encodedClass() { return Number.class; } + + @Override + public Number decode(PositionedByteRange src) { + if (OrderedBytes.isNumericInfinite(src) || OrderedBytes.isNumericNaN(src)) { + return OrderedBytes.decodeNumericAsDouble(src); + } + return OrderedBytes.decodeNumericAsBigDecimal(src); + } + + @Override + public int encode(PositionedByteRange dst, Number val) { + if (null == val) { + return OrderedBytes.encodeNull(dst, order); + } else if (val instanceof BigDecimal) { + return OrderedBytes.encodeNumeric(dst, (BigDecimal) val, order); + } else if (val instanceof BigInteger) { + return OrderedBytes.encodeNumeric(dst, new BigDecimal((BigInteger) val), order); + } else if (val instanceof Double || val instanceof Float) { + return OrderedBytes.encodeNumeric(dst, val.doubleValue(), order); + } else { + // TODO: other instances of Numeric to consider? + return OrderedBytes.encodeNumeric(dst, val.longValue(), order); + } + } + + /** + * Read a {@code long} value from the buffer {@code src}. + */ + public long decodeLong(PositionedByteRange src) { + return OrderedBytes.decodeNumericAsLong(src); + } + + /** + * Write instance {@code val} into buffer {@code dst}. + */ + public int encodeLong(PositionedByteRange dst, long val) { + return OrderedBytes.encodeNumeric(dst, val, order); + } + + /** + * Read a {@code double} value from the buffer {@code src}. + */ + public double decodeDouble(PositionedByteRange src) { + return OrderedBytes.decodeNumericAsLong(src); + } + + /** + * Write instance {@code val} into buffer {@code dst}. + */ + public int encodeDouble(PositionedByteRange dst, double val) { + return OrderedBytes.encodeNumeric(dst, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedString.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedString.java new file mode 100644 index 0000000..be464e8 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedString.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * A {@code String} of variable-length. Built on + * {@link OrderedBytes#encodeString(PositionedByteRange, String, Order)}. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OrderedString extends OrderedBytesBase { + + public static final OrderedString ASCENDING = new OrderedString(Order.ASCENDING); + public static final OrderedString DESCENDING = new OrderedString(Order.DESCENDING); + + protected OrderedString(Order order) { super(order); } + + @Override + public int encodedLength(String val) { + // TODO: use of UTF8 here is a leaky abstraction. + return null == val ? 1 : val.getBytes(OrderedBytes.UTF8).length + 2; + } + + @Override + public Class encodedClass() { return String.class; } + + @Override + public String decode(PositionedByteRange src) { + return OrderedBytes.decodeString(src); + } + + @Override + public int encode(PositionedByteRange dst, String val) { + return OrderedBytes.encodeString(dst, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawBytes.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawBytes.java new file mode 100644 index 0000000..8469c89 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawBytes.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * An {@code DataType} for interacting with variable-length values + * encoded using {@link Bytes#putBytes(byte[], int, byte[], int, int)}. + * Intended to make it easier to transition away from direct use of + * {@link Bytes}. + * @see Bytes#putBytes(byte[], int, byte[], int, int) + * @see RawBytesTerminated + * @see RawBytesFixedLength + * @see OrderedBlob + * @see OrderedBlobVar + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class RawBytes implements DataType { + + public static final RawBytes ASCENDING = new RawBytes(Order.ASCENDING); + public static final RawBytes DESCENDING = new RawBytes(Order.DESCENDING); + + protected final Order order; + + protected RawBytes() { this.order = Order.ASCENDING; } + protected RawBytes(Order order) { this.order = order; } + + @Override + public boolean isOrderPreserving() { return true; } + + @Override + public Order getOrder() { return order; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return false; } + + @Override + public int skip(PositionedByteRange src) { + int skipped = src.getRemaining(); + src.setPosition(src.getLength()); + return skipped; + } + + @Override + public int encodedLength(byte[] val) { return val.length; } + + @Override + public Class encodedClass() { return byte[].class; } + + @Override + public byte[] decode(PositionedByteRange src) { + return decode(src, src.getRemaining()); + } + + @Override + public int encode(PositionedByteRange dst, byte[] val) { + return encode(dst, val, 0, val.length); + } + + /** + * Read a {@code byte[]} from the buffer {@code src}. + */ + public byte[] decode(PositionedByteRange src, int length) { + byte[] val = new byte[length]; + src.get(val); + return val; + } + + /** + * Write {@code val} into {@code dst}, respecting {@code voff} and {@code vlen}. + * @return number of bytes written. + */ + public int encode(PositionedByteRange dst, byte[] val, int voff, int vlen) { + Bytes.putBytes(dst.getBytes(), dst.getOffset() + dst.getPosition(), val, voff, vlen); + dst.setPosition(dst.getPosition() + vlen); + return vlen; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawBytesFixedLength.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawBytesFixedLength.java new file mode 100644 index 0000000..aac34e4 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawBytesFixedLength.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * An {@code DataType} that encodes fixed-length values encoded using + * {@link Bytes#putBytes(byte[], int, byte[], int, int)}. Intended to make it + * easier to transition away from direct use of {@link Bytes}. + * @see Bytes#putBytes(byte[], int, byte[], int, int) + * @see RawBytes + * @see OrderedBlob + * @see OrderedBlobVar + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class RawBytesFixedLength extends FixedLengthWrapper { + + /** + * Create a {@code RawBytesFixedLength} using the specified {@code order} + * and {@code length}. + */ + public RawBytesFixedLength(Order order, int length) { + super(new RawBytes(order), length); + } + + /** + * Create a {@code RawBytesFixedLength} of the specified {@code length}. + */ + public RawBytesFixedLength(int length) { + super(new RawBytes(), length); + } + + /** + * Read a {@code byte[]} from the buffer {@code src}. + */ + public byte[] decode(PositionedByteRange src, int length) { + return ((RawBytes) base).decode(src, length); + } + + /** + * Write {@code val} into {@code buff}, respecting {@code offset} and + * {@code length}. + */ + public int encode(PositionedByteRange dst, byte[] val, int voff, int vlen) { + return ((RawBytes) base).encode(dst, val, voff, vlen); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawBytesTerminated.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawBytesTerminated.java new file mode 100644 index 0000000..db71aaf --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawBytesTerminated.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * An {@code DataType} that encodes variable-length values encoded using + * {@link Bytes#putBytes(byte[], int, byte[], int, int)}. Includes a + * termination marker following the raw {@code byte[]} value. Intended to + * make it easier to transition away from direct use of {@link Bytes}. + * @see Bytes#putBytes(byte[], int, byte[], int, int) + * @see RawBytes + * @see OrderedBlob + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class RawBytesTerminated extends TerminatedWrapper { + + /** + * Create a {@code RawBytesTerminated} using the specified terminator and + * {@code order}. + * @throws IllegalArgumentException if {@code term} is {@code null} or empty. + */ + public RawBytesTerminated(Order order, byte[] term) { + super(new RawBytes(order), term); + } + + /** + * Create a {@code RawBytesTerminated} using the specified terminator and + * {@code order}. + * @throws IllegalArgumentException if {@code term} is {@code null} or empty. + */ + public RawBytesTerminated(Order order, String term) { + super(new RawBytes(order), term); + } + + /** + * Create a {@code RawBytesTerminated} using the specified terminator. + * @throws IllegalArgumentException if {@code term} is {@code null} or empty. + */ + public RawBytesTerminated(byte[] term) { + super(new RawBytes(), term); + } + + /** + * Create a {@code RawBytesTerminated} using the specified terminator. + * @throws IllegalArgumentException if {@code term} is {@code null} or empty. + */ + public RawBytesTerminated(String term) { + super(new RawBytes(), term); + } + + /** + * Read a {@code byte[]} from the buffer {@code src}. + */ + public byte[] decode(PositionedByteRange src, int length) { + return ((RawBytes) wrapped).decode(src, length); + } + + /** + * Write {@code val} into {@code dst}, respecting {@code offset} and + * {@code length}. + * @return number of bytes written. + */ + public int encode(PositionedByteRange dst, byte[] val, int voff, int vlen) { + return ((RawBytes) wrapped).encode(dst, val, voff, vlen); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawDouble.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawDouble.java new file mode 100644 index 0000000..45ad9b3 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawDouble.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * An {@code DataType} for interacting with values encoded using + * {@link Bytes#putDouble(byte[], int, double)}. Intended to make it easier to + * transition away from direct use of {@link Bytes}. + * @see Bytes#putDouble(byte[], int, double) + * @see Bytes#toDouble(byte[]) + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class RawDouble implements DataType { + + @Override + public boolean isOrderPreserving() { return false; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(Double val) { return Bytes.SIZEOF_DOUBLE; } + + @Override + public Class encodedClass() { return Double.class; } + + @Override + public int skip(PositionedByteRange src) { + src.setPosition(src.getPosition() + Bytes.SIZEOF_DOUBLE); + return Bytes.SIZEOF_DOUBLE; + } + + @Override + public Double decode(PositionedByteRange src) { + double val = Bytes.toDouble(src.getBytes(), src.getOffset() + src.getPosition()); + skip(src); + return val; + } + + @Override + public int encode(PositionedByteRange dst, Double val) { + Bytes.putDouble(dst.getBytes(), dst.getOffset() + dst.getPosition(), val); + return skip(dst); + } + + /** + * Read a {@code double} value from the buffer {@code buff}. + */ + public double decodeDouble(byte[] buff, int offset) { + double val = Bytes.toDouble(buff, offset); + return val; + } + + /** + * Write instance {@code val} into buffer {@code buff}. + */ + public int encodeDouble(byte[] buff, int offset, double val) { + return Bytes.putDouble(buff, offset, val); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawFloat.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawFloat.java new file mode 100644 index 0000000..266b0d8 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawFloat.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * An {@code DataType} for interacting with values encoded using + * {@link Bytes#putFloat(byte[], int, float)}. Intended to make it easier to + * transition away from direct use of {@link Bytes}. + * @see Bytes#putFloat(byte[], int, float) + * @see Bytes#toFloat(byte[]) + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class RawFloat implements DataType { + + @Override + public boolean isOrderPreserving() { return false; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(Float val) { return Bytes.SIZEOF_FLOAT; } + + @Override + public Class encodedClass() { return Float.class; } + + @Override + public int skip(PositionedByteRange src) { + src.setPosition(src.getPosition() + Bytes.SIZEOF_FLOAT); + return Bytes.SIZEOF_FLOAT; + } + + @Override + public Float decode(PositionedByteRange src) { + float val = Bytes.toFloat(src.getBytes(), src.getOffset() + src.getPosition()); + skip(src); + return val; + } + + @Override + public int encode(PositionedByteRange dst, Float val) { + Bytes.putFloat(dst.getBytes(), dst.getOffset() + dst.getPosition(), val); + return skip(dst); + } + + /** + * Read a {@code float} value from the buffer {@code buff}. + */ + public float decodeFloat(byte[] buff, int offset) { + return Bytes.toFloat(buff, offset); + } + + /** + * Write instance {@code val} into buffer {@code buff}. + */ + public int encodeFloat(byte[] buff, int offset, float val) { + return Bytes.putFloat(buff, offset, val); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawInteger.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawInteger.java new file mode 100644 index 0000000..4340d49 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawInteger.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * An {@code DataType} for interacting with values encoded using + * {@link Bytes#putInt(byte[], int, int)}. Intended to make it easier to + * transition away from direct use of {@link Bytes}. + * @see Bytes#putInt(byte[], int, int) + * @see Bytes#toInt(byte[]) + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class RawInteger implements DataType { + + @Override + public boolean isOrderPreserving() { return false; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(Integer val) { return Bytes.SIZEOF_INT; } + + @Override + public Class encodedClass() { return Integer.class; } + + @Override + public int skip(PositionedByteRange src) { + src.setPosition(src.getPosition() + Bytes.SIZEOF_INT); + return Bytes.SIZEOF_INT; + } + + @Override + public Integer decode(PositionedByteRange src) { + int val = Bytes.toInt(src.getBytes(), src.getOffset() + src.getPosition()); + skip(src); + return val; + } + + @Override + public int encode(PositionedByteRange dst, Integer val) { + Bytes.putInt(dst.getBytes(), dst.getOffset() + dst.getPosition(), val); + return skip(dst); + } + + /** + * Read an {@code int} value from the buffer {@code buff}. + */ + public int decodeInt(byte[] buff, int offset) { + return Bytes.toInt(buff, offset); + } + + /** + * Write instance {@code val} into buffer {@code buff}. + */ + public int encodeInt(byte[] buff, int offset, int val) { + return Bytes.putInt(buff, offset, val); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawLong.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawLong.java new file mode 100644 index 0000000..b560dd8 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawLong.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * An {@code DataType} for interacting with values encoded using + * {@link Bytes#putLong(byte[], int, long)}. Intended to make it easier to + * transition away from direct use of {@link Bytes}. + * @see Bytes#putLong(byte[], int, long) + * @see Bytes#toLong(byte[]) + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class RawLong implements DataType { + + @Override + public boolean isOrderPreserving() { return false; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(Long val) { return Bytes.SIZEOF_LONG; } + + @Override + public Class encodedClass() { return Long.class; } + + @Override + public int skip(PositionedByteRange src) { + src.setPosition(src.getPosition() + Bytes.SIZEOF_LONG); + return Bytes.SIZEOF_LONG; + } + + @Override + public Long decode(PositionedByteRange src) { + long val = Bytes.toLong(src.getBytes(), src.getOffset() + src.getPosition()); + skip(src); + return val; + } + + @Override + public int encode(PositionedByteRange dst, Long val) { + Bytes.putLong(dst.getBytes(), dst.getOffset() + dst.getPosition(), val); + return skip(dst); + } + + /** + * Read a {@code long} value from the buffer {@code buff}. + */ + public long decodeLong(byte[] buff, int offset) { + return Bytes.toLong(buff, offset); + } + + /** + * Write instance {@code val} into buffer {@code buff}. + */ + public int encodeLong(byte[] buff, int offset, long val) { + return Bytes.putLong(buff, offset, val); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawString.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawString.java new file mode 100644 index 0000000..ecd7e20 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawString.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * An {@code DataType} for interacting with values encoded using + * {@link Bytes#toBytes(String)}. Intended to make it easier to transition + * away from direct use of {@link Bytes}. + * @see Bytes#toBytes(String) + * @see Bytes#toString(byte[]) + * @see RawStringTerminated + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class RawString implements DataType { + + public static final RawString ASCENDING = new RawString(Order.ASCENDING); + public static final RawString DESCENDING = new RawString(Order.DESCENDING); + + protected final Order order; + + protected RawString() { this.order = Order.ASCENDING; } + protected RawString(Order order) { this.order = order; } + + @Override + public boolean isOrderPreserving() { return true; } + + @Override + public Order getOrder() { return order; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return false; } + + @Override + public int skip(PositionedByteRange src) { + int skipped = src.getRemaining(); + src.setPosition(src.getLength()); + return skipped; + } + + @Override + public int encodedLength(String val) { return Bytes.toBytes(val).length; } + + @Override + public Class encodedClass() { return String.class; } + + @Override + public String decode(PositionedByteRange src) { + if (Order.ASCENDING == this.order) { + // avoid unnecessary array copy for ASC case. + String val = + Bytes.toString(src.getBytes(), src.getOffset() + src.getPosition(), src.getRemaining()); + src.setPosition(src.getLength()); + return val; + } else { + byte[] b = new byte[src.getRemaining()]; + src.get(b); + order.apply(b, 0, b.length); + return Bytes.toString(b); + } + } + + @Override + public int encode(PositionedByteRange dst, String val) { + byte[] s = Bytes.toBytes(val); + order.apply(s); + dst.put(s); + return s.length; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringFixedLength.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringFixedLength.java new file mode 100644 index 0000000..d9b434e --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringFixedLength.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An {@code DataType} that encodes fixed-length values encoded using + * {@link Bytes#toBytes(String)}. Intended to make it easier to transition + * away from direct use of {@link Bytes}. + * @see Bytes#toBytes(String) + * @see Bytes#toString(byte[], int, int) + * @see RawString + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class RawStringFixedLength extends FixedLengthWrapper { + + /** + * Create a {@code RawStringFixedLength} using the specified + * {@code order} and {@code length}. + */ + public RawStringFixedLength(Order order, int length) { + super(new RawString(order), length); + } + + /** + * Create a {@code RawStringFixedLength} of the specified {@code length}. + */ + public RawStringFixedLength(int length) { + super(new RawString(), length); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringTerminated.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringTerminated.java new file mode 100644 index 0000000..ebbd2b5 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringTerminated.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An {@code DataType} that encodes variable-length values encoded using + * {@link Bytes#toBytes(String)}. Includes a termination marker following the + * raw {@code byte[]} value. Intended to make it easier to transition + * away from direct use of {@link Bytes}. + * @see Bytes#toBytes(String) + * @see Bytes#toString(byte[], int, int) + * @see RawString + * @see OrderedString + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class RawStringTerminated extends TerminatedWrapper { + + /** + * Create a {@code RawStringTerminated} using the specified terminator and + * {@code order}. + * @throws IllegalArgumentException if {@code term} is {@code null} or empty. + */ + public RawStringTerminated(Order order, byte[] term) { + super(new RawString(order), term); + } + + /** + * Create a {@code RawStringTerminated} using the specified terminator and + * {@code order}. + * @throws IllegalArgumentException if {@code term} is {@code null} or empty. + */ + public RawStringTerminated(Order order, String term) { + super(new RawString(order), term); + } + + /** + * Create a {@code RawStringTerminated} using the specified terminator. + * @throws IllegalArgumentException if {@code term} is {@code null} or empty. + */ + public RawStringTerminated(byte[] term) { + super(new RawString(), term); + } + + /** + * Create a {@code RawStringTerminated} using the specified terminator. + * @throws IllegalArgumentException if {@code term} is {@code null} or empty. + */ + public RawStringTerminated(String term) { + super(new RawString(), term); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java new file mode 100644 index 0000000..8c2c38d --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.util.Iterator; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + *

+ * {@code Struct} is a simple {@link DataType} for implementing "compound + * rowkey" and "compound qualifier" schema design strategies. + *

+ *

Encoding

+ *

+ * {@code Struct} member values are encoded onto the target byte[] in the order + * in which they are declared. A {@code Struct} may be used as a member of + * another {@code Struct}. {@code Struct}s are not {@code nullable} but their + * component fields may be. + *

+ *

Sort Order

+ *

+ * {@code Struct} instances sort according to the composite order of their + * fields, that is, left-to-right and depth-first. This can also be thought of + * as lexicographic comparison of concatenated members. + *

+ *

+ * {@link StructIterator} is provided as a convenience for consuming the + * sequence of values. Users may find it more appropriate to provide their own + * custom {@link DataType} for encoding application objects rather than using + * this {@code Object[]} implementation. Examples are provided in test. + *

+ * @see StructIterator + * @see DataType#isNullable() + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class Struct implements DataType { + + @SuppressWarnings("rawtypes") + protected final DataType[] fields; + protected final boolean isOrderPreserving; + protected final boolean isSkippable; + + /** + * Create a new {@code Struct} instance defined as the sequence of + * {@code HDataType}s in {@code memberTypes}. + *

+ * A {@code Struct} is {@code orderPreserving} when all of its fields + * are {@code orderPreserving}. A {@code Struct} is {@code skippable} when + * all of its fields are {@code skippable}. + *

+ */ + @SuppressWarnings("rawtypes") + public Struct(DataType[] memberTypes) { + this.fields = memberTypes; + // a Struct is not orderPreserving when any of its fields are not. + boolean preservesOrder = true; + // a Struct is not skippable when any of its fields are not. + boolean skippable = true; + for (int i = 0; i < this.fields.length; i++) { + DataType dt = this.fields[i]; + if (!dt.isOrderPreserving()) preservesOrder = false; + if (i < this.fields.length - 2 && !dt.isSkippable()) { + throw new IllegalArgumentException("Field in position " + i + + " is not skippable. Non-right-most struct fields must be skippable."); + } + if (!dt.isSkippable()) skippable = false; + } + this.isOrderPreserving = preservesOrder; + this.isSkippable = skippable; + } + + @Override + public boolean isOrderPreserving() { return isOrderPreserving; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return isSkippable; } + + @SuppressWarnings("unchecked") + @Override + public int encodedLength(Object[] val) { + assert fields.length == val.length; + int sum = 0; + for (int i = 0; i < fields.length; i++) + sum += fields[i].encodedLength(val[i]); + return sum; + } + + @Override + public Class encodedClass() { return Object[].class; } + + /** + * Retrieve an {@link Iterator} over the values encoded in {@code src}. + * {@code src}'s position is consumed by consuming this iterator. + */ + public StructIterator iterator(PositionedByteRange src) { + return new StructIterator(src, fields); + } + + @Override + public int skip(PositionedByteRange src) { + StructIterator it = iterator(src); + int skipped = 0; + while (it.hasNext()) + skipped += it.skip(); + return skipped; + } + + @Override + public Object[] decode(PositionedByteRange src) { + int i = 0; + Object[] ret = new Object[fields.length]; + Iterator it = iterator(src); + while (it.hasNext()) + ret[i++] = it.next(); + return ret; + } + + /** + * Read the field at {@code index}. {@code src}'s position is not affected. + */ + public Object decode(PositionedByteRange src, int index) { + assert index >= 0; + StructIterator it = iterator(src.shallowCopy()); + for (; index > 0; index--) + it.skip(); + return it.next(); + } + + @SuppressWarnings("unchecked") + @Override + public int encode(PositionedByteRange dst, Object[] val) { + assert fields.length == val.length; + int written = 0; + for (int i = 0; i < fields.length; i++) { + written += fields[i].encode(dst, val[i]); + } + return written; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructBuilder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructBuilder.java new file mode 100644 index 0000000..0bf65bc --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructBuilder.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * A helper for building {@link Struct} instances. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class StructBuilder { + + protected final List> fields = new ArrayList>(); + + /** + * Create an empty {@code StructBuilder}. + */ + public StructBuilder() {} + + /** + * Append {@code field} to the sequence of accumulated fields. + */ + public StructBuilder add(DataType field) { fields.add(field); return this; } + + /** + * Retrieve the {@link Struct} represented by {@code this}. + */ + public Struct toStruct() { return new Struct(fields.toArray(new DataType[fields.size()])); } + + /** + * Reset the sequence of accumulated fields. + */ + public StructBuilder reset() { fields.clear(); return this; } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructIterator.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructIterator.java new file mode 100644 index 0000000..12b4225 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructIterator.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * An {@link Iterator} over encoded {@code Struct} members. + *

+ * This iterates over each serialized {@code Struct} field from the specified + * {@code DataTypes[]} definition. It allows you to read the field or skip + * over its serialized bytes using {@link #next()} and {@link #skip()}, + * respectively. This is in contrast to the {@code Struct} method which allow + * you to {@link Struct#decode(PositionedByteRange)} or + * {@link Struct#skip(PositionedByteRange)} over the entire {@code Struct} at + * once. + *

+ *

+ * This iterator may also be used to read bytes from any {@code Struct} for + * which the specified {@code DataType[]} is a prefix. For example, if the + * specified {@code Struct} definition has a {@link RawInteger} and a + * {@link RawStringTerminated} field, you may parse the serialized output + * of a {@code Struct} whose fields are {@link RawInteger}, + * {@link RawStringTerminated}, and {@link RawBytes}. The iterator would + * return a number followed by a {@code String}. The trailing {@code byte[]} + * would be ignored. + *

+ */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class StructIterator implements Iterator { + + protected final PositionedByteRange src; + protected int idx = 0; + @SuppressWarnings("rawtypes") + protected final DataType[] types; + + /** + * Construct {@code StructIterator} over the values encoded in {@code src} + * using the specified {@code types} definition. + * @param src The buffer from which to read encoded values. + * @param types The sequence of types to use as the schema for this + * {@code Struct}. + */ + public StructIterator(PositionedByteRange src, @SuppressWarnings("rawtypes") DataType[] types) { + this.src = src; + this.types = types; + } + + @Override + public boolean hasNext() { + return idx < types.length; + } + + @Override + public void remove() { throw new UnsupportedOperationException(); } + + @Override + public Object next() { + if (!hasNext()) throw new NoSuchElementException(); + return types[idx++].decode(src); + } + + /** + * Bypass the next encoded value. + * @return the number of bytes skipped. + */ + public int skip() { + if (!hasNext()) throw new NoSuchElementException(); + return types[idx++].skip(src); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/TerminatedWrapper.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/TerminatedWrapper.java new file mode 100644 index 0000000..c854850 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/TerminatedWrapper.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; +import org.apache.hadoop.hbase.util.SimplePositionedByteRange; + +/** + * Wraps an existing {@code DataType} implementation as a terminated + * version of itself. This has the useful side-effect of turning an existing + * {@code DataType} which is not {@code skippable} into a + * {@code skippable} variant. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class TerminatedWrapper implements DataType { + + protected final DataType wrapped; + protected final byte[] term; + + /** + * Create a terminated version of the {@code wrapped}. + * @throws IllegalArgumentException when {@code term} is null or empty. + */ + public TerminatedWrapper(DataType wrapped, byte[] term) { + if (null == term || term.length == 0) + throw new IllegalArgumentException("terminator must be non-null and non-empty."); + this.wrapped = wrapped; + wrapped.getOrder().apply(term); + this.term = term; + } + + /** + * Create a terminated version of the {@code wrapped}. + * {@code term} is converted to a {@code byte[]} using + * {@link Bytes#toBytes(String)}. + * @throws IllegalArgumentException when {@code term} is null or empty. + */ + public TerminatedWrapper(DataType wrapped, String term) { + this(wrapped, Bytes.toBytes(term)); + } + + @Override + public boolean isOrderPreserving() { return wrapped.isOrderPreserving(); } + + @Override + public Order getOrder() { return wrapped.getOrder(); } + + @Override + public boolean isNullable() { return wrapped.isNullable(); } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(T val) { + return wrapped.encodedLength(val) + term.length; + } + + @Override + public Class encodedClass() { return wrapped.encodedClass(); } + + /** + * Return the position at which {@code term} begins within {@code src}, + * or {@code -1} if {@code term} is not found. + */ + protected int terminatorPosition(PositionedByteRange src) { + byte[] a = src.getBytes(); + final int offset = src.getOffset(); + int i; + SKIP: for (i = src.getPosition(); i < src.getLength(); i++) { + if (a[offset + i] != term[0]) continue; + int j; + for (j = 1; j < term.length && offset + j < src.getLength(); j++) { + if (a[offset + i + j] != term[j]) continue SKIP; + } + if (j == term.length) return i; // success + } + return -1; + } + + /** + * Skip {@code src}'s position forward over one encoded value. + * @param src the buffer containing the encoded value. + * @return number of bytes skipped. + * @throws IllegalArgumentException when the terminator sequence is not found. + */ + @Override + public int skip(PositionedByteRange src) { + if (wrapped.isSkippable()) { + return wrapped.skip(src); + } else { + // find the terminator position + final int start = src.getPosition(); + int skipped = terminatorPosition(src); + if (-1 == skipped) throw new IllegalArgumentException("Terminator sequence not found."); + skipped += term.length; + src.setPosition(skipped); + return skipped - start; + } + } + + @Override + public T decode(PositionedByteRange src) { + if (wrapped.isSkippable()) { + return wrapped.decode(src); + } else { + // find the terminator position + int term = terminatorPosition(src); + if (-1 == term) throw new IllegalArgumentException("Terminator sequence not found."); + byte[] b = new byte[term - src.getPosition()]; + src.get(b); + // TODO: should we assert that b.position == b.length? + T ret = wrapped.decode(new SimplePositionedByteRange(b)); + src.get(this.term); + return ret; + } + } + + /** + * Write instance {@code val} into buffer {@code dst}. + * @throws IllegalArgumentException when the encoded representation of + * {@code val} contains the {@code term} sequence. + */ + @Override + public int encode(PositionedByteRange dst, T val) { + final int start = dst.getPosition(); + int written = wrapped.encode(dst, val); + PositionedByteRange b = dst.shallowCopy(); + b.setLength(dst.getPosition()); + b.setPosition(start); + if (-1 != terminatorPosition(b)) { + dst.setPosition(start); + throw new IllegalArgumentException("Encoded value contains terminator sequence."); + } + dst.put(term); + return written + term.length; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union2.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union2.java new file mode 100644 index 0000000..4265dc8 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union2.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * The {@code Union} family of {@link DataType}s encode one of a fixed + * set of {@code Object}s. They provide convenience methods which handle + * type casting on your behalf. + */ +@SuppressWarnings("unchecked") +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class Union2 implements DataType { + + protected final DataType typeA; + protected final DataType typeB; + + /** + * Create an instance of {@code Union2} over the set of specified + * types. + */ + public Union2(DataType typeA, DataType typeB) { + this.typeA = typeA; + this.typeB = typeB; + } + + @Override + public boolean isOrderPreserving() { + return typeA.isOrderPreserving() && typeB.isOrderPreserving(); + } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { + return typeA.isNullable() && typeB.isNullable(); + } + + @Override + public boolean isSkippable() { + return typeA.isSkippable() && typeB.isSkippable(); + } + + @Override + public Class encodedClass() { + throw new UnsupportedOperationException( + "Union types do not expose a definitive encoded class."); + } + + /** + * Read an instance of the first type parameter from buffer {@code src}. + */ + public A decodeA(PositionedByteRange src) { + return (A) decode(src); + } + + /** + * Read an instance of the second type parameter from buffer {@code src}. + */ + public B decodeB(PositionedByteRange src) { + return (B) decode(src); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union3.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union3.java new file mode 100644 index 0000000..b736b75 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union3.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * The {@code Union} family of {@link DataType}s encode one of a fixed + * collection of Objects. They provide convenience methods which handle type + * casting on your behalf. + * @see Union2 + */ +@SuppressWarnings("unchecked") +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class Union3 extends Union2 { + + protected final DataType typeC; + + /** + * Create an instance of {@code Union3} over the set of specified + * types. + */ + public Union3(DataType typeA, DataType typeB, DataType typeC) { + super(typeA, typeB); + this.typeC = typeC; + } + + @Override + public boolean isOrderPreserving() { + return super.isOrderPreserving() && typeC.isOrderPreserving(); + } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { + return super.isNullable() && typeC.isNullable(); + } + + @Override + public boolean isSkippable() { + return super.isSkippable() && typeC.isSkippable(); + } + + /** + * Read an instance of the third type parameter from buffer {@code src}. + */ + public C decodeC(PositionedByteRange src) { + return (C) decode(src); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union4.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union4.java new file mode 100644 index 0000000..c284e18 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union4.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; + +/** + * The {@code Union} family of {@link DataType}s encode one of a fixed + * collection of Objects. They provide convenience methods which handle type + * casting on your behalf. + */ +@SuppressWarnings("unchecked") +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class Union4 extends Union3 { + + protected final DataType typeD; + + /** + * Create an instance of {@code Union4} over the set of specified + * types. + */ + public Union4(DataType typeA, DataType typeB, DataType typeC, DataType typeD) { + super(typeA, typeB, typeC); + this.typeD = typeD; + } + + @Override + public boolean isOrderPreserving() { + return super.isOrderPreserving() && typeD.isOrderPreserving(); + } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { + return super.isNullable() && typeD.isNullable(); + } + + @Override + public boolean isSkippable() { + return super.isSkippable() && typeD.isSkippable(); + } + + /** + * Read an instance of the fourth type parameter from buffer {@code src}. + */ + public D decodeD(PositionedByteRange src) { + return (D) decode(src); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/package-info.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/package-info.java new file mode 100644 index 0000000..237760d --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/package-info.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + *

+ * This package provides the definition and implementation of HBase's + * extensible data type API. {@link org.apache.hadoop.hbase.types.DataType} + * is the entry point. Basic type implementations are provided based on two + * different encoding schemes: the {@code Raw*} implementations use the + * {@code toXXX} methods in {@link org.apache.hadoop.hbase.util.Bytes} and + * the {@code Ordered*} implementations use the encoding scheme defined in + * {@link org.apache.hadoop.hbase.util.OrderedBytes}. Complex types are also + * supported in the form of {@link org.apache.hadoop.hbase.types.Struct} and + * the abstract {@code Union} classes. + *

+ *

+ * {@link org.apache.hadoop.hbase.types.DataType} implementations are used to + * convert a POJO into a {@code byte[]} while maintaining application-level + * constraints over the values produces and consumed. They also provide hints + * to consumers about the nature of encoded values as well as the relationship + * between different instances. See the class comments on + * {@link org.apache.hadoop.hbase.types.DataType} for details. + *

+ *

+ * The {@link org.apache.hadoop.hbase.types.DataType} interface is primarily + * of use for creating rowkeys and column qualifiers. It can also be used as a + * an encoder for primitive values. It does not support concerns of complex + * object serialization, concepts like schema version and migration. These + * concepts are handled more thoroughly by tools like Thrift, Avro, and + * Protobuf. + *

+ * + * @since 0.95.2 + */ +package org.apache.hadoop.hbase.types; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestFixedLengthWrapper.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestFixedLengthWrapper.java new file mode 100644 index 0000000..40cfd4a --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestFixedLengthWrapper.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; +import org.apache.hadoop.hbase.util.SimplePositionedByteRange; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestFixedLengthWrapper { + + static final byte[][] VALUES = new byte[][] { + Bytes.toBytes(""), Bytes.toBytes("1"), Bytes.toBytes("22"), Bytes.toBytes("333"), + Bytes.toBytes("4444"), Bytes.toBytes("55555"), Bytes.toBytes("666666"), + Bytes.toBytes("7777777"), Bytes.toBytes("88888888"), Bytes.toBytes("999999999"), + }; + + /** + * all values of {@code limit} are >= max length of a member of + * {@code VALUES}. + */ + static final int[] limits = { 9, 12, 15 }; + + @Test + public void testReadWrite() { + for (int limit : limits) { + PositionedByteRange buff = new SimplePositionedByteRange(limit); + for (Order ord : new Order[] { Order.ASCENDING, Order.DESCENDING }) { + for (byte[] val : VALUES) { + buff.setPosition(0); + DataType type = new FixedLengthWrapper(new RawBytes(ord), limit); + assertEquals(limit, type.encode(buff, val)); + byte[] expected = Arrays.copyOf(val, limit); + buff.setPosition(0); + byte[] actual = type.decode(buff); + assertArrayEquals(expected, actual); + buff.setPosition(0); + assertEquals(limit, type.skip(buff)); + } + } + } + } + + @Test(expected = IllegalArgumentException.class) + public void testInsufficientRemainingRead() { + PositionedByteRange buff = new SimplePositionedByteRange(0); + DataType type = new FixedLengthWrapper(new RawBytes(), 3); + type.decode(buff); + } + + @Test(expected = IllegalArgumentException.class) + public void testInsufficientRemainingWrite() { + PositionedByteRange buff = new SimplePositionedByteRange(0); + DataType type = new FixedLengthWrapper(new RawBytes(), 3); + type.encode(buff, Bytes.toBytes("")); + } + + @Test(expected = IllegalArgumentException.class) + public void testOverflowPassthrough() { + PositionedByteRange buff = new SimplePositionedByteRange(3); + DataType type = new FixedLengthWrapper(new RawBytes(), 0); + type.encode(buff, Bytes.toBytes("foo")); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlob.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlob.java new file mode 100644 index 0000000..412bb2d --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlob.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.PositionedByteRange; +import org.apache.hadoop.hbase.util.SimplePositionedByteRange; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestOrderedBlob { + + static final byte[][] VALUES = new byte[][] { + null, Bytes.toBytes(""), Bytes.toBytes("1"), Bytes.toBytes("22"), Bytes.toBytes("333"), + Bytes.toBytes("4444"), Bytes.toBytes("55555"), Bytes.toBytes("666666"), + Bytes.toBytes("7777777"), Bytes.toBytes("88888888"), Bytes.toBytes("999999999"), + }; + + @Test + public void testEncodedLength() { + PositionedByteRange buff = new SimplePositionedByteRange(20); + for (DataType type : new OrderedBlob[] { OrderedBlob.ASCENDING, OrderedBlob.DESCENDING }) { + for (byte[] val : VALUES) { + buff.setPosition(0); + type.encode(buff, val); + assertEquals( + "encodedLength does not match actual, " + Bytes.toStringBinary(val), + buff.getPosition(), type.encodedLength(val)); + } + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlobVar.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlobVar.java new file mode 100644 index 0000000..18c6a66 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlobVar.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.PositionedByteRange; +import org.apache.hadoop.hbase.util.SimplePositionedByteRange; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestOrderedBlobVar { + + static final byte[][] VALUES = new byte[][] { + null, Bytes.toBytes(""), Bytes.toBytes("1"), Bytes.toBytes("22"), Bytes.toBytes("333"), + Bytes.toBytes("4444"), Bytes.toBytes("55555"), Bytes.toBytes("666666"), + Bytes.toBytes("7777777"), Bytes.toBytes("88888888"), Bytes.toBytes("999999999"), + }; + + @Test + public void testEncodedLength() { + PositionedByteRange buff = new SimplePositionedByteRange(20); + for (DataType type : + new OrderedBlobVar[] { OrderedBlobVar.ASCENDING, OrderedBlobVar.DESCENDING }) { + for (byte[] val : VALUES) { + buff.setPosition(0); + type.encode(buff, val); + assertEquals( + "encodedLength does not match actual, " + Bytes.toStringBinary(val), + buff.getPosition(), type.encodedLength(val)); + } + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedString.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedString.java new file mode 100644 index 0000000..68c6f55 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedString.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.PositionedByteRange; +import org.apache.hadoop.hbase.util.SimplePositionedByteRange; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestOrderedString { + + static final String[] VALUES = + new String[] { null, "", "1", "22", "333", "4444", "55555", "666666", + "7777777", "88888888", "999999999" }; + + @Test + public void testEncodedLength() { + PositionedByteRange buff = new SimplePositionedByteRange(20); + for (DataType type : new OrderedString[] { OrderedString.ASCENDING, OrderedString.DESCENDING }) { + for (String val : VALUES) { + buff.setPosition(0); + type.encode(buff, val); + assertEquals( + "encodedLength does not match actual, " + val, + buff.getPosition(), type.encodedLength(val)); + } + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestRawString.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestRawString.java new file mode 100644 index 0000000..dc00f2c --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestRawString.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; +import org.apache.hadoop.hbase.util.SimplePositionedByteRange; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestRawString { + + static final String[] VALUES = new String[] { + "", "1", "22", "333", "4444", "55555", "666666", "7777777", "88888888", "999999999", + }; + + @Test + public void testReadWrite() { + for (Order ord : new Order[] { Order.ASCENDING, Order.DESCENDING }) { + RawString type = + Order.ASCENDING == ord ? RawString.ASCENDING : RawString.DESCENDING; + for (String val : VALUES) { + PositionedByteRange buff = new SimplePositionedByteRange(Bytes.toBytes(val).length); + assertEquals(buff.getLength(), type.encode(buff, val)); + byte[] expected = Bytes.toBytes(val); + ord.apply(expected); + assertArrayEquals(expected, buff.getBytes()); + buff.setPosition(0); + assertEquals(val, type.decode(buff)); + buff.setPosition(0); + assertEquals(buff.getLength(), type.skip(buff)); + assertEquals(buff.getLength(), buff.getPosition()); + } + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestStruct.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestStruct.java new file mode 100644 index 0000000..9c73171 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestStruct.java @@ -0,0 +1,364 @@ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.lang.reflect.Constructor; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; +import org.apache.hadoop.hbase.util.SimplePositionedByteRange; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * This class both tests and demonstrates how to construct compound rowkeys + * from a POJO. The code under test is {@link Struct}. + * {@link SpecializedPojo1Type1} demonstrates how one might create their own + * custom data type extension for an application POJO. + */ +@RunWith(Parameterized.class) +@Category(SmallTests.class) +public class TestStruct { + + private Struct generic; + @SuppressWarnings("rawtypes") + private DataType specialized; + private Object[][] constructorArgs; + + public TestStruct(Struct generic, @SuppressWarnings("rawtypes") DataType specialized, + Object[][] constructorArgs) { + this.generic = generic; + this.specialized = specialized; + this.constructorArgs = constructorArgs; + } + + @Parameters + public static Collection params() { + Object[][] pojo1Args = { + new Object[] { "foo", 5, 10.001 }, + new Object[] { "foo", 100, 7.0 }, + new Object[] { "foo", 100, 10.001 }, + new Object[] { "bar", 5, 10.001 }, + new Object[] { "bar", 100, 10.001 }, + new Object[] { "baz", 5, 10.001 }, + }; + + Object[][] pojo2Args = { + new Object[] { new byte[0], "it".getBytes(), "was", "the".getBytes() }, + new Object[] { "best".getBytes(), new byte[0], "of", "times,".getBytes() }, + new Object[] { "it".getBytes(), "was".getBytes(), "", "the".getBytes() }, + new Object[] { "worst".getBytes(), "of".getBytes(), "times,", new byte[0] }, + new Object[] { new byte[0], new byte[0], "", new byte[0] }, + }; + + Object[][] params = new Object[][] { + { SpecializedPojo1Type1.GENERIC, new SpecializedPojo1Type1(), pojo1Args }, + { SpecializedPojo2Type1.GENERIC, new SpecializedPojo2Type1(), pojo2Args }, + }; + return Arrays.asList(params); + } + + static final Comparator NULL_SAFE_BYTES_COMPARATOR = + new Comparator() { + @Override + public int compare(byte[] o1, byte[] o2) { + if (o1 == o2) return 0; + if (null == o1) return -1; + if (null == o2) return 1; + return Bytes.compareTo(o1, o2); + } + }; + + /** + * A simple object to serialize. + */ + private static class Pojo1 implements Comparable { + final String stringFieldAsc; + final int intFieldAsc; + final double doubleFieldAsc; + final transient String str; + + public Pojo1(Object... argv) { + stringFieldAsc = (String) argv[0]; + intFieldAsc = (Integer) argv[1]; + doubleFieldAsc = (Double) argv[2]; + str = new StringBuilder() + .append("{ ") + .append(null == stringFieldAsc ? "" : "\"") + .append(stringFieldAsc) + .append(null == stringFieldAsc ? "" : "\"").append(", ") + .append(intFieldAsc).append(", ") + .append(doubleFieldAsc) + .append(" }") + .toString(); + } + + @Override + public String toString() { + return str; + } + + @Override + public int compareTo(Pojo1 o) { + int cmp = stringFieldAsc.compareTo(o.stringFieldAsc); + if (cmp != 0) return cmp; + cmp = Integer.valueOf(intFieldAsc).compareTo(Integer.valueOf(o.intFieldAsc)); + if (cmp != 0) return cmp; + return Double.compare(doubleFieldAsc, o.doubleFieldAsc); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (null == o) return false; + if (!(o instanceof Pojo1)) return false; + Pojo1 that = (Pojo1) o; + return 0 == this.compareTo(that); + } + } + + /** + * A simple object to serialize. + */ + private static class Pojo2 implements Comparable { + final byte[] byteField1Asc; + final byte[] byteField2Dsc; + final String stringFieldDsc; + final byte[] byteField3Dsc; + final transient String str; + + public Pojo2(Object... vals) { + byte[] empty = new byte[0]; + byteField1Asc = vals.length > 0 ? (byte[]) vals[0] : empty; + byteField2Dsc = vals.length > 1 ? (byte[]) vals[1] : empty; + stringFieldDsc = vals.length > 2 ? (String) vals[2] : ""; + byteField3Dsc = vals.length > 3 ? (byte[]) vals[3] : empty; + str = new StringBuilder() + .append("{ ") + .append(Bytes.toStringBinary(byteField1Asc)).append(", ") + .append(Bytes.toStringBinary(byteField2Dsc)).append(", ") + .append(null == stringFieldDsc ? "" : "\"") + .append(stringFieldDsc) + .append(null == stringFieldDsc ? "" : "\"").append(", ") + .append(Bytes.toStringBinary(byteField3Dsc)) + .append(" }") + .toString(); + } + + @Override + public String toString() { + return str; + } + + @Override + public int compareTo(Pojo2 o) { + int cmp = NULL_SAFE_BYTES_COMPARATOR.compare(byteField1Asc, o.byteField1Asc); + if (cmp != 0) return cmp; + cmp = -NULL_SAFE_BYTES_COMPARATOR.compare(byteField2Dsc, o.byteField2Dsc); + if (cmp != 0) return cmp; + if (stringFieldDsc == o.stringFieldDsc) cmp = 0; + else if (null == stringFieldDsc) cmp = 1; + else if (null == o.stringFieldDsc) cmp = -1; + else cmp = -stringFieldDsc.compareTo(o.stringFieldDsc); + if (cmp != 0) return cmp; + return -NULL_SAFE_BYTES_COMPARATOR.compare(byteField3Dsc, o.byteField3Dsc); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (null == o) return false; + if (!(o instanceof Pojo2)) return false; + Pojo2 that = (Pojo2) o; + return 0 == this.compareTo(that); + } + } + + /** + * A custom data type implementation specialized for {@link Pojo1}. + */ + private static class SpecializedPojo1Type1 implements DataType { + + private static final RawStringTerminated stringField = new RawStringTerminated("/"); + private static final RawInteger intField = new RawInteger(); + private static final RawDouble doubleField = new RawDouble(); + + /** + * The {@link Struct} equivalent of this type. + */ + public static Struct GENERIC = + new StructBuilder().add(stringField) + .add(intField) + .add(doubleField) + .toStruct(); + + @Override + public boolean isOrderPreserving() { return true; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(Pojo1 val) { + return + stringField.encodedLength(val.stringFieldAsc) + + intField.encodedLength(val.intFieldAsc) + + doubleField.encodedLength(val.doubleFieldAsc); + } + + @Override + public Class encodedClass() { return Pojo1.class; } + + @Override + public int skip(PositionedByteRange src) { + int skipped = stringField.skip(src); + skipped += intField.skip(src); + skipped += doubleField.skip(src); + return skipped; + } + + @Override + public Pojo1 decode(PositionedByteRange src) { + Object[] ret = new Object[3]; + ret[0] = stringField.decode(src); + ret[1] = intField.decode(src); + ret[2] = doubleField.decode(src); + return new Pojo1(ret); + } + + @Override + public int encode(PositionedByteRange dst, Pojo1 val) { + int written = stringField.encode(dst, val.stringFieldAsc); + written += intField.encode(dst, val.intFieldAsc); + written += doubleField.encode(dst, val.doubleFieldAsc); + return written; + } + } + + /** + * A custom data type implementation specialized for {@link Pojo2}. + */ + private static class SpecializedPojo2Type1 implements DataType { + + private static RawBytesTerminated byteField1 = new RawBytesTerminated("/"); + private static RawBytesTerminated byteField2 = + new RawBytesTerminated(Order.DESCENDING, "/"); + private static RawStringTerminated stringField = + new RawStringTerminated(Order.DESCENDING, new byte[] { 0x00 }); + private static RawBytes byteField3 = RawBytes.DESCENDING; + + /** + * The {@link Struct} equivalent of this type. + */ + public static Struct GENERIC = + new StructBuilder().add(byteField1) + .add(byteField2) + .add(stringField) + .add(byteField3) + .toStruct(); + + @Override + public boolean isOrderPreserving() { return true; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(Pojo2 val) { + return + byteField1.encodedLength(val.byteField1Asc) + + byteField2.encodedLength(val.byteField2Dsc) + + stringField.encodedLength(val.stringFieldDsc) + + byteField3.encodedLength(val.byteField3Dsc); + } + + @Override + public Class encodedClass() { return Pojo2.class; } + + @Override + public int skip(PositionedByteRange src) { + int skipped = byteField1.skip(src); + skipped += byteField2.skip(src); + skipped += stringField.skip(src); + skipped += byteField3.skip(src); + return skipped; + } + + @Override + public Pojo2 decode(PositionedByteRange src) { + Object[] ret = new Object[4]; + ret[0] = byteField1.decode(src); + ret[1] = byteField2.decode(src); + ret[2] = stringField.decode(src); + ret[3] = byteField3.decode(src); + return new Pojo2(ret); + } + + @Override + public int encode(PositionedByteRange dst, Pojo2 val) { + int written = byteField1.encode(dst, val.byteField1Asc); + written += byteField2.encode(dst, val.byteField2Dsc); + written += stringField.encode(dst, val.stringFieldDsc); + written += byteField3.encode(dst, val.byteField3Dsc); + return written; + } + } + + @Test + @SuppressWarnings("unchecked") + public void testOrderPreservation() throws Exception { + Object[] vals = new Object[constructorArgs.length]; + PositionedByteRange[] encodedGeneric = new PositionedByteRange[constructorArgs.length]; + PositionedByteRange[] encodedSpecialized = new PositionedByteRange[constructorArgs.length]; + Constructor ctor = specialized.encodedClass().getConstructor(Object[].class); + for (int i = 0; i < vals.length; i++) { + vals[i] = ctor.newInstance(new Object[] { constructorArgs[i] }); + encodedGeneric[i] = new SimplePositionedByteRange(generic.encodedLength(constructorArgs[i])); + encodedSpecialized[i] = new SimplePositionedByteRange(specialized.encodedLength(vals[i])); + } + + // populate our arrays + for (int i = 0; i < vals.length; i++) { + generic.encode(encodedGeneric[i], constructorArgs[i]); + encodedGeneric[i].setPosition(0); + specialized.encode(encodedSpecialized[i], vals[i]); + encodedSpecialized[i].setPosition(0); + assertArrayEquals(encodedGeneric[i].getBytes(), encodedSpecialized[i].getBytes()); + } + + Arrays.sort(vals); + Arrays.sort(encodedGeneric); + Arrays.sort(encodedSpecialized); + + for (int i = 0; i < vals.length; i++) { + assertEquals( + "Struct encoder does not preserve sort order at position " + i, + vals[i], + ctor.newInstance(new Object[] { generic.decode(encodedGeneric[i]) })); + assertEquals( + "Specialized encoder does not preserve sort order at position " + i, + vals[i], specialized.decode(encodedSpecialized[i])); + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestTerminatedWrapper.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestTerminatedWrapper.java new file mode 100644 index 0000000..7e7de21 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestTerminatedWrapper.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; +import org.apache.hadoop.hbase.util.SimplePositionedByteRange; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestTerminatedWrapper { + + static final byte[][] VALUES = new byte[][] { + Bytes.toBytes(""), Bytes.toBytes("1"), Bytes.toBytes("22"), Bytes.toBytes("333"), + Bytes.toBytes("4444"), Bytes.toBytes("55555"), Bytes.toBytes("666666"), + Bytes.toBytes("7777777"), Bytes.toBytes("88888888"), Bytes.toBytes("999999999"), + }; + + static final byte[][] TERMINATORS = new byte[][] { new byte[] { -1 }, Bytes.toBytes("foo") }; + + @Test(expected = IllegalArgumentException.class) + public void testEmptyDelimiter() { + new TerminatedWrapper(new RawBytes(), ""); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullDelimiter() { + new RawBytesTerminated((byte[]) null); + // new TerminatedWrapper(new RawBytes(), (byte[]) null); + } + + @Test(expected = IllegalArgumentException.class) + public void testEncodedValueContainsTerm() { + DataType type = new TerminatedWrapper(new RawBytes(), "foo"); + PositionedByteRange buff = new SimplePositionedByteRange(16); + type.encode(buff, Bytes.toBytes("hello foobar!")); + } + + @Test + public void testReadWrite() { + PositionedByteRange buff = new SimplePositionedByteRange(12); + for (Order ord : new Order[] { Order.ASCENDING, Order.DESCENDING }) { + for (byte[] term : TERMINATORS) { + for (byte[] val : VALUES) { + buff.setPosition(0); + DataType type = new TerminatedWrapper(new RawBytes(ord), term); + assertEquals(val.length + term.length, type.encode(buff, val)); + buff.setPosition(0); + assertArrayEquals(val, type.decode(buff)); + assertEquals(val.length + term.length, buff.getPosition()); + } + } + } + } + + @Test + public void testSkip() { + PositionedByteRange buff = new SimplePositionedByteRange(12); + for (Order ord : new Order[] { Order.ASCENDING, Order.DESCENDING }) { + for (byte[] term : TERMINATORS) { + for (byte[] val : VALUES) { + buff.setPosition(0); + DataType type = new TerminatedWrapper(new RawBytes(ord), term); + int expected = type.encode(buff, val); + buff.setPosition(0); + assertEquals(expected, type.skip(buff)); + assertEquals(expected, buff.getPosition()); + } + } + } + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidSkip() { + PositionedByteRange buff = new SimplePositionedByteRange(Bytes.toBytes("foo")); + DataType type = new TerminatedWrapper(new RawBytes(), new byte[] { 0x00 }); + type.skip(buff); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestUnion2.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestUnion2.java new file mode 100644 index 0000000..6396193 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestUnion2.java @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.PositionedByteRange; +import org.apache.hadoop.hbase.util.SimplePositionedByteRange; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestUnion2 { + + /** + * An example Union + */ + private static class SampleUnion1 extends Union2 { + + private static final byte IS_INTEGER = 0x00; + private static final byte IS_STRING = 0x01; + + public SampleUnion1() { + super(new RawInteger(), new RawStringTerminated(Order.DESCENDING, ".")); + } + + @Override + public int skip(PositionedByteRange src) { + switch (src.get()) { + case IS_INTEGER: + return 1 + typeA.skip(src); + case IS_STRING: + return 1 + typeB.skip(src); + default: + throw new IllegalArgumentException("Unrecognized encoding format."); + } + } + + @Override + public Object decode(PositionedByteRange src) { + switch (src.get()) { + case IS_INTEGER: + return typeA.decode(src); + case IS_STRING: + return typeB.decode(src); + default: + throw new IllegalArgumentException("Unrecognized encoding format."); + } + } + + @Override + public int encodedLength(Object val) { + Integer i = null; + String s = null; + try { + i = (Integer) val; + } catch (ClassCastException e) {} + try { + s = (String) val; + } catch (ClassCastException e) {} + + if (null != i) return 1 + typeA.encodedLength(i); + if (null != s) return 1 + typeB.encodedLength(s); + throw new IllegalArgumentException("val is not a valid member of this union."); + } + + @Override + public int encode(PositionedByteRange dst, Object val) { + Integer i = null; + String s = null; + try { + i = (Integer) val; + } catch (ClassCastException e) {} + try { + s = (String) val; + } catch (ClassCastException e) {} + + if (null != i) { + dst.put(IS_INTEGER); + return 1 + typeA.encode(dst, i); + } else if (null != s) { + dst.put(IS_STRING); + return 1 + typeB.encode(dst, s); + } + else + throw new IllegalArgumentException("val is not of a supported type."); + } + } + + @Test + public void testEncodeDecode() { + Integer intVal = Integer.valueOf(10); + String strVal = "hello"; + PositionedByteRange buff = new SimplePositionedByteRange(10); + SampleUnion1 type = new SampleUnion1(); + + type.encode(buff, intVal); + buff.setPosition(0); + assertTrue(0 == intVal.compareTo(type.decodeA(buff))); + buff.setPosition(0); + type.encode(buff, strVal); + buff.setPosition(0); + assertTrue(0 == strVal.compareTo(type.decodeB(buff))); + } + + @Test + public void testSkip() { + Integer intVal = Integer.valueOf(10); + String strVal = "hello"; + PositionedByteRange buff = new SimplePositionedByteRange(10); + SampleUnion1 type = new SampleUnion1(); + + int len = type.encode(buff, intVal); + buff.setPosition(0); + assertEquals(len, type.skip(buff)); + buff.setPosition(0); + len = type.encode(buff, strVal); + buff.setPosition(0); + assertEquals(len, type.skip(buff)); + } +} -- 1.8.3.2