From 65620c6b93f04c41f39413382b0cfc056713d573 Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Fri, 2 Aug 2013 14:39:46 -0700 Subject: [PATCH] HBASE-8693 Extensible data types API This patch introduces an extensible data types API for HBase. It is inspired by the following systems: - PostgreSQL. Postgres has a user-extensible data type API, which has been used to great effect by it's user community (ie, PostGIS). The desire is for HBase to expose an equally extensible data type API. One aspect of the Postgres data type is the ability to provide equivalence functions for index operations. This appears to be of critical performance utility for its execution engine. - Orderly. Orderly handles the issue of compound rowkeys by providing convenience classes for handling these kinds of data types. This influence is reflected in the Struct and Union family of classes. - Phoenix. The PDataType enum used in Phoenix provides type hints, similar Postgres's equivalence functions. These appear to be used during query execution for numerical type promotion. This patch introduces an interface, DataType, along with a number of data type implementations based on the Bytes encoding. Also included are Struct and Union types, demonstrating simple implementations of compound types. Helper classes around the Struct implementation are also provided. This patch does not address the type compatibility concerns expressed by Phoenix's PDataType API (ie, isComparableTo, isCoercibleTo); these will be addressed in HBASE-8863. This patch also provides DataType implementations based on the OrderedBytes encoding from HBASE-8201. --- .../org/apache/hadoop/hbase/types/DataType.java | 105 ++++++ .../hadoop/hbase/types/FixedLengthWrapper.java | 88 +++++ .../org/apache/hadoop/hbase/types/LegacyBytes.java | 96 ++++++ .../hadoop/hbase/types/LegacyBytesFixedLength.java | 67 ++++ .../hadoop/hbase/types/LegacyBytesTerminated.java | 87 +++++ .../apache/hadoop/hbase/types/LegacyDouble.java | 84 +++++ .../org/apache/hadoop/hbase/types/LegacyFloat.java | 82 +++++ .../apache/hadoop/hbase/types/LegacyInteger.java | 82 +++++ .../org/apache/hadoop/hbase/types/LegacyLong.java | 82 +++++ .../apache/hadoop/hbase/types/LegacyString.java | 79 +++++ .../hbase/types/LegacyStringFixedLength.java | 52 +++ .../hadoop/hbase/types/LegacyStringTerminated.java | 72 ++++ .../org/apache/hadoop/hbase/types/OrderedBlob.java | 66 ++++ .../apache/hadoop/hbase/types/OrderedBlobVar.java | 62 ++++ .../hadoop/hbase/types/OrderedBytesBase.java | 55 ++++ .../apache/hadoop/hbase/types/OrderedFloat32.java | 71 ++++ .../apache/hadoop/hbase/types/OrderedFloat64.java | 71 ++++ .../apache/hadoop/hbase/types/OrderedInt32.java | 71 ++++ .../apache/hadoop/hbase/types/OrderedInt64.java | 71 ++++ .../apache/hadoop/hbase/types/OrderedNumeric.java | 103 ++++++ .../apache/hadoop/hbase/types/OrderedString.java | 54 +++ .../java/org/apache/hadoop/hbase/types/Struct.java | 163 ++++++++++ .../apache/hadoop/hbase/types/StructBuilder.java | 54 +++ .../apache/hadoop/hbase/types/StructIterator.java | 95 ++++++ .../hadoop/hbase/types/TerminatedWrapper.java | 141 ++++++++ .../java/org/apache/hadoop/hbase/types/Union2.java | 85 +++++ .../java/org/apache/hadoop/hbase/types/Union3.java | 70 ++++ .../java/org/apache/hadoop/hbase/types/Union4.java | 69 ++++ .../hadoop/hbase/types/TestFixedLengthWrapper.java | 83 +++++ .../hadoop/hbase/types/TestLegacyString.java | 46 +++ .../apache/hadoop/hbase/types/TestOrderedBlob.java | 50 +++ .../hadoop/hbase/types/TestOrderedBlobVar.java | 51 +++ .../hadoop/hbase/types/TestOrderedString.java | 47 +++ .../org/apache/hadoop/hbase/types/TestStruct.java | 361 +++++++++++++++++++++ .../hadoop/hbase/types/TestTerminatedWrapper.java | 92 ++++++ .../org/apache/hadoop/hbase/types/TestUnion2.java | 131 ++++++++ 36 files changed, 3138 insertions(+) create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/DataType.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedLengthWrapper.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytes.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytesFixedLength.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytesTerminated.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyDouble.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyFloat.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyInteger.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyLong.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyString.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyStringFixedLength.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyStringTerminated.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlob.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlobVar.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBytesBase.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat32.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat64.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt32.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt64.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedNumeric.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedString.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructBuilder.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructIterator.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/TerminatedWrapper.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union2.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union3.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union4.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestFixedLengthWrapper.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestLegacyString.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlob.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlobVar.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedString.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestStruct.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestTerminatedWrapper.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestUnion2.java diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/DataType.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/DataType.java new file mode 100644 index 0000000..c43d4b9 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/DataType.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; + +/** + * HDataType is the base class for all HBase data types. Data + * type implementations are designed to be serialized to and deserialized from + * byte[]. Serialized representations can retain the natural sort ordering of + * the source object, when a suitable encoding is supported by the underlying + * implementation. This is a desirable feature for use in rowkeys and column + * qualifiers. + *

+ * Data type instances are designed to be stateless, thread-safe, and reused. + * Implementations should provide static final instances + * corresponding to each variation on configurable parameters. For instance, + * order-preserving types should provide static ASCENDING and DESCENDING + * instances. It is also encouraged for implementations operating on Java + * primitive types to provide primitive implementations of the + * read and write methods. This advice is a + * performance consideration to clients reading and writing values in tight + * loops. + *

+ */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public interface DataType { + + /** + * Indicates whether this instance writes encoded byte[]'s + * which preserve the natural sort order of the unencoded value. + * @return true when natural order is preserved, + * false otherwise. + */ + public boolean isOrderPreserving(); + + /** + * Retrieve the sort {@link Order} honored by this data type, or + * null when natural ordering is not preserved. + */ + public Order getOrder(); + + /** + * Indicates whether this instance supports encoding null + * values. This depends on the implementation details of the encoding + * format. All HDataTypes that support null should + * treat null as comparing less than any non-NULL + * value for default sort ordering purposes. + * @return true when null is supported, + * false otherwise. + */ + public boolean isNullable(); + + /** + * Indicates whether this instance is able to skip over it's encoded value. + * HDataTypes that are not skippable can only be used as the + * right-most field of a {@link Struct}. + */ + public boolean isSkippable(); + + /** + * Inform consumers how long the encoded byte[] will be. + */ + public int encodedLength(T val); + + /** + * Inform consumers over what type this HDataType operates. + */ + public Class encodedClass(); + + /** + * Skip offset forward over one encoded value. + * @return updated offset. + */ + public int skip(byte[] buff, int offset); + + /** + * Read an instance of T from the buffer buff. + */ + public T decode(byte[] buff, int offset); + + /** + * Write instance val into buffer buff. + * @return updated offset. + */ + public int encode(byte[] buff, int offset, T val); +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedLengthWrapper.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedLengthWrapper.java new file mode 100644 index 0000000..c1ffe7d --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedLengthWrapper.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; + +/** + * Wraps an existing HDataType implementation as a fixed-length + * version of itself. This has the useful side-effect of turning an existing + * HDataType which is not skippable into a + * skippable variant. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class FixedLengthWrapper implements DataType { + + protected final DataType wrapped; + protected final int length; + + /** + * Create a fixed-length version of the wrapped. + */ + public FixedLengthWrapper(DataType wrapped, int length) { + this.wrapped = wrapped; + this.length = length; + } + + @Override + public boolean isOrderPreserving() { return wrapped.isOrderPreserving(); } + + @Override + public Order getOrder() { return wrapped.getOrder(); } + + @Override + public boolean isNullable() { return wrapped.isNullable(); } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(T val) { return length; } + + @Override + public Class encodedClass() { return wrapped.encodedClass(); } + + @Override + public int skip(byte[] buff, int offset) { return offset + length; } + + @Override + public T decode(byte[] buff, int offset) { + if (buff.length - offset < length) + throw new IllegalArgumentException("Not enough buffer remaining."); + // create a copy range limited to length bytes. boo. + byte[] b = new byte[length]; + System.arraycopy(buff, offset, b, 0, length); + return wrapped.decode(b, offset); + } + + @Override + public int encode(byte[] buff, int offset, T val) { + if (buff.length - offset < length) + throw new IllegalArgumentException("Not enough buffer remaining."); + int start = offset; + offset = wrapped.encode(buff, offset, val); + if (offset - start > length) + throw new IllegalArgumentException("Encoded value exceeds length."); + // is the zero-padding appropriate? + while (offset < length) buff[offset++] = (byte) 0x00; + return offset; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytes.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytes.java new file mode 100644 index 0000000..a3effa8 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytes.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An HDataType for interacting with variable-length values + * encoded using {@link Bytes#putBytes(byte[], int, byte[], int, int)}. + * Intended to make it easier to transition away from direct use of + * {@link Bytes}. + * @see Bytes#putBytes(byte[], int, byte[], int, int) + * @see LegacyBytesTerminated + * @see OrderedBlob + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class LegacyBytes implements DataType { + + public static final LegacyBytes ASCENDING = new LegacyBytes(Order.ASCENDING); + public static final LegacyBytes DESCENDING = new LegacyBytes(Order.DESCENDING); + + protected final Order order; + + protected LegacyBytes() { this.order = Order.ASCENDING; } + protected LegacyBytes(Order order) { this.order = order; } + + @Override + public boolean isOrderPreserving() { return true; } + + @Override + public Order getOrder() { return order; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return false; } + + @Override + public int skip(byte[] buff, int offset) { return buff.length; } + + @Override + public int encodedLength(byte[] val) { return val.length; } + + @Override + public Class encodedClass() { return byte[].class; } + + @Override + public byte[] decode(byte[] buff, int offset) { + return decode(buff, offset, buff.length - offset); + } + + @Override + public int encode(byte[] buff, int offset, byte[] val) { + System.arraycopy(val, 0, buff, offset, val.length); + return offset + val.length; + } + + /** + * Read a byte[] from the buffer buff. + */ + public byte[] decode(byte[] buff, int offset, int length) { + byte[] val = new byte[length]; + System.arraycopy(buff, offset, val, 0, length); + return val; + } + + /** + * Write val into buff, respecting + * offset and length. + * @return updated offset. + */ + public int encode(byte[] buff, int offset, byte[] val, int voff, int length) { + System.arraycopy(val, voff, buff, offset, length); + return offset + length; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytesFixedLength.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytesFixedLength.java new file mode 100644 index 0000000..93cacf7 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytesFixedLength.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An HDataType that encodes fixed-length values encoded using + * {@link Bytes#putBytes(byte[], int, byte[], int, int)}. Intended to make it + * easier to transition away from direct use of {@link Bytes}. + * @see Bytes#putBytes(byte[], int, byte[], int, int) + * @see LegacyBytes + * @see OrderedBlob + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class LegacyBytesFixedLength extends FixedLengthWrapper { + + /** + * Create a LegacyBytesFixedLength using the specified + * order and length. + */ + public LegacyBytesFixedLength(Order order, int length) { + super(new LegacyBytes(order), length); + } + + /** + * Create a LegacyBytesFixedLength of the specified + * length>. + */ + public LegacyBytesFixedLength(int length) { + super(new LegacyBytes(), length); + } + + /** + * Read a byte[] from the buffer buff. + */ + public byte[] decode(byte[] buff, int offset, int length) { + return ((LegacyBytes) wrapped).decode(buff, offset, length); + } + + /** + * Write val into buff, respecting + * offset and length. + */ + public int encode(byte[] buff, int offset, byte[] val, int voff, int length) { + return ((LegacyBytes) wrapped).encode(buff, offset, val, voff, length); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytesTerminated.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytesTerminated.java new file mode 100644 index 0000000..c285c2e --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytesTerminated.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An HDataType that encodes variable-length values encoded using + * {@link Bytes#putBytes(byte[], int, byte[], int, int)}. Includes a + * termination marker following the raw byte[] value. Intended to + * make it easier to transition away from direct use of {@link Bytes}. + * @see Bytes#putBytes(byte[], int, byte[], int, int) + * @see LegacyBytes + * @see OrderedBlob + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class LegacyBytesTerminated extends TerminatedWrapper { + + /** + * Create a LegacyBytesTerminated using the specified terminator and + * order. + * @throws IllegalArgumentException if term is null or empty. + */ + public LegacyBytesTerminated(Order order, byte[] term) { + super(new LegacyBytes(order), term); + } + + /** + * Create a LegacyBytesTerminated using the specified terminator and + * order. + * @throws IllegalArgumentException if term is null or empty. + */ + public LegacyBytesTerminated(Order order, String term) { + super(new LegacyBytes(order), term); + } + + /** + * Create a LegacyBytesTerminated using the specified terminator. + * @throws IllegalArgumentException if term is null or empty. + */ + public LegacyBytesTerminated(byte[] term) { + super(new LegacyBytes(), term); + } + + /** + * Create a LegacyBytesTerminated using the specified terminator. + * @throws IllegalArgumentException if term is null or empty. + */ + public LegacyBytesTerminated(String term) { + super(new LegacyBytes(), term); + } + + /** + * Read a byte[] from the buffer buff. + */ + public byte[] decode(byte[] buff, int offset, int length) { + return ((LegacyBytes) wrapped).decode(buff, offset, length); + } + + /** + * Write val into buff, respecting + * offset and length. + * @return updated offset. + */ + public int encode(byte[] buff, int offset, byte[] val, int voff, int vlen) { + return ((LegacyBytes) wrapped).encode(buff, offset, val, voff, vlen); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyDouble.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyDouble.java new file mode 100644 index 0000000..0411929 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyDouble.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An DataType for interacting with values encoded using + * {@link Bytes#putDouble(byte[], int, double)}. Intended to make it easier to + * transition away from direct use of {@link Bytes}. + * @see Bytes#putDouble(byte[], int, double) + * @see Bytes#toDouble(byte[]) + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class LegacyDouble implements DataType { + + @Override + public boolean isOrderPreserving() { return false; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(Double val) { return Bytes.SIZEOF_DOUBLE; } + + @Override + public Class encodedClass() { return Double.class; } + + @Override + public int skip(byte[] buff, int offset) { + return offset + Bytes.SIZEOF_DOUBLE; + } + + @Override + public Double decode(byte[] buff, int offset) { + double val = Bytes.toDouble(buff, offset); + return val; + } + + @Override + public int encode(byte[] buff, int offset, Double val) { + return Bytes.putDouble(buff, offset, val); + } + + /** + * Read a double value from the buffer buff. + */ + public double decodeDouble(byte[] buff, int offset) { + double val = Bytes.toDouble(buff, offset); + return val; + } + + /** + * Write instance val into buffer buff. + */ + public int encodeDouble(byte[] buff, int offset, double val) { + return Bytes.putDouble(buff, offset, val); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyFloat.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyFloat.java new file mode 100644 index 0000000..efb938a --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyFloat.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An DataType for interacting with values encoded using + * {@link Bytes#putFloat(byte[], int, float)}. Intended to make it easier to + * transition away from direct use of {@link Bytes}. + * @see Bytes#putFloat(byte[], int, float) + * @see Bytes#toFloat(byte[]) + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class LegacyFloat implements DataType { + + @Override + public boolean isOrderPreserving() { return false; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(Float val) { return Bytes.SIZEOF_FLOAT; } + + @Override + public Class encodedClass() { return Float.class; } + + @Override + public int skip(byte[] buff, int offset) { + return offset + Bytes.SIZEOF_FLOAT; + } + + @Override + public Float decode(byte[] buff, int offset) { + return Bytes.toFloat(buff, offset); + } + + @Override + public int encode(byte[] buff, int offset, Float val) { + return Bytes.putFloat(buff, offset, val); + } + + /** + * Read a float value from the buffer buff. + */ + public float decodeFloat(byte[] buff, int offset) { + return Bytes.toFloat(buff, offset); + } + + /** + * Write instance val into buffer buff. + */ + public int encodeFloat(byte[] buff, int offset, float val) { + return Bytes.putFloat(buff, offset, val); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyInteger.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyInteger.java new file mode 100644 index 0000000..56c08a8 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyInteger.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An DataType for interacting with values encoded using + * {@link Bytes#putInt(byte[], int, int)}. Intended to make it easier to + * transition away from direct use of {@link Bytes}. + * @see Bytes#putInt(byte[], int, int) + * @see Bytes#toInt(byte[]) + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class LegacyInteger implements DataType { + + @Override + public boolean isOrderPreserving() { return false; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(Integer val) { return Bytes.SIZEOF_INT; } + + @Override + public Class encodedClass() { return Integer.class; } + + @Override + public int skip(byte[] buff, int offset) { + return offset + Bytes.SIZEOF_INT; + } + + @Override + public Integer decode(byte[] buff, int offset) { + return Bytes.toInt(buff, offset); + } + + @Override + public int encode(byte[] buff, int offset, Integer val) { + return Bytes.putInt(buff, offset, val); + } + + /** + * Read an int value from the buffer buff. + */ + public int decodeInt(byte[] buff, int offset) { + return Bytes.toInt(buff, offset); + } + + /** + * Write instance val into buffer buff. + */ + public int encodeInt(byte[] buff, int offset, int val) { + return Bytes.putInt(buff, offset, val); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyLong.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyLong.java new file mode 100644 index 0000000..ca656a3 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyLong.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An DataType for interacting with values encoded using + * {@link Bytes#putLong(byte[], int, long)}. Intended to make it easier to + * transition away from direct use of {@link Bytes}. + * @see Bytes#putLong(byte[], int, long) + * @see Bytes#toLong(byte[]) + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class LegacyLong implements DataType { + + @Override + public boolean isOrderPreserving() { return false; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(Long val) { return Bytes.SIZEOF_LONG; } + + @Override + public Class encodedClass() { return Long.class; } + + @Override + public int skip(byte[] buff, int offset) { + return offset + Bytes.SIZEOF_LONG; + } + + @Override + public Long decode(byte[] buff, int offset) { + return Bytes.toLong(buff, offset); + } + + @Override + public int encode(byte[] buff, int offset, Long val) { + return Bytes.putLong(buff, offset, val); + } + + /** + * Read a long value from the buffer buff. + */ + public long decodeLong(byte[] buff, int offset) { + return Bytes.toLong(buff, offset); + } + + /** + * Write instance val into buffer buff. + */ + public int encodeLong(byte[] buff, int offset, long val) { + return Bytes.putLong(buff, offset, val); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyString.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyString.java new file mode 100644 index 0000000..6fd9233 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyString.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An DataType for interacting with values encoded using + * {@link Bytes#toBytes(String)}. Intended to make it easier to transition + * away from direct use of {@link Bytes}. + * @see Bytes#toBytes(String) + * @see Bytes#toString(byte[]) + * @see LegacyStringTerminated + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class LegacyString implements DataType { + + public static final LegacyString ASCENDING = new LegacyString(Order.ASCENDING); + public static final LegacyString DESCENDING = new LegacyString(Order.DESCENDING); + + protected final Order order; + + protected LegacyString() { this.order = Order.ASCENDING; } + protected LegacyString(Order order) { this.order = order; } + + @Override + public boolean isOrderPreserving() { return true; } + + @Override + public Order getOrder() { return order; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return false; } + + @Override + public int skip(byte[] buff, int offset) { return buff.length; } + + @Override + public int encodedLength(String val) { return Bytes.toBytes(val).length; } + + @Override + public Class encodedClass() { return String.class; } + + @Override + public String decode(byte[] buff, int offset) { + order.apply(buff, offset, buff.length - offset); + return Bytes.toString(buff, offset, buff.length - offset); + } + + @Override + public int encode(byte[] buff, int offset, String val) { + byte[] s = Bytes.toBytes(val); + order.apply(s); + System.arraycopy(s, 0, buff, offset, s.length); + return offset + s.length; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyStringFixedLength.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyStringFixedLength.java new file mode 100644 index 0000000..267bdc1 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyStringFixedLength.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An DataType that encodes fixed-length values encoded using + * {@link Bytes#toBytes(String)}. Intended to make it easier to transition + * away from direct use of {@link Bytes}. + * @see Bytes#toBytes(String) + * @see Bytes#toString(byte[], int, int) + * @see LegacyString + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class LegacyStringFixedLength extends FixedLengthWrapper { + + /** + * Create a LegacyStringFixedLength using the specified + * order and length. + */ + public LegacyStringFixedLength(Order order, int length) { + super(new LegacyString(order), length); + } + + /** + * Create a LegacyStringFixedLength of the specified + * length>. + */ + public LegacyStringFixedLength(int length) { + super(new LegacyString(), length); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyStringTerminated.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyStringTerminated.java new file mode 100644 index 0000000..673db5c --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyStringTerminated.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An HDataType that encodes variable-length values encoded using + * {@link Bytes#toBytes(String)}. Includes a termination marker following the + * raw byte[] value. Intended to make it easier to transition + * away from direct use of {@link Bytes}. + * @see Bytes#toBytes(String) + * @see Bytes#toString(byte[], int, int) + * @see LegacyString + * @see OrderedString + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class LegacyStringTerminated extends TerminatedWrapper { + + /** + * Create a LegacyStringTerminated using the specified terminator and + * order. + * @throws IllegalArgumentException if term is null or empty. + */ + public LegacyStringTerminated(Order order, byte[] term) { + super(new LegacyString(order), term); + } + + /** + * Create a LegacyStringTerminated using the specified terminator and + * order. + * @throws IllegalArgumentException if term is null or empty. + */ + public LegacyStringTerminated(Order order, String term) { + super(new LegacyString(order), term); + } + + /** + * Create a LegacyStringTerminated using the specified terminator. + * @throws IllegalArgumentException if term is null or empty. + */ + public LegacyStringTerminated(byte[] term) { + super(new LegacyString(), term); + } + + /** + * Create a LegacyStringTerminated using the specified terminator. + * @throws IllegalArgumentException if term is null or empty. + */ + public LegacyStringTerminated(String term) { + super(new LegacyString(), term); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlob.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlob.java new file mode 100644 index 0000000..03aa2f7 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlob.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + +/** + * A byte[] of variable-length. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OrderedBlob extends OrderedBytesBase { + + public static final OrderedBlob ASCENDING = new OrderedBlob(Order.ASCENDING); + public static final OrderedBlob DESCENDING = new OrderedBlob(Order.DESCENDING); + + protected OrderedBlob(Order order) { super(order); } + + @Override + public boolean isSkippable() { return false; } + + @Override + public int encodedLength(byte[] val) { + return null == val ? + (Order.ASCENDING == order ? 1 : 2) : + (Order.ASCENDING == order ? val.length + 1 : val.length + 2); + } + + @Override + public Class encodedClass() { return byte[].class; } + + @Override + public byte[] decode(byte[] buff, int offset) { + return OrderedBytes.decodeBlobCopy(buff, offset); + } + + @Override + public int encode(byte[] buff, int offset, byte[] val) { + return OrderedBytes.encodeBlobCopy(buff, offset, val, order); + } + + /** + * Write a subset of val to buff. + */ + public int encode(byte[] buff, int offset, byte[] val, int voff, int vlen) { + return OrderedBytes.encodeBlobCopy(buff, offset, val, voff, vlen, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlobVar.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlobVar.java new file mode 100644 index 0000000..e907834 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBlobVar.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + +/** + * An alternative to {@link OrderedBlob} for use by {@link Struct} fields that + * do not terminate the fields list. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class OrderedBlobVar extends OrderedBytesBase { + + public static final OrderedBlobVar ASCENDING = new OrderedBlobVar(Order.ASCENDING); + public static final OrderedBlobVar DESCENDING = new OrderedBlobVar(Order.DESCENDING); + + protected OrderedBlobVar(Order order) { super(order); } + + @Override + public int encodedLength(byte[] val) { + return null == val ? 1 : OrderedBytes.blobVarEncodedLength(val.length); + } + + @Override + public Class encodedClass() { return byte[].class; } + + @Override + public byte[] decode(byte[] buff, int offset) { + return OrderedBytes.decodeBlobVar(buff, offset); + } + + @Override + public int encode(byte[] buff, int offset, byte[] val) { + return OrderedBytes.encodeBlobVar(buff, offset, val, order); + } + + /** + * Write a subset of val to buff. + */ + public int encode(byte[] buff, int offset, byte[] val, int voff, int vlen) { + return OrderedBytes.encodeBlobVar(buff, offset, val, voff, vlen, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBytesBase.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBytesBase.java new file mode 100644 index 0000000..1427957 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBytesBase.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + +/** + * Base class for data types backed by the {@link OrderedBytes} encoding + * implementations. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class OrderedBytesBase implements DataType { + + protected final Order order; + + protected OrderedBytesBase(Order order) { this.order = order; } + + @Override + public boolean isOrderPreserving() { return true; } + + @Override + public Order getOrder() { return order; } + + // almost all OrderedBytes implementations are nullable. + @Override + public boolean isNullable() { return true; } + + // almost all OrderedBytes implementations are skippable. + @Override + public boolean isSkippable() { return true; } + + @Override + public int skip(byte[] buff, int offset) { + return OrderedBytes.skip(buff, offset); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat32.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat32.java new file mode 100644 index 0000000..dc474a2 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat32.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + + +/** + * A float of 32-bits using a fixed-length encoding. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OrderedFloat32 extends OrderedBytesBase { + + public static final OrderedFloat32 ASCENDING = new OrderedFloat32(Order.ASCENDING); + public static final OrderedFloat32 DESCENDING = new OrderedFloat32(Order.DESCENDING); + + protected OrderedFloat32(Order order) { super(order); } + + @Override + public boolean isNullable() { return false; } + + @Override + public int encodedLength(Float val) { return 5; } + + @Override + public Class encodedClass() { return Float.class; } + + @Override + public Float decode(byte[] buff, int offset) { + return OrderedBytes.decodeFloat32(buff, offset); + } + + @Override + public int encode(byte[] buff, int offset, Float val) { + if (null == val) throw new IllegalArgumentException("Null values not supported."); + return OrderedBytes.encodeFloat32(buff, offset, val, order); + } + + /** + * Read a float value from the buffer buff. + */ + public float decodeFloat(byte[] buff, int offset) { + return OrderedBytes.decodeFloat32(buff, offset); + } + + /** + * Write instance val into buffer buff. + */ + public int encodeFloat(byte[] buff, int offset, float val) { + return OrderedBytes.encodeFloat32(buff, offset, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat64.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat64.java new file mode 100644 index 0000000..d04287a --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedFloat64.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + + +/** + * A double of 64-bits using a fixed-length encoding. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OrderedFloat64 extends OrderedBytesBase { + + public static final OrderedFloat64 ASCENDING = new OrderedFloat64(Order.ASCENDING); + public static final OrderedFloat64 DESCENDING = new OrderedFloat64(Order.DESCENDING); + + protected OrderedFloat64(Order order) { super(order); } + + @Override + public boolean isNullable() { return false; } + + @Override + public int encodedLength(Double val) { return 9; } + + @Override + public Class encodedClass() { return Double.class; } + + @Override + public Double decode(byte[] buff, int offset) { + return OrderedBytes.decodeFloat64(buff, offset); + } + + @Override + public int encode(byte[] buff, int offset, Double val) { + if (null == val) throw new IllegalArgumentException("Null values not supported."); + return OrderedBytes.encodeFloat64(buff, offset, val, order); + } + + /** + * Read a double value from the buffer buff. + */ + public double decodeDouble(byte[] buff, int offset) { + return OrderedBytes.decodeFloat64(buff, offset); + } + + /** + * Write instance val into buffer buff. + */ + public int encodeDouble(byte[] buff, int offset, double val) { + return OrderedBytes.encodeFloat64(buff, offset, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt32.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt32.java new file mode 100644 index 0000000..f792c02 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt32.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + + +/** + * An int of 32-bits using a fixed-length encoding. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OrderedInt32 extends OrderedBytesBase { + + public static final OrderedInt32 ASCENDING = new OrderedInt32(Order.ASCENDING); + public static final OrderedInt32 DESCENDING = new OrderedInt32(Order.DESCENDING); + + protected OrderedInt32(Order order) { super(order); } + + @Override + public boolean isNullable() { return false; } + + @Override + public int encodedLength(Integer val) { return 5; } + + @Override + public Class encodedClass() { return Integer.class; } + + @Override + public Integer decode(byte[] buff, int offset) { + return OrderedBytes.decodeInt32(buff, offset); + } + + @Override + public int encode(byte[] buff, int offset, Integer val) { + if (null == val) throw new IllegalArgumentException("Null values not supported."); + return OrderedBytes.encodeInt32(buff, offset, val, order); + } + + /** + * Read an int value from the buffer buff. + */ + public int decodeInt(byte[] buff, int offset) { + return OrderedBytes.decodeInt32(buff, offset); + } + + /** + * Write instance val into buffer buff. + */ + public int encodeInt(byte[] buff, int offset, int val) { + return OrderedBytes.encodeInt32(buff, offset, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt64.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt64.java new file mode 100644 index 0000000..af71ae7 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedInt64.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + + +/** + * A long of 64-bits using a fixed-length encoding. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OrderedInt64 extends OrderedBytesBase { + + public static final OrderedInt64 ASCENDING = new OrderedInt64(Order.ASCENDING); + public static final OrderedInt64 DESCENDING = new OrderedInt64(Order.DESCENDING); + + protected OrderedInt64(Order order) { super(order); } + + @Override + public boolean isNullable() { return false; } + + @Override + public int encodedLength(Long val) { return 9; } + + @Override + public Class encodedClass() { return Long.class; } + + @Override + public Long decode(byte[] buff, int offset) { + return OrderedBytes.decodeInt64(buff, offset); + } + + @Override + public int encode(byte[] buff, int offset, Long val) { + if (null == val) throw new IllegalArgumentException("Null values not supported."); + return OrderedBytes.encodeInt64(buff, offset, val, order); + } + + /** + * Read a long value from the buffer buff. + */ + public long decodeLong(byte[] buff, int offset) { + return OrderedBytes.decodeInt64(buff, offset); + } + + /** + * Write instance val into buffer buff. + */ + public int encodeLong(byte[] buff, int offset, long val) { + return OrderedBytes.encodeInt64(buff, offset, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedNumeric.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedNumeric.java new file mode 100644 index 0000000..4e4a4e8 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedNumeric.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.math.BigDecimal; +import java.math.BigInteger; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Numeric; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + +/** + * An {@link Number} of arbitrary precision and variable-length encoding. The + * resulting length of encoded values is determined by the numerical (base + * 100) precision, not absolute value. Use this data type anywhere you would + * expect to use a DECIMAL type, a {@link BigDecimal}, a + * {@link BigInteger}, or any time you've parsed floating precision values + * from text. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OrderedNumeric extends OrderedBytesBase { + + public static final OrderedNumeric ASCENDING = new OrderedNumeric(Order.ASCENDING); + public static final OrderedNumeric DESCENDING = new OrderedNumeric(Order.DESCENDING); + + protected OrderedNumeric(Order order) { super(order); } + + @Override + public int encodedLength(Numeric val) { + // TODO: this could be done better. + byte[] buff = new byte[100]; + int offset = 0, start = 0; + offset = encode(buff, offset, val); + return offset - start; + } + + @Override + public Class encodedClass() { return Numeric.class; } + + @Override + public Numeric decode(byte[] buff, int offset) { + return OrderedBytes.decodeNumeric(buff, offset); + } + + @Override + public int encode(byte[] buff, int offset, Numeric val) { + if (null == val) { + return OrderedBytes.encodeNull(buff, offset, order); + } else if (val.isInteger()) { + return OrderedBytes.encodeNumeric(buff, offset, val.longValue(), order); + } else if (val.isReal()) { + return OrderedBytes.encodeNumeric(buff, offset, val.doubleValue(), order); + } else { + return OrderedBytes.encodeNumeric(buff, offset, val.exactValue(), order); + } + } + + /** + * Read a long value from the buffer buff. + */ + public long decodeLong(byte[] buff, int offset) { + return OrderedBytes.decodeNumeric(buff, offset).longValue(); + } + + /** + * Write instance val into buffer buff. + */ + public int encodeLong(byte[] buff, int offset, long val) { + return OrderedBytes.encodeNumeric(buff, offset, val, order); + } + + /** + * Read a double value from the buffer buff. + */ + public double decodeDouble(byte[] buff, int offset) { + return OrderedBytes.decodeNumeric(buff, offset).doubleValue(); + } + + /** + * Write instance val into buffer buff. + */ + public int encodeDouble(byte[] buff, int offset, double val) { + return OrderedBytes.encodeNumeric(buff, offset, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedString.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedString.java new file mode 100644 index 0000000..efacdab --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedString.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + +/** + * A String of variable-length. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OrderedString extends OrderedBytesBase { + + public static final OrderedString ASCENDING = new OrderedString(Order.ASCENDING); + public static final OrderedString DESCENDING = new OrderedString(Order.DESCENDING); + + protected OrderedString(Order order) { super(order); } + + @Override + public int encodedLength(String val) { + return null == val ? 1 : val.getBytes(OrderedBytes.UTF8).length + 2; + } + + @Override + public Class encodedClass() { return String.class; } + + @Override + public String decode(byte[] buff, int offset) { + return OrderedBytes.decodeString(buff, offset); + } + + @Override + public int encode(byte[] buff, int offset, String val) { + return OrderedBytes.encodeString(buff, offset, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java new file mode 100644 index 0000000..46c9110 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.util.Iterator; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; + +/** + *

+ * Struct is a simple {@link DataType} for implementing the + * "compound rowkey" schema design strategy. + *

+ *

Encoding

+ *

+ * Struct member values are encoded onto the target + * byte[] in the order in which they are declared. A Struct + * may be used as a member of another Struct. + * Structs are not nullable but their component + * fields may be. + *

+ *

Sort Order

+ *

+ * Struct instances sort according to the composite order of + * their fields, that is, left-to-right and depth-first. This can also be + * thought of as lexicographic comparison of concatenated members. + *

+ *

+ * {@link StructIterator} is provided as a convenience for consuming the + * sequence of values. Users may find it more appropriate to provide their own + * custom {@link DataType} for encoding application objects rather than using + * this Object[] implementation. Examples are provided in test. + *

+ * @see StructIterator + * @see DataType#isNullable() + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class Struct implements DataType { + + @SuppressWarnings("rawtypes") + protected final DataType[] fields; + protected final boolean isOrderPreserving; + protected final boolean isSkippable; + + /** + * Create a new Struct instance defined as the sequence of + * HDataTypes in memberTypes. + *

+ * A Struct is orderPreserving when all of its + * fields are orderPreserving. A Struct is + * skippable when all of its fields are skippable. + *

+ */ + @SuppressWarnings("rawtypes") + public Struct(DataType[] memberTypes) { + this.fields = memberTypes; + // a Struct is not orderPreserving when any of its fields are not. + boolean preservesOrder = true; + // a Struct is not skippable when any of its fields are not. + boolean skippable = true; + for (int i = 0; i < this.fields.length; i++) { + DataType dt = this.fields[i]; + if (!dt.isOrderPreserving()) preservesOrder = false; + if (i < this.fields.length - 2 && !dt.isSkippable()) + throw new IllegalArgumentException("Non-right-most struct fields must be skippable."); + if (!dt.isSkippable()) skippable = false; + } + this.isOrderPreserving = preservesOrder; + this.isSkippable = skippable; + } + + @Override + public boolean isOrderPreserving() { return isOrderPreserving; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return isSkippable; } + + @SuppressWarnings("unchecked") + @Override + public int encodedLength(Object[] val) { + assert fields.length == val.length; + int sum = 0; + for (int i = 0; i < fields.length; i++) + sum += fields[i].encodedLength(val[i]); + return sum; + } + + @Override + public Class encodedClass() { return Object[].class; } + + /** + * Retrieve an {@link Iterator} over the values encoded in buff. + * The byte[] backing buff is not modified by use of + * the Iterator. + */ + public StructIterator iterator(byte[] buff, int offset) { + return new StructIterator(buff, offset, fields); + } + + @Override + public int skip(byte[] buff, int offset) { + StructIterator it = iterator(buff, offset); + while (it.hasNext()) + offset = it.skip(); + return offset; + } + + @Override + public Object[] decode(byte[] buff, int offset) { + int i = 0; + Object[] ret = new Object[fields.length]; + Iterator it = iterator(buff, offset); + while (it.hasNext()) + ret[i++] = it.next(); + return ret; + } + + /** + * Read the field at index. buff is left + * unmodified. + */ + public Object read(byte[] buff, int offset, int index) { + assert index >= 0; + StructIterator it = iterator(buff, offset); + for (; index > 0; index--) + it.skip(); + return it.next(); + } + + @SuppressWarnings("unchecked") + @Override + public int encode(byte[] buff, int offset, Object[] val) { + assert fields.length == val.length; + for (int i = 0; i < fields.length; i++) { + offset = fields[i].encode(buff, offset, val[i]); + } + return offset; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructBuilder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructBuilder.java new file mode 100644 index 0000000..c17b190 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructBuilder.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * A helper for building {@link Struct} instances. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class StructBuilder { + + protected final List> fields = new ArrayList>(); + + /** + * Create an empty StructBuilder. + */ + public StructBuilder() {} + + /** + * Append field to the sequence of accumulated fields. + */ + public StructBuilder add(DataType field) { fields.add(field); return this; } + + /** + * Retrieve the {@link Struct} represented by this. + */ + public Struct toStruct() { return new Struct(fields.toArray(new DataType[fields.size()])); } + + /** + * Reset the sequence of accumulated fields. + */ + public StructBuilder reset() { fields.clear(); return this; } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructIterator.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructIterator.java new file mode 100644 index 0000000..6e32e45 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructIterator.java @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.util.Iterator; +import java.util.NoSuchElementException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * An {@link Iterator} over encoded Struct members. + *

+ * This iterates over each serialized Struct field from the + * specified DataTypes[] definition. It allows you to read + * the field or skip over its serialized bytes using {@link #next()} and + * {@link #skip()}, respectively. This is in contrast to the + * Struct method which allow you to + * {@link Struct#decode(byte[], int)} or {@link Struct#skip(byte[], int)} over + * the entire Struct at once. + *

+ *

+ * This iterator may also be used to read bytes from any Struct + * for which the specified DataType[] is a prefix. For + * example, if the specified Struct definition has a + * {@link LegacyInteger} and a {@link LegacyStringTerminated} field, you may + * parse the serialized output of a Struct whose fields are + * {@link LegacyInteger}, {@link LegacyStringTerminated}, and + * {@link LegacyBytes}. The iterator would return a number followed by a + * String. The trailing byte[] would be ignored. + *

+ */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class StructIterator implements Iterator { + + protected final byte[] buff; + protected int offset, idx = 0; + @SuppressWarnings("rawtypes") + protected final DataType[] types; + + /** + * Construct StructIterator over the values encoded in + * buff using the specified types definition. + * @param buff The buffer from which to read encoded values. + * @param types The sequence of types to use as the schema for this + * Struct. + */ + public StructIterator(final byte[] buff, int offset, + @SuppressWarnings("rawtypes") DataType[] types) { + this.buff = buff; + this.offset = offset; + this.types = types; + } + + @Override + public boolean hasNext() { + return idx < types.length; + } + + @Override + public void remove() { throw new UnsupportedOperationException(); } + + @Override + public Object next() { + if (!hasNext()) throw new NoSuchElementException(); + Object ret = types[idx].decode(buff, offset); + offset = types[idx++].skip(buff, offset); + return ret; + } + + /** + * Bypass the next encoded value. + */ + public int skip() { + if (!hasNext()) throw new NoSuchElementException(); + offset = types[idx++].skip(buff, offset); + return offset; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/TerminatedWrapper.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/TerminatedWrapper.java new file mode 100644 index 0000000..1892504 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/TerminatedWrapper.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * Wraps an existing DataType implementation as a terminated + * version of itself. This has the useful side-effect of turning an existing + * DataType which is not skippable into a + * skippable variant. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class TerminatedWrapper implements DataType { + + protected final DataType wrapped; + protected final byte[] term; + + /** + * Create a terminated version of the wrapped. + * @throws IllegalArgumentException when term is null or empty. + */ + public TerminatedWrapper(DataType wrapped, byte[] term) { + if (null == term || term.length == 0) + throw new IllegalArgumentException("terminator must be non-null and non-empty."); + this.wrapped = wrapped; + wrapped.getOrder().apply(term); + this.term = term; + } + + /** + * Create a terminated version of the wrapped. + * term is converted to a byte[] using + * {@link Bytes#toBytes(String)}. + * @throws IllegalArgumentException when term is null or empty. + */ + public TerminatedWrapper(DataType wrapped, String term) { + this(wrapped, Bytes.toBytes(term)); + } + + @Override + public boolean isOrderPreserving() { return wrapped.isOrderPreserving(); } + + @Override + public Order getOrder() { return wrapped.getOrder(); } + + @Override + public boolean isNullable() { return wrapped.isNullable(); } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(T val) { + return wrapped.encodedLength(val) + term.length; + } + + @Override + public Class encodedClass() { return wrapped.encodedClass(); } + + /** + * Return the index at which term begins within + * buff, or -1 if term is not found. + */ + protected int terminatorPosition(byte[] buff, int offset) { + SKIP: for (; offset < buff.length; offset++) { + if (buff[offset] != term[0]) continue; + int j; + for (j = 1; j < term.length && offset + j < buff.length; j++) { + if (buff[offset + j] != term[j]) continue SKIP; + } + if (j == term.length) return offset; // success + } + return -1; + } + + /** + * Skip offset over the encoded value. + * @throws IllegalArgumentException when the terminator sequence is not found. + */ + @Override + public int skip(byte[] buff, int offset) { + if (wrapped.isSkippable()) { + offset = wrapped.skip(buff, offset); + } else { + offset = terminatorPosition(buff, offset); + if (-1 == offset) throw new IllegalArgumentException("Terminator sequence not found."); + } + return offset + term.length; + } + + @Override + public T decode(byte[] buff, int offset) { + if (wrapped.isSkippable()) { + return wrapped.decode(buff, offset); + } else { + int term = terminatorPosition(buff, offset); + if (-1 == term) throw new IllegalArgumentException("Terminator sequence not found."); + byte[] b = new byte[term - offset]; + System.arraycopy(buff, offset, b, 0, b.length); + return wrapped.decode(b, 0); + } + } + + /** + * Write instance val into buffer buff. + * @throws IllegalArgumentException when the encoded representation of + * val contains the term sequence. + */ + @Override + public int encode(byte[] buff, int offset, T val) { + int start = offset; + offset = wrapped.encode(buff, offset, val); + byte[] b = new byte[offset - start]; + System.arraycopy(buff, start, b, 0, b.length); + if (-1 != terminatorPosition(b, 0)) { + throw new IllegalArgumentException("Encoded value contains terminator sequence."); + } + System.arraycopy(term, 0, buff, offset, term.length); + return offset + term.length; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union2.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union2.java new file mode 100644 index 0000000..7f14a75 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union2.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; + +/** + * The Union family of {@link DataType}s encode one of a fixed + * set of Objects. They provide convenience methods which handle + * type casting on your behalf. + */ +@SuppressWarnings("unchecked") +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class Union2 implements DataType { + + protected final DataType typeA; + protected final DataType typeB; + + /** + * Create an instance of Union2 over the set of specified + * types. + */ + public Union2(DataType typeA, DataType typeB) { + this.typeA = typeA; + this.typeB = typeB; + } + + @Override + public boolean isOrderPreserving() { + return typeA.isOrderPreserving() && typeB.isOrderPreserving(); + } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { + return typeA.isNullable() && typeB.isNullable(); + } + + @Override + public boolean isSkippable() { + return typeA.isSkippable() && typeB.isSkippable(); + } + + @Override + public Class encodedClass() { + throw new UnsupportedOperationException( + "Union types do not expose a definitive encoded class."); + } + + /** + * Read an instance of the first type parameter from buffer + * buff. + */ + public A decodeA(byte[] buff, int offset) { + return (A) decode(buff, offset); + } + + /** + * Read an instance of the second type parameter from buffer + * buff. + */ + public B decodeB(byte[] buff, int offset) { + return (B) decode(buff, offset); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union3.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union3.java new file mode 100644 index 0000000..3ce6e35 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union3.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; + +/** + * The Union family of {@link DataType}s encode one of a fixed + * collection of Objects. They provide convenience methods which handle type + * casting on your behalf. + * @see Union2 + */ +@SuppressWarnings("unchecked") +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class Union3 extends Union2 { + + protected final DataType typeC; + + /** + * Create an instance of Union3 over the set of specified + * types. + */ + public Union3(DataType typeA, DataType typeB, DataType typeC) { + super(typeA, typeB); + this.typeC = typeC; + } + + @Override + public boolean isOrderPreserving() { + return super.isOrderPreserving() && typeC.isOrderPreserving(); + } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { + return super.isNullable() && typeC.isNullable(); + } + + @Override + public boolean isSkippable() { + return super.isSkippable() && typeC.isSkippable(); + } + + /** + * Read an instance of the third type parameter from buffer b. + */ + public C decodeC(byte[] buff, int offset) { + return (C) decode(buff, offset); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union4.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union4.java new file mode 100644 index 0000000..8e9b413 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union4.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; + +/** + * The Union family of {@link DataType}s encode one of a fixed + * collection of Objects. They provide convenience methods which handle type + * casting on your behalf. + */ +@SuppressWarnings("unchecked") +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class Union4 extends Union3 { + + protected final DataType typeD; + + /** + * Create an instance of Union4 over the set of specified + * types. + */ + public Union4(DataType typeA, DataType typeB, DataType typeC, DataType typeD) { + super(typeA, typeB, typeC); + this.typeD = typeD; + } + + @Override + public boolean isOrderPreserving() { + return super.isOrderPreserving() && typeD.isOrderPreserving(); + } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { + return super.isNullable() && typeD.isNullable(); + } + + @Override + public boolean isSkippable() { + return super.isSkippable() && typeD.isSkippable(); + } + + /** + * Read an instance of the fourth type parameter from buffer b. + */ + public D decodeD(byte[] buff, int offset) { + return (D) decode(buff, offset); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestFixedLengthWrapper.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestFixedLengthWrapper.java new file mode 100644 index 0000000..087dc3c --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestFixedLengthWrapper.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestFixedLengthWrapper { + + static final byte[][] VALUES = new byte[][] { + Bytes.toBytes(""), Bytes.toBytes("1"), Bytes.toBytes("22"), Bytes.toBytes("333"), + Bytes.toBytes("4444"), Bytes.toBytes("55555"), Bytes.toBytes("666666"), + Bytes.toBytes("7777777"), Bytes.toBytes("88888888"), Bytes.toBytes("999999999"), + }; + + /** + * all values of limit are >= max length of a member of + * VALUES. + */ + static final int[] limits = { 9, 12, 15 }; + + @Test + public void testReadWrite() { + for (int limit : limits) { + byte[] buff = new byte[limit]; + for (Order ord : new Order[] { Order.ASCENDING, Order.DESCENDING }) { + for (byte[] val : VALUES) { + DataType type = new FixedLengthWrapper(new LegacyBytes(ord), limit); + assertEquals(buff.length, type.encode(buff, 0, val)); + byte[] expected = Arrays.copyOf(val, limit); + byte[] actual = type.decode(buff, 0); + assertArrayEquals(expected, actual); + assertEquals(buff.length, type.skip(buff, 0)); + } + } + } + } + + @Test(expected = IllegalArgumentException.class) + public void testInsufficientRemainingRead() { + byte[] buff = new byte[0]; + DataType type = new FixedLengthWrapper(new LegacyBytes(), 3); + type.decode(buff, 0); + } + + @Test(expected = IllegalArgumentException.class) + public void testInsufficientRemainingWrite() { + byte[] buff = new byte[0]; + DataType type = new FixedLengthWrapper(new LegacyBytes(), 3); + type.encode(buff, 0, Bytes.toBytes("")); + } + + @Test(expected = IllegalArgumentException.class) + public void testOverflowPassthrough() { + byte[] buff = new byte[3]; + DataType type = new FixedLengthWrapper(new LegacyBytes(), 0); + type.encode(buff, 0, Bytes.toBytes("foo")); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestLegacyString.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestLegacyString.java new file mode 100644 index 0000000..b8857f0 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestLegacyString.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestLegacyString { + + static final String[] VALUES = new String[] { + "", "1", "22", "333", "4444", "55555", "666666", "7777777", "88888888", "999999999", + }; + + @Test + public void testReadWrite() { + for (LegacyString type : new LegacyString[] { + LegacyString.ASCENDING, LegacyString.DESCENDING }) { + for (String val : VALUES) { + byte[] buff = new byte[Bytes.toBytes(val).length + 1]; + assertEquals(buff.length, type.encode(buff, 1, val)); + assertEquals(val, type.decode(buff, 1)); + assertEquals(buff.length, type.skip(buff, 1)); + } + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlob.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlob.java new file mode 100644 index 0000000..7386612 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlob.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestOrderedBlob { + + static final byte[][] VALUES = new byte[][] { + null, Bytes.toBytes(""), Bytes.toBytes("1"), Bytes.toBytes("22"), Bytes.toBytes("333"), + Bytes.toBytes("4444"), Bytes.toBytes("55555"), Bytes.toBytes("666666"), + Bytes.toBytes("7777777"), Bytes.toBytes("88888888"), Bytes.toBytes("999999999"), + }; + + @Test + public void testEncodedLength() { + byte[] buff = new byte[20]; + int offset; + for (DataType type : new OrderedBlob[] { OrderedBlob.ASCENDING, OrderedBlob.DESCENDING }) { + for (byte[] val : VALUES) { + offset = 0; + offset = type.encode(buff, offset, val); + assertEquals( + "encodedLength does not match actual, " + Bytes.toStringBinary(val), + offset, type.encodedLength(val)); + } + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlobVar.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlobVar.java new file mode 100644 index 0000000..8a70ae6 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedBlobVar.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestOrderedBlobVar { + + static final byte[][] VALUES = new byte[][] { + null, Bytes.toBytes(""), Bytes.toBytes("1"), Bytes.toBytes("22"), Bytes.toBytes("333"), + Bytes.toBytes("4444"), Bytes.toBytes("55555"), Bytes.toBytes("666666"), + Bytes.toBytes("7777777"), Bytes.toBytes("88888888"), Bytes.toBytes("999999999"), + }; + + @Test + public void testEncodedLength() { + byte[] buff = new byte[20]; + int offset; + for (DataType type : + new OrderedBlobVar[] { OrderedBlobVar.ASCENDING, OrderedBlobVar.DESCENDING }) { + for (byte[] val : VALUES) { + offset = 0; + offset = type.encode(buff, offset, val); + assertEquals( + "encodedLength does not match actual, " + Bytes.toStringBinary(val), + offset, type.encodedLength(val)); + } + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedString.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedString.java new file mode 100644 index 0000000..a35484e --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestOrderedString.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestOrderedString { + + static final String[] VALUES = + new String[] { null, "", "1", "22", "333", "4444", "55555", "666666", + "7777777", "88888888", "999999999" }; + + @Test + public void testEncodedLength() { + byte[] buff = new byte[20]; + int offset; + for (DataType type : new OrderedString[] { OrderedString.ASCENDING, OrderedString.DESCENDING }) { + for (String val : VALUES) { + offset = 0; + offset = type.encode(buff, offset, val); + assertEquals( + "encodedLength does not match actual, " + val, + offset, type.encodedLength(val)); + } + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestStruct.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestStruct.java new file mode 100644 index 0000000..d1f07cd --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestStruct.java @@ -0,0 +1,361 @@ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.lang.reflect.Constructor; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * This class both tests and demonstrates how to construct compound rowkeys + * from a POJO. The code under test is {@link Struct}. + * {@link SpecializedPojo1Type1} demonstrates how one might create their own + * custom data type extension for an application POJO. + */ +@RunWith(Parameterized.class) +@Category(SmallTests.class) +public class TestStruct { + + private Struct generic; + @SuppressWarnings("rawtypes") + private DataType specialized; + private Object[][] constructorArgs; + + public TestStruct(Struct generic, @SuppressWarnings("rawtypes") DataType specialized, + Object[][] constructorArgs) { + this.generic = generic; + this.specialized = specialized; + this.constructorArgs = constructorArgs; + } + + @Parameters + public static Collection params() { + Object[][] pojo1Args = { + new Object[] { "foo", 5, 10.001 }, + new Object[] { "foo", 100, 7.0 }, + new Object[] { "foo", 100, 10.001 }, + new Object[] { "bar", 5, 10.001 }, + new Object[] { "bar", 100, 10.001 }, + new Object[] { "baz", 5, 10.001 }, + }; + + Object[][] pojo2Args = { + new Object[] { new byte[0], "it".getBytes(), "was", "the".getBytes() }, + new Object[] { "best".getBytes(), new byte[0], "of", "times,".getBytes() }, + new Object[] { "it".getBytes(), "was".getBytes(), "", "the".getBytes() }, + new Object[] { "worst".getBytes(), "of".getBytes(), "times,", new byte[0] }, + new Object[] { new byte[0], new byte[0], "", new byte[0] }, + }; + + Object[][] params = new Object[][] { + { SpecializedPojo1Type1.GENERIC, new SpecializedPojo1Type1(), pojo1Args }, + { SpecializedPojo2Type1.GENERIC, new SpecializedPojo2Type1(), pojo2Args }, + }; + return Arrays.asList(params); + } + + static final Comparator NULL_SAFE_BYTES_COMPARATOR = + new Comparator() { + @Override + public int compare(byte[] o1, byte[] o2) { + if (o1 == o2) return 0; + if (null == o1) return -1; + if (null == o2) return 1; + return Bytes.compareTo(o1, o2); + } + }; + + /** + * A simple object to serialize. + */ + private static class Pojo1 implements Comparable { + final String stringFieldAsc; + final int intFieldAsc; + final double doubleFieldAsc; + final transient String str; + + public Pojo1(Object... argv) { + stringFieldAsc = (String) argv[0]; + intFieldAsc = (Integer) argv[1]; + doubleFieldAsc = (Double) argv[2]; + str = new StringBuilder() + .append("{ ") + .append(null == stringFieldAsc ? "" : "\"") + .append(stringFieldAsc) + .append(null == stringFieldAsc ? "" : "\"").append(", ") + .append(intFieldAsc).append(", ") + .append(doubleFieldAsc) + .append(" }") + .toString(); + } + + @Override + public String toString() { + return str; + } + + @Override + public int compareTo(Pojo1 o) { + int cmp = stringFieldAsc.compareTo(o.stringFieldAsc); + if (cmp != 0) return cmp; + cmp = Integer.valueOf(intFieldAsc).compareTo(Integer.valueOf(o.intFieldAsc)); + if (cmp != 0) return cmp; + return Double.compare(doubleFieldAsc, o.doubleFieldAsc); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (null == o) return false; + if (!(o instanceof Pojo1)) return false; + Pojo1 that = (Pojo1) o; + return 0 == this.compareTo(that); + } + } + + /** + * A simple object to serialize. + */ + private static class Pojo2 implements Comparable { + final byte[] byteField1Asc; + final byte[] byteField2Dsc; + final String stringFieldDsc; + final byte[] byteField3Dsc; + final transient String str; + + public Pojo2(Object... vals) { + byte[] empty = new byte[0]; + byteField1Asc = vals.length > 0 ? (byte[]) vals[0] : empty; + byteField2Dsc = vals.length > 1 ? (byte[]) vals[1] : empty; + stringFieldDsc = vals.length > 2 ? (String) vals[2] : ""; + byteField3Dsc = vals.length > 3 ? (byte[]) vals[3] : empty; + str = new StringBuilder() + .append("{ ") + .append(Bytes.toStringBinary(byteField1Asc)).append(", ") + .append(Bytes.toStringBinary(byteField2Dsc)).append(", ") + .append(null == stringFieldDsc ? "" : "\"") + .append(stringFieldDsc) + .append(null == stringFieldDsc ? "" : "\"").append(", ") + .append(Bytes.toStringBinary(byteField3Dsc)) + .append(" }") + .toString(); + } + + @Override + public String toString() { + return str; + } + + @Override + public int compareTo(Pojo2 o) { + int cmp = NULL_SAFE_BYTES_COMPARATOR.compare(byteField1Asc, o.byteField1Asc); + if (cmp != 0) return cmp; + cmp = -NULL_SAFE_BYTES_COMPARATOR.compare(byteField2Dsc, o.byteField2Dsc); + if (cmp != 0) return cmp; + if (stringFieldDsc == o.stringFieldDsc) cmp = 0; + else if (null == stringFieldDsc) cmp = 1; + else if (null == o.stringFieldDsc) cmp = -1; + else cmp = -stringFieldDsc.compareTo(o.stringFieldDsc); + if (cmp != 0) return cmp; + return -NULL_SAFE_BYTES_COMPARATOR.compare(byteField3Dsc, o.byteField3Dsc); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (null == o) return false; + if (!(o instanceof Pojo2)) return false; + Pojo2 that = (Pojo2) o; + return 0 == this.compareTo(that); + } + } + + /** + * A custom data type implementation specialized for {@link Pojo1}. + */ + private static class SpecializedPojo1Type1 implements DataType { + + private static final LegacyStringTerminated stringField = new LegacyStringTerminated("/"); + private static final LegacyInteger intField = new LegacyInteger(); + private static final LegacyDouble doubleField = new LegacyDouble(); + + /** + * The {@link Struct} equivalent of this type. + */ + public static Struct GENERIC = + new StructBuilder().add(stringField) + .add(intField) + .add(doubleField) + .toStruct(); + + @Override + public boolean isOrderPreserving() { return true; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(Pojo1 val) { + return + stringField.encodedLength(val.stringFieldAsc) + + intField.encodedLength(val.intFieldAsc) + + doubleField.encodedLength(val.doubleFieldAsc); + } + + @Override + public Class encodedClass() { return Pojo1.class; } + + @Override + public int skip(byte[] buff, int offset) { + offset = stringField.skip(buff, offset); + offset = intField.skip(buff, offset); + return doubleField.skip(buff, offset); + } + + @Override + public Pojo1 decode(byte[] buff, int offset) { + Object[] ret = new Object[3]; + ret[0] = stringField.decode(buff, offset); + offset = stringField.skip(buff, offset); + ret[1] = intField.decode(buff, offset); + offset = intField.skip(buff, offset); + ret[2] = doubleField.decode(buff, offset); + return new Pojo1(ret); + } + + @Override + public int encode(byte[] buff, int offset, Pojo1 val) { + offset = stringField.encode(buff, offset, val.stringFieldAsc); + offset = intField.encode(buff, offset, val.intFieldAsc); + return doubleField.encode(buff, offset, val.doubleFieldAsc); + } + } + + /** + * A custom data type implementation specialized for {@link Pojo2}. + */ + private static class SpecializedPojo2Type1 implements DataType { + + private static LegacyBytesTerminated byteField1 = new LegacyBytesTerminated("/"); + private static LegacyBytesTerminated byteField2 = + new LegacyBytesTerminated(Order.DESCENDING, "/"); + private static LegacyStringTerminated stringField = + new LegacyStringTerminated(Order.DESCENDING, new byte[] { 0x00 }); + private static LegacyBytes byteField3 = LegacyBytes.DESCENDING; + + /** + * The {@link Struct} equivalent of this type. + */ + public static Struct GENERIC = + new StructBuilder().add(byteField1) + .add(byteField2) + .add(stringField) + .add(byteField3) + .toStruct(); + + @Override + public boolean isOrderPreserving() { return true; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @Override + public boolean isSkippable() { return true; } + + @Override + public int encodedLength(Pojo2 val) { + return + byteField1.encodedLength(val.byteField1Asc) + + byteField2.encodedLength(val.byteField2Dsc) + + stringField.encodedLength(val.stringFieldDsc) + + byteField3.encodedLength(val.byteField3Dsc); + } + + @Override + public Class encodedClass() { return Pojo2.class; } + + @Override + public int skip(byte[] buff, int offset) { + offset = byteField1.skip(buff, offset); + offset = byteField2.skip(buff, offset); + offset = stringField.skip(buff, offset); + return byteField3.skip(buff, offset); + } + + @Override + public Pojo2 decode(byte[] buff, int offset) { + Object[] ret = new Object[4]; + ret[0] = byteField1.decode(buff, offset); + offset = byteField1.skip(buff, offset); + ret[1] = byteField2.decode(buff, offset); + offset = byteField2.skip(buff, offset); + ret[2] = stringField.decode(buff, offset); + offset = stringField.skip(buff, offset); + ret[3] = byteField3.decode(buff, offset); + return new Pojo2(ret); + } + + @Override + public int encode(byte[] buff, int offset, Pojo2 val) { + offset = byteField1.encode(buff, offset, val.byteField1Asc); + offset = byteField2.encode(buff, offset, val.byteField2Dsc); + offset = stringField.encode(buff, offset, val.stringFieldDsc); + return byteField3.encode(buff, offset, val.byteField3Dsc); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testOrderPreservation() throws Exception { + Object[] vals = new Object[constructorArgs.length]; + byte[][] encodedGeneric = new byte[constructorArgs.length][]; + byte[][] encodedSpecialized = new byte[constructorArgs.length][]; + Constructor ctor = specialized.encodedClass().getConstructor(Object[].class); + for (int i = 0; i < vals.length; i++) { + vals[i] = ctor.newInstance(new Object[] { constructorArgs[i] }); + encodedGeneric[i] = new byte[generic.encodedLength(constructorArgs[i])]; + encodedSpecialized[i] = new byte[specialized.encodedLength(vals[i])]; + } + + // populate our arrays + for (int i = 0; i < vals.length; i++) { + generic.encode(encodedGeneric[i], 0, constructorArgs[i]); + specialized.encode(encodedSpecialized[i], 0, vals[i]); + assertArrayEquals(encodedGeneric[i], encodedSpecialized[i]); + } + + Arrays.sort(vals); + Arrays.sort(encodedGeneric, NULL_SAFE_BYTES_COMPARATOR); + Arrays.sort(encodedSpecialized, NULL_SAFE_BYTES_COMPARATOR); + + for (int i = 0; i < vals.length; i++) { + assertEquals( + "Struct encoder does not preserve sort order at position " + i, + vals[i], + ctor.newInstance(new Object[] { generic.decode(encodedGeneric[i], 0) })); + assertEquals( + "Specialized encoder does not preserve sort order at position " + i, + vals[i], specialized.decode(encodedSpecialized[i], 0)); + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestTerminatedWrapper.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestTerminatedWrapper.java new file mode 100644 index 0000000..ee93167 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestTerminatedWrapper.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestTerminatedWrapper { + + static final byte[][] VALUES = new byte[][] { + Bytes.toBytes(""), Bytes.toBytes("1"), Bytes.toBytes("22"), Bytes.toBytes("333"), + Bytes.toBytes("4444"), Bytes.toBytes("55555"), Bytes.toBytes("666666"), + Bytes.toBytes("7777777"), Bytes.toBytes("88888888"), Bytes.toBytes("999999999"), + }; + + static final byte[][] TERMINATORS = new byte[][] { new byte[] { -1 }, Bytes.toBytes("foo") }; + + @Test(expected = IllegalArgumentException.class) + public void testEmptyDelimiter() { + new TerminatedWrapper(new LegacyBytes(), ""); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullDelimiter() { + new LegacyBytesTerminated((byte[]) null); + // new TerminatedWrapper(new LegacyBytes(), (byte[]) null); + } + + @Test(expected = IllegalArgumentException.class) + public void testEncodedValueContainsTerm() { + DataType type = new TerminatedWrapper(new LegacyBytes(), "foo"); + byte[] buff = new byte[16]; + type.encode(buff, 0, Bytes.toBytes("hello foobar!")); + } + + @Test + public void testReadWrite() { + byte[] buff = new byte[12]; + for (Order ord : new Order[] { Order.ASCENDING, Order.DESCENDING }) { + for (byte[] term : TERMINATORS) { + for (byte[] val : VALUES) { + DataType type = new TerminatedWrapper(new LegacyBytes(ord), term); + assertEquals(val.length + term.length, type.encode(buff, 0, val)); + assertArrayEquals(val, type.decode(buff, 0)); + } + } + } + } + + @Test + public void testSkip() { + byte[] buff = new byte[12]; + for (Order ord : new Order[] { Order.ASCENDING, Order.DESCENDING }) { + for (byte[] term : TERMINATORS) { + for (byte[] val : VALUES) { + DataType type = new TerminatedWrapper(new LegacyBytes(ord), term); + int expected = type.encode(buff, 0, val); + assertEquals(expected, type.skip(buff, 0)); + } + } + } + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidSkip() { + byte[] buff = Bytes.toBytes("foo"); + DataType type = new TerminatedWrapper(new LegacyBytes(), new byte[] { 0x00 }); + type.skip(buff, 0); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestUnion2.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestUnion2.java new file mode 100644 index 0000000..da5bfa7 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestUnion2.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.util.Order; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestUnion2 { + + /** + * An example Union + */ + private static class SampleUnion1 extends Union2 { + + private static final byte IS_INTEGER = 0x00; + private static final byte IS_STRING = 0x01; + + public SampleUnion1() { + super(new LegacyInteger(), new LegacyStringTerminated(Order.DESCENDING, ".")); + } + + @Override + public int skip(byte[] buff, int offset) { + switch (buff[offset++]) { + case IS_INTEGER: + return typeA.skip(buff, offset); + case IS_STRING: + return typeB.skip(buff, offset); + default: + throw new IllegalArgumentException("Unrecognized encoding format."); + } + } + + @Override + public Object decode(byte[] buff, int offset) { + switch (buff[offset++]) { + case IS_INTEGER: + return typeA.decode(buff, offset); + case IS_STRING: + return typeB.decode(buff, offset); + default: + throw new IllegalArgumentException("Unrecognized encoding format."); + } + } + + @Override + public int encodedLength(Object val) { + Integer i = null; + String s = null; + try { + i = (Integer) val; + } catch (ClassCastException e) {} + try { + s = (String) val; + } catch (ClassCastException e) {} + + if (null != i) return 1 + typeA.encodedLength(i); + if (null != s) return 1 + typeB.encodedLength(s); + throw new IllegalArgumentException("val is not a valid member of this union."); + } + + @Override + public int encode(byte[] buff, int offset, Object val) { + Integer i = null; + String s = null; + try { + i = (Integer) val; + } catch (ClassCastException e) {} + try { + s = (String) val; + } catch (ClassCastException e) {} + + if (null != i) { + buff[offset++] = IS_INTEGER; + return typeA.encode(buff, offset, i); + } else if (null != s) { + buff[offset++] = IS_STRING; + return typeB.encode(buff, offset, s); + } + else + throw new IllegalArgumentException("val is not of a supported type."); + } + } + + @Test + public void testEncodeDecode() { + Integer intVal = Integer.valueOf(10); + String strVal = "hello"; + byte[] buff = new byte[10]; + SampleUnion1 type = new SampleUnion1(); + + type.encode(buff, 0, intVal); + assertTrue(0 == intVal.compareTo(type.decodeA(buff, 0))); + type.encode(buff, 0, strVal); + assertTrue(0 == strVal.compareTo(type.decodeB(buff, 0))); + } + + @Test + public void testSkip() { + Integer intVal = Integer.valueOf(10); + String strVal = "hello"; + byte[] buff = new byte[10]; + SampleUnion1 type = new SampleUnion1(); + + int offset = type.encode(buff, 0, intVal); + assertEquals(offset, type.skip(buff, 0)); + offset = type.encode(buff, 0, strVal); + assertEquals(offset, type.skip(buff, 0)); + } +} -- 1.8.3.2