From e24b8e5cd35413b4d169e61c78fe2f11ea8f97a8 Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Thu, 11 Jul 2013 11:16:33 -0700 Subject: [PATCH] HBASE-8693 Extensible data types API WIP. This patch introduces an extensible data types API for HBase. It is inspired by the following systems: - PostgreSQL. Postgres has a user-extensible data type API, which has been used to great effect by it's user community (ie, PostGIS). The desire is for HBase to expose an equally extensible data type API. One aspect of the Postgres data type is the ability to provide equivalence functions for index operations. This appears to be of critical performance utility for its execution engine. - Orderly. Orderly handles the issue of compound rowkeys by providing convenience classes for handling these kinds of data types. This influence is reflected in the Struct and Union family of classes. - Phoenix. The PDataType enum used in Phoenix provides type hints, similar Postgres's equivalence functions. These appear to be used during query execution for numerical type promotion. This WIP patch is intended to exercise the OrderedBytes API, particularly around numerical value support. The other intention is to establish the set of data types HBase should provide out of the box. This list is becoming quite sizable and includes both the "standard" types as well as "legacy" types to help developers transition their existing applications. The final intention of this WIP patch is to shop around the data types and API for their definition with a wider audience. It does not address the type compatibility concerns expressed by Phoenix's PDataType API (ie, isComparableTo, isCoercibleTo); these will be address in a different JIRA. Patch TODOs include: - better test coverage - more javadocs --- .../apache/hadoop/hbase/types/FixedFloat32.java | 73 ++++ .../apache/hadoop/hbase/types/FixedFloat64.java | 73 ++++ .../org/apache/hadoop/hbase/types/FixedInt32.java | 75 ++++ .../org/apache/hadoop/hbase/types/FixedInt64.java | 74 ++++ .../org/apache/hadoop/hbase/types/HDataType.java | 99 +++++ .../org/apache/hadoop/hbase/types/LegacyBytes.java | 107 ++++++ .../apache/hadoop/hbase/types/LegacyDouble.java | 96 +++++ .../org/apache/hadoop/hbase/types/LegacyFloat.java | 96 +++++ .../apache/hadoop/hbase/types/LegacyInteger.java | 96 +++++ .../org/apache/hadoop/hbase/types/LegacyLong.java | 96 +++++ .../apache/hadoop/hbase/types/LegacyString.java | 110 ++++++ .../org/apache/hadoop/hbase/types/Numeric.java | 103 +++++ .../hadoop/hbase/types/OrderedBytesBase.java | 52 +++ .../java/org/apache/hadoop/hbase/types/Struct.java | 132 +++++++ .../apache/hadoop/hbase/types/StructIterator.java | 86 +++++ .../java/org/apache/hadoop/hbase/types/Union2.java | 82 ++++ .../java/org/apache/hadoop/hbase/types/Union3.java | 67 ++++ .../java/org/apache/hadoop/hbase/types/Union4.java | 66 ++++ .../org/apache/hadoop/hbase/types/Varbinary.java | 65 ++++ .../apache/hadoop/hbase/types/VarbinaryMid.java | 64 +++ .../org/apache/hadoop/hbase/types/Varchar.java | 56 +++ .../org/apache/hadoop/hbase/types/TestStruct.java | 428 +++++++++++++++++++++ .../org/apache/hadoop/hbase/types/TestUnion2.java | 144 +++++++ .../apache/hadoop/hbase/types/TestVarbinary.java | 48 +++ .../hadoop/hbase/types/TestVarbinaryMid.java | 49 +++ .../org/apache/hadoop/hbase/types/TestVarchar.java | 45 +++ 26 files changed, 2482 insertions(+) create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedFloat32.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedFloat64.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedInt32.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedInt64.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/HDataType.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytes.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyDouble.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyFloat.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyInteger.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyLong.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyString.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Numeric.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBytesBase.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructIterator.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union2.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union3.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union4.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Varbinary.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/VarbinaryMid.java create mode 100644 hbase-common/src/main/java/org/apache/hadoop/hbase/types/Varchar.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestStruct.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestUnion2.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestVarbinary.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestVarbinaryMid.java create mode 100644 hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestVarchar.java diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedFloat32.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedFloat32.java new file mode 100644 index 0000000..2efd85d --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedFloat32.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + + +/** + * A float of 32-bits using a fixed-length encoding. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class FixedFloat32 extends OrderedBytesBase { + + public static final FixedFloat32 ASCENDING = new FixedFloat32(Order.ASCENDING); + public static final FixedFloat32 DESCENDING = new FixedFloat32(Order.DESCENDING); + + protected FixedFloat32(Order order) { super(order); } + + @Override + public boolean isNullable() { return false; } + + @Override + public int encodedLength(Float val) { return 5; } + + @Override + public Class encodedClass() { return Float.class; } + + @Override + public Float read(ByteBuffer buff) { + return OrderedBytes.decodeFloat32(buff); + } + + @Override + public void write(ByteBuffer buff, Float val) { + if (null == val) throw new IllegalArgumentException("Null values not supported."); + OrderedBytes.encodeFloat32(buff, val, order); + } + + /** + * Read a float value from the buffer buff. + */ + public float readFloat(ByteBuffer buff) { + return OrderedBytes.decodeFloat32(buff); + } + + /** + * Write instance val into buffer buff. + */ + public void writeFloat(ByteBuffer buff, float val) { + OrderedBytes.encodeFloat32(buff, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedFloat64.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedFloat64.java new file mode 100644 index 0000000..6f57161 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedFloat64.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + + +/** + * A double of 64-bits using a fixed-length encoding. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class FixedFloat64 extends OrderedBytesBase { + + public static final FixedFloat64 ASCENDING = new FixedFloat64(Order.ASCENDING); + public static final FixedFloat64 DESCENDING = new FixedFloat64(Order.DESCENDING); + + protected FixedFloat64(Order order) { super(order); } + + @Override + public boolean isNullable() { return false; } + + @Override + public int encodedLength(Double val) { return 9; } + + @Override + public Class encodedClass() { return Double.class; } + + @Override + public Double read(ByteBuffer buff) { + return OrderedBytes.decodeFloat64(buff); + } + + @Override + public void write(ByteBuffer buff, Double val) { + if (null == val) throw new IllegalArgumentException("Null values not supported."); + OrderedBytes.encodeFloat64(buff, val, order); + } + + /** + * Read a double value from the buffer buff. + */ + public double readDouble(ByteBuffer buff) { + return OrderedBytes.decodeFloat64(buff); + } + + /** + * Write instance val into buffer buff. + */ + public void writeDouble(ByteBuffer buff, double val) { + OrderedBytes.encodeFloat64(buff, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedInt32.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedInt32.java new file mode 100644 index 0000000..59b564c --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedInt32.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + + +/** + * An int of 32-bits using a fixed-length encoding. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class FixedInt32 extends OrderedBytesBase { + + public static final FixedInt32 ASCENDING = new FixedInt32(Order.ASCENDING); + public static final FixedInt32 DESCENDING = new FixedInt32(Order.DESCENDING); + + protected FixedInt32(Order order) { super(order); } + + @Override + public boolean isNullable() { return false; } + + @Override + public int encodedLength(Integer val) { + return 5; + } + + @Override + public Class encodedClass() { return Integer.class; } + + @Override + public Integer read(ByteBuffer buff) { + return OrderedBytes.decodeInt32(buff); + } + + @Override + public void write(ByteBuffer buff, Integer val) { + if (null == val) throw new IllegalArgumentException("Null values not supported."); + OrderedBytes.encodeInt32(buff, val, order); + } + + /** + * Read an int value from the buffer buff. + */ + public int readInt(ByteBuffer buff) { + return OrderedBytes.decodeInt32(buff); + } + + /** + * Write instance val into buffer buff. + */ + public void writeInt(ByteBuffer buff, int val) { + OrderedBytes.encodeInt32(buff, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedInt64.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedInt64.java new file mode 100644 index 0000000..c0c29f2 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/FixedInt64.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + + +/** + * A long of 64-bits using a fixed-length encoding. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class FixedInt64 extends OrderedBytesBase { + + public static final FixedInt64 ASCENDING = new FixedInt64(Order.ASCENDING); + public static final FixedInt64 DESCENDING = new FixedInt64(Order.DESCENDING); + + protected FixedInt64(Order order) { super(order); } + + @Override + public boolean isNullable() { return false; } + + @Override + public int encodedLength(Long val) { return 9; } + + @Override + public Class encodedClass() { return Long.class; } + + @Override + public Long read(ByteBuffer buff) { + return OrderedBytes.decodeInt64(buff); + } + + @Override + public void write(ByteBuffer buff, Long val) { + if (null == val) throw new IllegalArgumentException("Null values not supported."); + OrderedBytes.encodeInt64(buff, val, order); + } + + /** + * Read a long value from the buffer buff. + */ + public long readLong(ByteBuffer buff) { + return OrderedBytes.decodeInt64(buff); + } + + /** + * Write instance val into buffer buff. + */ + public void write(ByteBuffer buff, long val) { + OrderedBytes.encodeInt64(buff, val, order); + } + +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/HDataType.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/HDataType.java new file mode 100644 index 0000000..a47308b --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/HDataType.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; + +/** + * HDataType is the base class for all HBase data types. Data + * type implementations are designed to be serialized to and deserialized from + * {@link ByteBuffer}s. Serialized representations can retain the natural sort + * ordering of the source object, when a suitable encoding is supported by the + * underlying implementation. This is a desirable feature for use in rowkeys + * and column qualifiers. + *

+ * Data type instances are designed to be stateless, thread-safe, and reused. + * Implementations should provide static final instances + * corresponding to each variation on configurable parameters. For instance, + * order-preserving types should provide static ASCENDING and DESCENDING + * instances. It is also encouraged for implementations operating on Java + * primitive types to provide primitive implementations of the + * read and write methods. This advice is a + * performance consideration to clients reading and writing values in tight + * loops. + *

+ */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public interface HDataType { + + /** + * Indicates whether this instance writes encoded byte[]'s + * which preserve the natural sort order of the unencoded value. + * @return true when natural order is preserved, + * false otherwise. + */ + public boolean isOrderPreserving(); + + /** + * Retrieve the sort {@link Order} honored by this data type, or + * null when natural ordering is not preserved. + */ + public Order getOrder(); + + /** + * Indicates whether this instance supports encoding null + * values. This depends on the implementation details of the encoding + * format. All HDataTypes that support null should + * treat null as comparing less than any non-NULL + * value for default sort ordering purposes. + * @return true when null is supported, + * false otherwise. + */ + public boolean isNullable(); + + /** + * Inform consumers how long the encoded byte[] will be. + */ + public int encodedLength(T val); + + /** + * Inform consumers over what type this HDataType operates. + * @return + */ + public Class encodedClass(); + + /** + * Skip the position of buff over the encoded value. + */ + public void skip(ByteBuffer buff); + + /** + * Read an instance of T from the buffer buff. + */ + public T read(ByteBuffer buff); + + /** + * Write instance val into buffer buff. + */ + public void write(ByteBuffer buff, T val); +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytes.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytes.java new file mode 100644 index 0000000..95693e1 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyBytes.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An HDataType for interacting with values encoded using + * {@link Bytes#putBytes(byte[], int, byte[], int, int)}. Intended to make it + * easier to transition away from direct use of {@link Bytes}. + * @see Bytes#putBytes(byte[], int, byte[], int, int) + * @see Varbinary + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +@Deprecated +public class LegacyBytes implements HDataType { + + @Override + public boolean isOrderPreserving() { + return true; + } + + @Override + public Order getOrder() { + return null; + } + + @Override + public boolean isNullable() { + return false; + } + + @Override + public void skip(ByteBuffer buff) { + buff.position(buff.limit()); + } + + @Override + public int encodedLength(byte[] val) { return val.length; } + + @Override + public Class encodedClass() { return byte[].class; } + + /** + * Skip buff ahead length bytes. + */ + public void skip(ByteBuffer buff, int length) { + buff.position(buff.position() + length); + } + + @Override + public byte[] read(ByteBuffer buff) { + return read(buff, buff.limit() - buff.position()); + } + + @Override + public void write(ByteBuffer buff, byte[] val) { + buff.put(val); + } + + /** + * Read a byte[] from the buffer buff. + */ + public byte[] read(ByteBuffer buff, int length) { + byte[] val = new byte[length]; + buff.get(val); + return val; + } + + /** + * Read a byte[] up to length long from the buffer + * buff. + */ + public byte[] read(byte[] buff, int offset, int length) { + return read(ByteBuffer.wrap(buff, offset, buff.length), length); + } + + /** + * Write a byte[] into buff at position + * offset. + * @return incremented offset. + */ + public int write(byte[] buff, int offset, byte[] val) { + return Bytes.putBytes(buff, offset, val, 0, val.length); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyDouble.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyDouble.java new file mode 100644 index 0000000..d2fccf8 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyDouble.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An HDataType for interacting with values encoded using + * {@link Bytes#putDouble(byte[], int, double)}. Intended to make it easier to + * transition away from direct use of {@link Bytes}. + * @see Bytes#putDouble(byte[], int, double) + * @see Bytes#toDouble(byte[]) + * @see FixedFloat64 + * @see Numeric + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +@Deprecated +public class LegacyDouble implements HDataType { + + @Override + public boolean isOrderPreserving() { + return false; + } + + @Override + public Order getOrder() { + return null; + } + + @Override + public boolean isNullable() { + return false; + } + + @Override + public int encodedLength(Double val) { return 8; } + + @Override + public Class encodedClass() { return Double.class; } + + @Override + public void skip(ByteBuffer buff) { + buff.position(buff.position() + 4); + } + + @Override + public Double read(ByteBuffer buff) { + double val = Bytes.toDouble(buff.array(), buff.position()); + skip(buff); + return val; + } + + @Override + public void write(ByteBuffer buff, Double val) { + Bytes.putDouble(buff.array(), buff.position(), val); + skip(buff); + } + + /** + * Read a double value from the buffer buff. + */ + public double readDouble(ByteBuffer buff) { + double val = Bytes.toDouble(buff.array(), buff.position()); + skip(buff); + return val; + } + + /** + * Write instance val into buffer buff. + */ + public void writeDouble(ByteBuffer buff, double val) { + Bytes.putDouble(buff.array(), buff.position(), val); + skip(buff); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyFloat.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyFloat.java new file mode 100644 index 0000000..7733a5c --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyFloat.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An HDataType for interacting with values encoded using + * {@link Bytes#putFloat(byte[], int, float)}. Intended to make it easier to + * transition away from direct use of {@link Bytes}. + * @see Bytes#putFloat(byte[], int, float) + * @see Bytes#toFloat(byte[]) + * @see FixedFloat32 + * @see Numeric + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +@Deprecated +public class LegacyFloat implements HDataType { + + @Override + public boolean isOrderPreserving() { + return false; + } + + @Override + public Order getOrder() { + return null; + } + + @Override + public boolean isNullable() { + return false; + } + + @Override + public int encodedLength(Float val) { return 4; } + + @Override + public Class encodedClass() { return Float.class; } + + @Override + public void skip(ByteBuffer buff) { + buff.position(buff.position() + 4); + } + + @Override + public Float read(ByteBuffer buff) { + float val = Bytes.toFloat(buff.array(), buff.position()); + skip(buff); + return val; + } + + @Override + public void write(ByteBuffer buff, Float val) { + Bytes.putFloat(buff.array(), buff.position(), val); + skip(buff); + } + + /** + * Read a float value from the buffer buff. + */ + public float readFloat(ByteBuffer buff) { + float val = Bytes.toFloat(buff.array(), buff.position()); + skip(buff); + return val; + } + + /** + * Write instance val into buffer buff. + */ + public void writeFloat(ByteBuffer buff, float val) { + Bytes.putFloat(buff.array(), buff.position(), val); + skip(buff); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyInteger.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyInteger.java new file mode 100644 index 0000000..d5ce0f1 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyInteger.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An HDataType for interacting with values encoded using + * {@link Bytes#putInt(byte[], int, int)}. Intended to make it easier to + * transition away from direct use of {@link Bytes}. + * @see Bytes#putInt(byte[], int, int) + * @see Bytes#toInt(byte[]) + * @see FixedInt32 + * @see Numeric + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +@Deprecated +public class LegacyInteger implements HDataType { + + @Override + public boolean isOrderPreserving() { + return false; + } + + @Override + public Order getOrder() { + return null; + } + + @Override + public boolean isNullable() { + return false; + } + + @Override + public int encodedLength(Integer val) { return 4; } + + @Override + public Class encodedClass() { return Integer.class; } + + @Override + public void skip(ByteBuffer buff) { + buff.position(buff.position() + 4); + } + + @Override + public Integer read(ByteBuffer buff) { + int val = Bytes.toInt(buff.array(), buff.position()); + skip(buff); + return val; + } + + @Override + public void write(ByteBuffer buff, Integer val) { + Bytes.putInt(buff.array(), buff.position(), val); + skip(buff); + } + + /** + * Read an int value from the buffer buff. + */ + public int readInt(ByteBuffer buff) { + int val = Bytes.toInt(buff.array(), buff.position()); + skip(buff); + return val; + } + + /** + * Write instance val into buffer buff. + */ + public void writeInt(ByteBuffer buff, int val) { + Bytes.putInt(buff.array(), buff.position(), val); + skip(buff); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyLong.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyLong.java new file mode 100644 index 0000000..a6a4e58 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyLong.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An HDataType for interacting with values encoded using + * {@link Bytes#putLong(byte[], int, long)}. Intended to make it easier to + * transition away from direct use of {@link Bytes}. + * @see Bytes#putLong(byte[], int, long) + * @see Bytes#toLong(byte[]) + * @see FixedInt64 + * @see Numeric + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +@Deprecated +public class LegacyLong implements HDataType { + + @Override + public boolean isOrderPreserving() { + return false; + } + + @Override + public Order getOrder() { + return null; + } + + @Override + public boolean isNullable() { + return false; + } + + @Override + public int encodedLength(Long val) { return 8; } + + @Override + public Class encodedClass() { return Long.class; } + + @Override + public void skip(ByteBuffer buff) { + buff.position(buff.position() + 4); + } + + @Override + public Long read(ByteBuffer buff) { + long val = Bytes.toLong(buff.array(), buff.position()); + skip(buff); + return val; + } + + @Override + public void write(ByteBuffer buff, Long val) { + Bytes.putLong(buff.array(), buff.position(), val); + skip(buff); + } + + /** + * Read a long value from the buffer buff. + */ + public long readLong(ByteBuffer buff) { + long val = Bytes.toLong(buff.array(), buff.position()); + skip(buff); + return val; + } + + /** + * Write instance val into buffer buff. + */ + public void writeLong(ByteBuffer buff, long val) { + Bytes.putLong(buff.array(), buff.position(), val); + skip(buff); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyString.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyString.java new file mode 100644 index 0000000..353af05 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/LegacyString.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Order; + +/** + * An HDataType for interacting with values encoded using + * {@link Bytes#toBytes(String)}. Intended to make it easier to transition + * away from direct use of {@link Bytes}. + * @see Bytes#toBytes(String) + * @see Bytes#toString(byte[]) + * @see Varchar + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +@Deprecated +public class LegacyString implements HDataType { + + @Override + public boolean isOrderPreserving() { + return false; + } + + @Override + public Order getOrder() { + return null; + } + + @Override + public boolean isNullable() { + return false; + } + + @Override + public void skip(ByteBuffer buff) { + buff.position(buff.limit()); + } + + @Override + public int encodedLength(String val) { return Bytes.toBytes(val).length; } + + @Override + public Class encodedClass() { return String.class; } + + /** + * Skip buff ahead length bytes. + */ + public void skip(ByteBuffer buff, int length) { + buff.position(buff.position() + length); + } + + @Override + public String read(ByteBuffer buff) { + String val = Bytes.toString(buff.array()); + skip(buff); + return val; + } + + /** + * Read a String from the buffer buff. + */ + public String read(ByteBuffer buff, int length) { + String val = Bytes.toString(buff.array(), buff.position(), length); + skip(buff, length); + return val; + } + + /** + * Read a String from the buffer buff. + */ + public String read(byte[] buff, int offset, int length) { + return Bytes.toString(buff, offset, length); + } + + @Override + public void write(ByteBuffer buff, String val) { + buff.put(Bytes.toBytes(val)); + } + + /** + * Write a String into buff at position + * offset. + * @return incremented offset. + */ + public int write(byte[] buff, int offset, String val) { + byte[] s = Bytes.toBytes(val); + return Bytes.putBytes(buff, offset, s, 0, s.length); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Numeric.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Numeric.java new file mode 100644 index 0000000..731bc6a --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Numeric.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.HNumeric; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + +/** + * An {@link Number} of arbitrary precision and variable-length encoding. The + * resulting length of encoded values is determined by the numerical (base + * 100) precision, not absolute value. Use this data type anywhere you would + * expect to use a DECIMAL type, a {@link BigDecimal}, a + * {@link BigInteger}, or any time you've parsed floating precision values + * from text. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class Numeric extends OrderedBytesBase { + + public static final Numeric ASCENDING = new Numeric(Order.ASCENDING); + public static final Numeric DESCENDING = new Numeric(Order.DESCENDING); + + protected Numeric(Order order) { super(order); } + + @Override + public int encodedLength(HNumeric val) { + // TODO: this could be done better. + ByteBuffer buff = ByteBuffer.allocate(100); + write(buff, val); + return buff.position(); + } + + @Override + public Class encodedClass() { return HNumeric.class; } + + @Override + public HNumeric read(ByteBuffer buff) { + return OrderedBytes.decodeNumeric(buff); + } + + @Override + public void write(ByteBuffer b, HNumeric val) { + if (null == val) { + OrderedBytes.encodeNull(b, order); + } else if (val.isInteger()) { + OrderedBytes.encodeNumeric(b, val.longValue(), order); + } else if (val.isReal()) { + OrderedBytes.encodeNumeric(b, val.doubleValue(), order); + } else { + OrderedBytes.encodeNumeric(b, val.exactValue(), order); + } + } + + /** + * Read a long value from the buffer buff. + */ + public long readLong(ByteBuffer buff) { + return OrderedBytes.decodeNumeric(buff).longValue(); + } + + /** + * Write instance val into buffer buff. + */ + public void writeLong(ByteBuffer buff, long val) { + OrderedBytes.encodeNumeric(buff, val, order); + } + + /** + * Read a double value from the buffer buff. + */ + public double readDouble(ByteBuffer buff) { + return OrderedBytes.decodeNumeric(buff).doubleValue(); + } + + /** + * Write instance val into buffer buff. + */ + public void writeDouble(ByteBuffer buff, double val) { + OrderedBytes.encodeNumeric(buff, val, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBytesBase.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBytesBase.java new file mode 100644 index 0000000..c4fc483 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/OrderedBytesBase.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + +/** + * Base class for data types backed by the {@link OrderedBytes} encoding + * implementations. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class OrderedBytesBase implements HDataType { + + protected final Order order; + + protected OrderedBytesBase(Order order) { this.order = order; } + + @Override + public boolean isOrderPreserving() { return true; } + + @Override + public Order getOrder() { return order; } + + @Override + public boolean isNullable() { return true; } + + @Override + public void skip(ByteBuffer buff) { + OrderedBytes.skip(buff); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java new file mode 100644 index 0000000..6a00506 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Iterator; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; + +/** + * Struct is a simple example of a {@link HDataType} for + * implementing the "compound rowkey" schema design strategy. The + * Struct member values are encoded onto the buffer + * in the order they are declared. It also provides a convenience + * {@link StructIterator} for consuming the sequence of values. The + * Struct sorts according to the composite order of its field + * values. Structs are not nullable but their component fields + * may be. Users may find is more appropriate to provide their own custom + * {@link HDataType} for encoding application objects rather than using this + * generic implementation. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class Struct implements HDataType { + + @SuppressWarnings("rawtypes") + protected final HDataType[] fields; + protected final boolean isOrderPreserving; + + /** + * Create a new Struct instance defined as the sequence of + * HDataTypes in memberTypes. + */ + @SuppressWarnings("rawtypes") + public Struct(HDataType[] memberTypes) { + this.fields = memberTypes; + // rewrite the fields array to use VarbinaryMid instead of Varbinary in + // non-terminal fields. + for (int i = 0; i < this.fields.length; i++) { + if (i != this.fields.length - 1) { + if (this.fields[i] == Varbinary.ASCENDING) this.fields[i] = VarbinaryMid.ASCENDING; + else if (this.fields[i] == Varbinary.DESCENDING) this.fields[i] = VarbinaryMid.DESCENDING; + } + } + boolean preservesOrder = true; + for (HDataType dt : this.fields) { + if (!dt.isOrderPreserving()) { + preservesOrder = false; + break; + } + } + this.isOrderPreserving = preservesOrder; + } + + @Override + public boolean isOrderPreserving() { return isOrderPreserving; } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { return false; } + + @SuppressWarnings("unchecked") + @Override + public int encodedLength(Object[] val) { + assert fields.length == val.length; + int sum = 0; + for (int i = 0; i < fields.length; i++) + sum += fields[i].encodedLength(val[i]); + return sum; + } + + @Override + public Class encodedClass() { return Object[].class; } + + /** + * Retrieve an {@link Iterator} over the values encoded in b. + * The byte[] backing b is not modified by use of + * the Iterator, however the state of b is. To + * create multiple Iterators over the same backing + * ByteBuffer, construct the Iterators over + * duplicates ( {@link ByteBuffer#duplicate()} ) of b. + */ + public StructIterator iterator(ByteBuffer buff) { + return new StructIterator(buff, fields); + } + + @Override + public void skip(ByteBuffer buff) { + StructIterator it = iterator(buff); + while (it.hasNext()) + it.skip(); + } + + @Override + public Object[] read(ByteBuffer buff) { + ArrayList values = new ArrayList(fields.length); + Iterator it = iterator(buff); + while (it.hasNext()) + values.add(it.next()); + assert values.size() == fields.length; + return values.toArray(); + } + + @SuppressWarnings("unchecked") + @Override + public void write(ByteBuffer buff, Object[] val) { + assert fields.length == val.length; + for (int i = 0; i < fields.length; i++) { + fields[i].write(buff, val[i]); + } + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructIterator.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructIterator.java new file mode 100644 index 0000000..37211a6 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructIterator.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * An {@link Iterator} over encoded Struct members. + *

+ * This iterates over each serialized Struct field from the + * specified HDataTypes[] definition. It allows you to read + * the field or skip over its serialized bytes using {@link #next()} and + * {@link #skip()}, respectively. This is in contrast to the + * Struct method which allow you to + * {@link Struct#read(ByteBuffer)} or {@link Struct#skip(ByteBuffer)} over the + * entire Struct at once. + *

+ *

+ * This iterator may also be used to read bytes from any Struct + * for which the specified HDataType[] is a prefix. For + * example, if the specified Struct definition has a + * {@link Numeric} and a {@link Varchar} field, you may parse the serialized + * output of a Struct whose fields are {@link Numeric}, + * {@link Varchar}, and {@link Binary}. The iterator would return a number + * followed by a String. The trailing byte[] would + * be ignored. + *

+ */ +public class StructIterator implements Iterator { + + protected final ByteBuffer buff; + @SuppressWarnings("rawtypes") + protected final HDataType[] types; + protected int idx = 0; + + /** + * Construct StructIterator over the values encoded in + * buff using the specified types definition. + * @param buff The buffer from which to read encoded values. + * @param types The sequence of types to use as the schema for this + * Struct. + */ + public StructIterator(final ByteBuffer buff, @SuppressWarnings("rawtypes") HDataType[] types) { + this.buff = buff; + this.types = types; + } + + @Override + public boolean hasNext() { + return idx < types.length && buff.position() != buff.limit(); + } + + @Override + public void remove() { throw new UnsupportedOperationException(); } + + @Override + public Object next() { + if (!hasNext()) throw new NoSuchElementException(); + return types[idx++].read(buff); + } + + /** + * Bypass the next encoded value. + */ + public void skip() { + if (!hasNext()) throw new NoSuchElementException(); + types[idx++].skip(buff); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union2.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union2.java new file mode 100644 index 0000000..edd163f --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union2.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; + +/** + * The Union family of {@link HDataType}s encode one of a fixed + * set of Objects. They provide convenience methods which handle + * type casting on your behalf. + */ +@SuppressWarnings("unchecked") +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class Union2 implements HDataType { + + protected final HDataType typeA; + protected final HDataType typeB; + + /** + * Create an instance of Union2 over the set of specified + * types. + */ + public Union2(HDataType typeA, HDataType typeB) { + this.typeA = typeA; + this.typeB = typeB; + } + + @Override + public boolean isOrderPreserving() { + return typeA.isOrderPreserving() && typeB.isOrderPreserving(); + } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { + return typeA.isNullable() && typeB.isNullable(); + } + + @Override + public Class encodedClass() { + throw new UnsupportedOperationException( + "Union types do not expose a definitive encoded class."); + } + + /** + * Read an instance of the first type parameter from buffer + * buff. + */ + public A readA(ByteBuffer buff) { + return (A) read(buff); + } + + /** + * Read an instance of the second type parameter from buffer + * buff. + */ + public B readB(ByteBuffer buff) { + return (B) read(buff); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union3.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union3.java new file mode 100644 index 0000000..e001917 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union3.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; + +/** + * The Union family of {@link HDataType}s encode one of a fixed + * collection of Objects. They provide convenience methods which handle type + * casting on your behalf. + * @see Union2 + */ +@SuppressWarnings("unchecked") +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class Union3 extends Union2 { + + protected final HDataType typeC; + + /** + * Create an instance of Union3 over the set of specified + * types. + */ + public Union3(HDataType typeA, HDataType typeB, HDataType typeC) { + super(typeA, typeB); + this.typeC = typeC; + } + + @Override + public boolean isOrderPreserving() { + return super.isOrderPreserving() && typeC.isOrderPreserving(); + } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { + return super.isNullable() && typeC.isNullable(); + } + + /** + * Read an instance of the third type parameter from buffer b. + */ + public C readC(ByteBuffer buff) { + return (C) read(buff); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union4.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union4.java new file mode 100644 index 0000000..72c8615 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Union4.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; + +/** + * The Union family of {@link HDataType}s encode one of a fixed + * collection of Objects. They provide convenience methods which handle type + * casting on your behalf. + */ +@SuppressWarnings("unchecked") +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class Union4 extends Union3 { + + protected final HDataType typeD; + + /** + * Create an instance of Union4 over the set of specified + * types. + */ + public Union4(HDataType typeA, HDataType typeB, HDataType typeC, HDataType typeD) { + super(typeA, typeB, typeC); + this.typeD = typeD; + } + + @Override + public boolean isOrderPreserving() { + return super.isOrderPreserving() && typeD.isOrderPreserving(); + } + + @Override + public Order getOrder() { return null; } + + @Override + public boolean isNullable() { + return super.isNullable() && typeD.isNullable(); + } + + /** + * Read an instance of the fourth type parameter from buffer b. + */ + public D readD(ByteBuffer b) { + return (D) read(b); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Varbinary.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Varbinary.java new file mode 100644 index 0000000..5dc8fbc --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Varbinary.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + +/** + * A byte[] of variable-length. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class Varbinary extends OrderedBytesBase { + + public static final Varbinary ASCENDING = new Varbinary(Order.ASCENDING); + public static final Varbinary DESCENDING = new Varbinary(Order.DESCENDING); + + protected Varbinary(Order order) { super(order); } + + @Override + public int encodedLength(byte[] val) { + return null == val ? + (Order.ASCENDING == order ? 1 : 2) : + (Order.ASCENDING == order ? val.length + 1 : val.length + 2); + } + + @Override + public Class encodedClass() { return byte[].class; } + + @Override + public byte[] read(ByteBuffer buff) { + return OrderedBytes.decodeBlobLast(buff); + } + + @Override + public void write(ByteBuffer buff, byte[] val) { + OrderedBytes.encodeBlobLast(buff, val, order); + } + + /** + * Write a subset of val to buff. + */ + public void write(ByteBuffer buff, byte[] val, int length, int offset) { + OrderedBytes.encodeBlobLast(buff, val, offset, length, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/VarbinaryMid.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/VarbinaryMid.java new file mode 100644 index 0000000..f942751 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/VarbinaryMid.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + +/** + * An alternative to {@link Varbinary} for use by {@link Struct} fields that + * do not terminate the fields list. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class VarbinaryMid extends OrderedBytesBase { + + public static final VarbinaryMid ASCENDING = new VarbinaryMid(Order.ASCENDING); + public static final VarbinaryMid DESCENDING = new VarbinaryMid(Order.DESCENDING); + + protected VarbinaryMid(Order order) { super(order); } + + @Override + public int encodedLength(byte[] val) { + return null == val ? 1 : OrderedBytes.blobMidEncodedLength(val.length); + } + + @Override + public Class encodedClass() { return byte[].class; } + + @Override + public byte[] read(ByteBuffer buff) { + return OrderedBytes.decodeBlobMid(buff); + } + + @Override + public void write(ByteBuffer buff, byte[] val) { + OrderedBytes.encodeBlobMid(buff, val, order); + } + + /** + * Write a subset of val to buff. + */ + public void write(ByteBuffer buff, byte[] val, int length, int offset) { + OrderedBytes.encodeBlobLast(buff, val, offset, length, order); + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Varchar.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Varchar.java new file mode 100644 index 0000000..f216b61 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Varchar.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; + +/** + * A String of variable-length. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class Varchar extends OrderedBytesBase { + + public static final Varchar ASCENDING = new Varchar(Order.ASCENDING); + public static final Varchar DESCENDING = new Varchar(Order.DESCENDING); + + protected Varchar(Order order) { super(order); } + + @Override + public int encodedLength(String val) { + return null == val ? 1 : val.getBytes(OrderedBytes.UTF8).length + 2; + } + + @Override + public Class encodedClass() { return String.class; } + + @Override + public String read(ByteBuffer buff) { + return OrderedBytes.decodeString(buff); + } + + @Override + public void write(ByteBuffer buff, String val) { + OrderedBytes.encodeString(buff, val, order); + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestStruct.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestStruct.java new file mode 100644 index 0000000..3c5a818 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestStruct.java @@ -0,0 +1,428 @@ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import java.lang.reflect.Constructor; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; + +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.HNumeric; +import org.apache.hadoop.hbase.util.Order; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * This class both tests and demonstrates how to construct compound rowkeys + * from a POJO. The code under test is {@link Struct}. + * {@link SpecializedPojo1Type1} demonstrates how one might create their own + * custom data type extension for an application POJO. + */ +@RunWith(Parameterized.class) +public class TestStruct { + + private Struct generic; + @SuppressWarnings("rawtypes") + private HDataType specialized; + private Object[][] constructorArgs; + + public TestStruct(Struct generic, @SuppressWarnings("rawtypes") HDataType specialized, + Object[][] constructorArgs) { + this.generic = generic; + this.specialized = specialized; + this.constructorArgs = constructorArgs; + } + + @Parameters + public static Collection params() { + Object[][] pojo1Type1Args = { + new Object[] { "foo", -5, new HNumeric(10.001) }, + new Object[] { "foo", 100, new HNumeric(-7.0) }, + new Object[] { "foo", 100, new HNumeric(10.001) }, + new Object[] { "bar", -5, new HNumeric(10.001) }, + new Object[] { "bar", 100, new HNumeric(10.001) }, + new Object[] { "baz", -5, new HNumeric(10.001) }, + }; + + Object[][] pojo1Type2Args = { + new Object[] { "foo", new HNumeric(-5), 10.001 }, + new Object[] { "foo", new HNumeric(100), -7.0 }, + new Object[] { "foo", new HNumeric(100), 10.001 }, + new Object[] { "bar", new HNumeric(-5), 10.001 }, + new Object[] { "bar", new HNumeric(100), 10.001 }, + new Object[] { "baz", new HNumeric(-5), 10.001 }, + }; + + Object[][] pojo2Type1Args = { + new Object[] { null, "it".getBytes(), "was", "the".getBytes() }, + new Object[] { "best".getBytes(), null, "of", "times,".getBytes() }, + new Object[] { "it".getBytes(), "was".getBytes(), null, "the".getBytes() }, + new Object[] { "worst".getBytes(), "of".getBytes(), "times,", null }, + new Object[] { null, null, null, null }, + }; + + Object[][] params = new Object[][] { + { SpecializedPojo1Type1.GENERIC, new SpecializedPojo1Type1(), pojo1Type1Args }, + { SpecializedPojo1Type2.GENERIC, new SpecializedPojo1Type2(), pojo1Type2Args }, + { SpecializedPojo2Type1.GENERIC, new SpecializedPojo2Type1(), pojo2Type1Args }, + }; + return Arrays.asList(params); + } + + static final Comparator NULL_SAFE_BYTES_COMPARATOR = + new Comparator() { + @Override + public int compare(byte[] o1, byte[] o2) { + if (o1 == o2) return 0; + if (null == o1) return -1; + if (null == o2) return 1; + return Bytes.compareTo(o1, o2); + } + }; + + /** + * A simple object to serialize. + */ + private static class Pojo1 implements Comparable { + final String stringFieldAsc; + final int intFieldDsc; + final double doubleFieldAsc; + final transient String str; + + public Pojo1(Object... argv) { + stringFieldAsc = (String) argv[0]; + intFieldDsc = + argv[1] instanceof Integer ? (Integer) argv[1] : ((HNumeric) argv[1]).intValue(); + doubleFieldAsc = + argv[2] instanceof Double ? (Double) argv[2] : ((HNumeric) argv[2]).doubleValue(); + str = new StringBuilder() + .append("{ ") + .append(null == stringFieldAsc ? "" : "\"") + .append(stringFieldAsc) + .append(null == stringFieldAsc ? "" : "\"").append(", ") + .append(intFieldDsc).append(", ") + .append(doubleFieldAsc) + .append(" }") + .toString(); + } + + @Override + public String toString() { + return str; + } + + @Override + public int compareTo(Pojo1 o) { + int cmp = stringFieldAsc.compareTo(o.stringFieldAsc); + if (cmp != 0) return cmp; + cmp = -Integer.valueOf(intFieldDsc).compareTo(Integer.valueOf(o.intFieldDsc)); + if (cmp != 0) return cmp; + return Double.compare(doubleFieldAsc, o.doubleFieldAsc); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (null == o) return false; + if (!(o instanceof Pojo1)) return false; + Pojo1 that = (Pojo1) o; + return 0 == this.compareTo(that); + } + } + + /** + * A simple object to serialize. + */ + private static class Pojo2 implements Comparable { + final byte[] byteField1Asc; + final byte[] byteField2Dsc; + final String stringFieldDsc; + final byte[] byteField3Dsc; + final transient String str; + + public Pojo2(Object... vals) { + byteField1Asc = (byte[]) vals[0]; + byteField2Dsc = (byte[]) vals[1]; + stringFieldDsc = (String) vals[2]; + byteField3Dsc = (byte[]) vals[3]; + str = new StringBuilder() + .append("{ ") + .append(Bytes.toStringBinary(byteField1Asc)).append(", ") + .append(Bytes.toStringBinary(byteField2Dsc)).append(", ") + .append(null == stringFieldDsc ? "" : "\"") + .append(stringFieldDsc) + .append(null == stringFieldDsc ? "" : "\"").append(", ") + .append(Bytes.toStringBinary(byteField3Dsc)) + .append(" }") + .toString(); + } + + @Override + public String toString() { + return str; + } + + @Override + public int compareTo(Pojo2 o) { + int cmp = NULL_SAFE_BYTES_COMPARATOR.compare(byteField1Asc, o.byteField1Asc); + if (cmp != 0) return cmp; + cmp = -NULL_SAFE_BYTES_COMPARATOR.compare(byteField2Dsc, o.byteField2Dsc); + if (cmp != 0) return cmp; + if (stringFieldDsc == o.stringFieldDsc) cmp = 0; + else if (null == stringFieldDsc) cmp = 1; + else if (null == o.stringFieldDsc) cmp = -1; + else cmp = -stringFieldDsc.compareTo(o.stringFieldDsc); + if (cmp != 0) return cmp; + return -NULL_SAFE_BYTES_COMPARATOR.compare(byteField3Dsc, o.byteField3Dsc); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (null == o) return false; + if (!(o instanceof Pojo2)) return false; + Pojo2 that = (Pojo2) o; + return 0 == this.compareTo(that); + } + } + + /** + * A custom data type implementation specialized for {@link Pojo1}. + */ + private static class SpecializedPojo1Type1 implements HDataType { + + private static final Varchar stringField = Varchar.ASCENDING; + private static final FixedInt32 intField = FixedInt32.DESCENDING; + private static final Numeric doubleField = Numeric.ASCENDING; + + /** + * The {@link Struct} equivalent of this type. + */ + public static Struct GENERIC = new Struct( + new HDataType[] { stringField, intField, doubleField }); + + @Override + public boolean isOrderPreserving() { + return true; + } + + @Override + public Order getOrder() { + return null; + } + + @Override + public boolean isNullable() { + return false; + } + + @Override + public int encodedLength(Pojo1 val) { + return + stringField.encodedLength(val.stringFieldAsc) + + intField.encodedLength(val.intFieldDsc) + + doubleField.encodedLength(new HNumeric(val.doubleFieldAsc)); + } + + @Override + public Class encodedClass() { return Pojo1.class; } + + @Override + public void skip(ByteBuffer buff) { + stringField.skip(buff); + intField.skip(buff); + doubleField.skip(buff); + } + + @Override + public Pojo1 read(ByteBuffer buff) { + return new Pojo1(new Object[] { + stringField.read(buff), + intField.readInt(buff), + doubleField.readDouble(buff) + }); + } + + @Override + public void write(ByteBuffer buff, Pojo1 val) { + stringField.write(buff, val.stringFieldAsc); + intField.writeInt(buff, val.intFieldDsc); + doubleField.writeDouble(buff, val.doubleFieldAsc); + } + } + + /** + * A custom data type implementation specialized for {@link Pojo1}. + */ + private static class SpecializedPojo1Type2 implements HDataType { + + private static final Varchar stringField = Varchar.ASCENDING; + private static final Numeric intField = Numeric.DESCENDING; + private static final FixedFloat64 doubleField = FixedFloat64.ASCENDING; + + /** + * The {@link Struct} equivalent of this type. + */ + public static Struct GENERIC = new Struct( + new HDataType[] { stringField, intField, doubleField }); + + @Override + public boolean isOrderPreserving() { + return true; + } + + @Override + public Order getOrder() { + return null; + } + + @Override + public boolean isNullable() { + return false; + } + + @Override + public int encodedLength(Pojo1 val) { + return + stringField.encodedLength(val.stringFieldAsc) + + intField.encodedLength(new HNumeric(val.intFieldDsc)) + + doubleField.encodedLength(val.doubleFieldAsc); + } + + @Override + public Class encodedClass() { return Pojo1.class; } + + @Override + public void skip(ByteBuffer buff) { + stringField.skip(buff); + intField.skip(buff); + doubleField.skip(buff); + } + + @Override + public Pojo1 read(ByteBuffer buff) { + return new Pojo1(new Object[] { + stringField.read(buff), + intField.read(buff).intValue(), + doubleField.readDouble(buff) + }); + } + + @Override + public void write(ByteBuffer buff, Pojo1 val) { + stringField.write(buff, val.stringFieldAsc); + intField.writeLong(buff, val.intFieldDsc); + doubleField.writeDouble(buff, val.doubleFieldAsc); + } + } + + /** + * A custom data type implementation specialized for {@link Pojo2}. + */ + private static class SpecializedPojo2Type1 implements HDataType { + + private static VarbinaryMid byteField1 = VarbinaryMid.ASCENDING; + private static VarbinaryMid byteField2 = VarbinaryMid.DESCENDING; + private static Varchar stringField = Varchar.DESCENDING; + private static Varbinary byteField3 = Varbinary.DESCENDING; + + + /** + * The {@link Struct} equivalent of this type. + */ + public static Struct GENERIC = new Struct( + new HDataType[] { byteField1, byteField2, stringField, byteField3 }); + + @Override + public boolean isOrderPreserving() { + return true; + } + + @Override + public Order getOrder() { + return null; + } + + @Override + public boolean isNullable() { + return false; + } + + @Override + public int encodedLength(Pojo2 val) { + return + byteField1.encodedLength(val.byteField1Asc) + + byteField2.encodedLength(val.byteField2Dsc) + + stringField.encodedLength(val.stringFieldDsc) + + byteField3.encodedLength(val.byteField3Dsc); + } + + @Override + public Class encodedClass() { return Pojo2.class; } + + @Override + public void skip(ByteBuffer buff) { + byteField1.skip(buff); + byteField2.skip(buff); + stringField.skip(buff); + byteField3.skip(buff); + } + + @Override + public Pojo2 read(ByteBuffer buff) { + return new Pojo2( + byteField1.read(buff), + byteField2.read(buff), + stringField.read(buff), + byteField3.read(buff)); + } + + @Override + public void write(ByteBuffer buff, Pojo2 val) { + byteField1.write(buff, val.byteField1Asc); + byteField2.write(buff, val.byteField2Dsc); + stringField.write(buff, val.stringFieldDsc); + byteField3.write(buff, val.byteField3Dsc); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testOrderPreservation() throws Exception { + Object[] vals = new Object[constructorArgs.length]; + byte[][] encodedGeneric = new byte[constructorArgs.length][]; + byte[][] encodedSpecialized = new byte[constructorArgs.length][]; + Constructor ctor = specialized.encodedClass().getConstructor(Object[].class); + for (int i = 0; i < vals.length; i++) { + vals[i] = ctor.newInstance(new Object[] { constructorArgs[i] }); + encodedGeneric[i] = new byte[generic.encodedLength(constructorArgs[i])]; + encodedSpecialized[i] = new byte[specialized.encodedLength(vals[i])]; + } + + // populate our arrays + for (int i = 0; i < vals.length; i++) { + generic.write(ByteBuffer.wrap(encodedGeneric[i]), constructorArgs[i]); + specialized.write(ByteBuffer.wrap(encodedSpecialized[i]), vals[i]); + assertArrayEquals(encodedGeneric[i], encodedSpecialized[i]); + } + + Arrays.sort(vals); + Arrays.sort(encodedGeneric, NULL_SAFE_BYTES_COMPARATOR); + Arrays.sort(encodedSpecialized, NULL_SAFE_BYTES_COMPARATOR); + + for (int i = 0; i < vals.length; i++) { + assertEquals( + "Struct encoder does not preserve sort order at position " + i, + vals[i], + ctor.newInstance(new Object[] { generic.read(ByteBuffer.wrap(encodedGeneric[i])) })); + assertEquals( + "Specialized encoder does not preserve sort order at position " + i, + vals[i], specialized.read(ByteBuffer.wrap(encodedSpecialized[i]))); + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestUnion2.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestUnion2.java new file mode 100644 index 0000000..7de7726 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestUnion2.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.hbase.util.Order; +import org.apache.hadoop.hbase.util.OrderedBytes; +import org.junit.Test; + +public class TestUnion2 { + + /** + * An example Union + */ + private static class SampleUnion1 extends Union2 { + + private final Order ord; + + public SampleUnion1(Order order) { + super( + Order.ASCENDING == order ? FixedInt32.ASCENDING : FixedInt32.DESCENDING, + Order.ASCENDING == order ? Varchar.ASCENDING : Varchar.DESCENDING); + ord = order; + } + + @Override + public void skip(ByteBuffer buff) { + // both types A and B support generic OrderedBytes.skip, so this is easy + OrderedBytes.skip(buff); + } + + @Override + public Object read(ByteBuffer buff) { + byte header = ord.apply(buff.array()[buff.position()]); + switch (header) { + case OrderedBytes.FIXED_INT32: + return typeA.read(buff); + case OrderedBytes.TEXT: + return typeB.read(buff); + default: + throw new IllegalArgumentException("Unrecognized encoding format."); + } + } + + @Override + public int encodedLength(Object val) { + Integer i = null; + String s = null; + try { + i = (Integer) val; + } catch (ClassCastException e) {} + try { + s = (String) val; + } catch (ClassCastException e) {} + + if (null != i) return typeA.encodedLength(i); + if (null != s) return typeB.encodedLength(s); + throw new IllegalArgumentException("val is not a valid member of this union."); + } + + @Override + public void write(ByteBuffer buff, Object val) { + Integer i = null; + String s = null; + try { + i = (Integer) val; + } catch (ClassCastException e) {} + try { + s = (String) val; + } catch (ClassCastException e) {} + + if (null != i) typeA.write(buff, i); + else if (null != s) typeB.write(buff, s); + else + throw new IllegalArgumentException("val is not of a supported type."); + } + + } + + @Test + public void testEncodeDecode() { + Integer intVal = Integer.valueOf(10); + String strVal = "hello"; + ByteBuffer buff = ByteBuffer.allocate(10); + + for (SampleUnion1 type : new SampleUnion1[] { + new SampleUnion1(Order.ASCENDING), + new SampleUnion1(Order.DESCENDING) + }) { + buff.clear(); + type.write(buff, intVal); + buff.flip(); + assertTrue(0 == intVal.compareTo(type.readA(buff))); + + buff.clear(); + type.write(buff, strVal); + buff.flip(); + assertTrue(0 == strVal.compareTo(type.readB(buff))); + } + } + + @Test + public void testSkip() { + Integer intVal = Integer.valueOf(10); + String strVal = "hello"; + ByteBuffer buff = ByteBuffer.allocate(10); + + for (SampleUnion1 type : new SampleUnion1[] { + new SampleUnion1(Order.ASCENDING), + new SampleUnion1(Order.DESCENDING) + }) { + buff.clear(); + type.write(buff, intVal); + buff.flip(); + type.skip(buff); + assertEquals(buff.limit(), buff.position()); + + buff.clear(); + type.write(buff, strVal); + buff.flip(); + type.skip(buff); + assertEquals(buff.limit(), buff.position()); + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestVarbinary.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestVarbinary.java new file mode 100644 index 0000000..9709371 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestVarbinary.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertEquals; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; + +public class TestVarbinary { + + static final byte[][] VALUES = new byte[][] { + null, Bytes.toBytes(""), Bytes.toBytes("1"), Bytes.toBytes("22"), Bytes.toBytes("333"), + Bytes.toBytes("4444"), Bytes.toBytes("55555"), Bytes.toBytes("666666"), + Bytes.toBytes("7777777"), Bytes.toBytes("88888888"), Bytes.toBytes("999999999"), + }; + + @Test + public void testEncodedLength() { + ByteBuffer buff = ByteBuffer.allocate(20); + for (HDataType type : new Varbinary[] { Varbinary.ASCENDING, Varbinary.DESCENDING }) { + for (byte[] val : VALUES) { + buff.clear(); + type.write(buff, val); + assertEquals( + "encodedLength does not match actual, " + Bytes.toStringBinary(val), + buff.position(), type.encodedLength(val)); + } + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestVarbinaryMid.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestVarbinaryMid.java new file mode 100644 index 0000000..7a77248 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestVarbinaryMid.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertEquals; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; + +public class TestVarbinaryMid { + + static final byte[][] VALUES = new byte[][] { + null, Bytes.toBytes(""), Bytes.toBytes("1"), Bytes.toBytes("22"), Bytes.toBytes("333"), + Bytes.toBytes("4444"), Bytes.toBytes("55555"), Bytes.toBytes("666666"), + Bytes.toBytes("7777777"), Bytes.toBytes("88888888"), Bytes.toBytes("999999999"), + }; + + @Test + public void testEncodedLength() { + ByteBuffer buff = ByteBuffer.allocate(20); + for (HDataType type : + new VarbinaryMid[] { VarbinaryMid.ASCENDING, VarbinaryMid.DESCENDING }) { + for (byte[] val : VALUES) { + buff.clear(); + type.write(buff, val); + assertEquals( + "encodedLength does not match actual, " + Bytes.toStringBinary(val), + buff.position(), type.encodedLength(val)); + } + } + } +} diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestVarchar.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestVarchar.java new file mode 100644 index 0000000..4bba91b --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/types/TestVarchar.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.types; + +import static org.junit.Assert.assertEquals; + +import java.nio.ByteBuffer; + +import org.junit.Test; + +public class TestVarchar { + + static final String[] VALUES = + new String[] { null, "", "1", "22", "333", "4444", "55555", "666666", + "7777777", "88888888", "999999999" }; + + @Test + public void testEncodedLength() { + ByteBuffer buff = ByteBuffer.allocate(20); + for (HDataType type : new Varchar[] { Varchar.ASCENDING, Varchar.DESCENDING }) { + for (String val : VALUES) { + buff.clear(); + type.write(buff, val); + assertEquals( + "encodedLength does not match actual, " + val, + buff.position(), type.encodedLength(val)); + } + } + } +} -- 1.8.3.2