From 74da753cde475b5693aeb107debab4b2546a277c Mon Sep 17 00:00:00 2001 From: QilinCao Date: Mon, 30 Oct 2017 20:55:11 +0800 Subject: [PATCH] HBASE-19103 Add BigDecimalComparator for filter --- .../hadoop/hbase/filter/BigDecimalComparator.java | 122 +++++++++++++++++++++ .../src/main/protobuf/Comparator.proto | 4 + hbase-protocol/src/main/protobuf/Comparator.proto | 4 + .../hbase/filter/TestBigDecimalComparator.java | 93 ++++++++++++++++ .../hbase/filter/TestComparatorSerialization.java | 9 ++ .../hadoop/hbase/regionserver/TestHRegion.java | 43 ++++++++ 6 files changed, 275 insertions(+) create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BigDecimalComparator.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestBigDecimalComparator.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BigDecimalComparator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BigDecimalComparator.java new file mode 100644 index 0000000..930fa65 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/BigDecimalComparator.java @@ -0,0 +1,122 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; + +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.ComparatorProtos; +import org.apache.hadoop.hbase.util.ByteBufferUtils; +import org.apache.hadoop.hbase.util.Bytes; + +import org.apache.yetus.audience.InterfaceAudience; + +/** + * A BigDecimal comparator which numerical compares against the specified byte array + */ +@InterfaceAudience.Public +public class BigDecimalComparator extends ByteArrayComparable { + private BigDecimal bigDecimal; + private final int hashCode; + + public BigDecimalComparator(BigDecimal value) { + super(Bytes.toBytes(value)); + this.bigDecimal = value; + this.hashCode = value.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || !(obj instanceof BigDecimalComparator)) { + return false; + } + if (this == obj) { + return true; + } + BigDecimalComparator bdc = (BigDecimalComparator) obj; + return bdc.hashCode() == hashCode && this.toString().equals(bdc.toString()); + } + + @Override + public int hashCode() { + return hashCode; + } + + @Override + public String toString() { + return this.bigDecimal.toString(); + } + + @Override + public int compareTo(byte[] value, int offset, int length) { + BigDecimal that = Bytes.toBigDecimal(value, offset, length); + return this.bigDecimal.compareTo(that); + } + + @Override + public int compareTo(ByteBuffer value, int offset, int length) { + BigDecimal that = ByteBufferUtils.toBigDecimal(value, offset, length); + return this.bigDecimal.compareTo(that); + } + + /** + * @return The comparator serialized using pb + */ + @Override + public byte[] toByteArray() { + ComparatorProtos.BigDecimalComparator.Builder builder = + ComparatorProtos.BigDecimalComparator.newBuilder(); + builder.setComparable(ProtobufUtil.toByteArrayComparable(this.value)); + return builder.build().toByteArray(); + } + + /** + * @param pbBytes A pb serialized {@link BigDecimalComparator} instance + * @return An instance of {@link BigDecimalComparator} made from bytes + * @throws DeserializationException + * @see #toByteArray + */ + public static BigDecimalComparator parseFrom(final byte[] pbBytes) + throws DeserializationException { + ComparatorProtos.BigDecimalComparator proto; + try { + proto = ComparatorProtos.BigDecimalComparator.parseFrom(pbBytes); + } catch (InvalidProtocolBufferException e) { + throw new DeserializationException(e); + } + return new BigDecimalComparator(Bytes.toBigDecimal(proto.getComparable().getValue() + .toByteArray())); + } + + /** + * @param other the other comparator + * @return true if and only if the fields of the comparator that are serialized are equal to the + * corresponding fields in other. Used for testing. + */ + boolean areSerializedFieldsEqual(BigDecimalComparator other) { + if (other == this) { + return true; + } + return super.areSerializedFieldsEqual(other); + } +} diff --git a/hbase-protocol-shaded/src/main/protobuf/Comparator.proto b/hbase-protocol-shaded/src/main/protobuf/Comparator.proto index 822fd2b..55253aa 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Comparator.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Comparator.proto @@ -73,3 +73,7 @@ message RegexStringComparator { message SubstringComparator { required string substr = 1; } + +message BigDecimalComparator { + required ByteArrayComparable comparable = 1; +} diff --git a/hbase-protocol/src/main/protobuf/Comparator.proto b/hbase-protocol/src/main/protobuf/Comparator.proto index 496b68d..878a179 100644 --- a/hbase-protocol/src/main/protobuf/Comparator.proto +++ b/hbase-protocol/src/main/protobuf/Comparator.proto @@ -72,3 +72,7 @@ message RegexStringComparator { message SubstringComparator { required string substr = 1; } + +message BigDecimalComparator { + required ByteArrayComparable comparable = 1; +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestBigDecimalComparator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestBigDecimalComparator.java new file mode 100644 index 0000000..428ef98 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestBigDecimalComparator.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hbase.filter; + +import java.math.BigDecimal; + +import org.apache.hadoop.hbase.testclassification.FilterTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({ FilterTests.class, SmallTests.class }) +public class TestBigDecimalComparator { + + @Test + public void testEqualsValue() { + // given + BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE); + BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE); + byte[] value1 = Bytes.toBytes(bd1); + byte[] value2 = Bytes.toBytes(bd2); + BigDecimalComparator comparator1 = new BigDecimalComparator(bd1); + BigDecimalComparator comparator2 = new BigDecimalComparator(bd2); + + // when + int comp1 = comparator1.compareTo(value1); + int comp2 = comparator2.compareTo(value2); + + // then + Assert.assertEquals(0, comp1); + Assert.assertEquals(0, comp2); + } + + @Test + public void testGreaterThanValue() { + // given + byte[] val1 = Bytes.toBytes(new BigDecimal("1000000000000000000000000000000.9999999999999999")); + byte[] val2 = Bytes.toBytes(new BigDecimal(0)); + byte[] val3 = Bytes.toBytes(new BigDecimal(Double.MIN_VALUE)); + BigDecimal bd = new BigDecimal(Double.MAX_VALUE); + BigDecimalComparator comparator = new BigDecimalComparator(bd); + + // when + int comp1 = comparator.compareTo(val1); + int comp2 = comparator.compareTo(val2); + int comp3 = comparator.compareTo(val3); + + // then + Assert.assertEquals(1, comp1); + Assert.assertEquals(1, comp2); + Assert.assertEquals(1, comp3); + } + + @Test + public void testLessThanValue() { + // given + byte[] val1 = Bytes.toBytes(new BigDecimal("-1000000000000000000000000000000")); + byte[] val2 = Bytes.toBytes(new BigDecimal(0)); + byte[] val3 = Bytes.toBytes(new BigDecimal(1)); + BigDecimal bd = new BigDecimal("-1000000000000000000000000000000.0000000000000001"); + BigDecimalComparator comparator = new BigDecimalComparator(bd); + + // when + int comp1 = comparator.compareTo(val1); + int comp2 = comparator.compareTo(val2); + int comp3 = comparator.compareTo(val3); + + // then + Assert.assertEquals(-1, comp1); + Assert.assertEquals(-1, comp2); + Assert.assertEquals(-1, comp3); + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestComparatorSerialization.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestComparatorSerialization.java index 2befef0..fef699b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestComparatorSerialization.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestComparatorSerialization.java @@ -22,6 +22,7 @@ package org.apache.hadoop.hbase.filter; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.math.BigDecimal; import java.util.regex.Pattern; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; @@ -86,4 +87,12 @@ public class TestComparatorSerialization { ProtobufUtil.toComparator(ProtobufUtil.toComparator(substringComparator)))); } + @Test + public void testBigDecimalComparator() throws Exception { + BigDecimal bigDecimal = new BigDecimal(Double.MIN_VALUE); + BigDecimalComparator bigDecimalComparator = new BigDecimalComparator(bigDecimal); + assertTrue(bigDecimalComparator.areSerializedFieldsEqual(ProtobufUtil.toComparator(ProtobufUtil + .toComparator(bigDecimalComparator)))); + } + } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 24e42bb..4d7f97b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -42,6 +42,7 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.io.InterruptedIOException; +import java.math.BigDecimal; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; @@ -112,6 +113,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException; +import org.apache.hadoop.hbase.filter.BigDecimalComparator; import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; @@ -1779,6 +1781,8 @@ public class TestHRegion { byte[] qf1 = Bytes.toBytes("qualifier"); byte[] val1 = Bytes.toBytes("value1"); byte[] val2 = Bytes.toBytes("value2"); + BigDecimal bd1 = new BigDecimal(Double.MAX_VALUE); + BigDecimal bd2 = new BigDecimal(Double.MIN_VALUE); // Setting up region this.region = initHRegion(tableName, method, CONF, fam1); @@ -1799,6 +1803,25 @@ public class TestHRegion { res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val2), put, true); assertEquals(false, res); + + // Putting data in key + put = new Put(row1); + put.addColumn(fam1, qf1, Bytes.toBytes(bd1)); + region.put(put); + + // checkAndPut with wrong value + res = + region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BigDecimalComparator( + bd2), put, true); + assertEquals(false, res); + + // checkAndDelete with wrong value + delete = new Delete(row1); + delete.addFamily(fam1); + res = + region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BigDecimalComparator( + bd2), put, true); + assertEquals(false, res); } finally { HBaseTestingUtility.closeRegionAndWAL(this.region); this.region = null; @@ -1811,6 +1834,7 @@ public class TestHRegion { byte[] fam1 = Bytes.toBytes("fam1"); byte[] qf1 = Bytes.toBytes("qualifier"); byte[] val1 = Bytes.toBytes("value1"); + BigDecimal bd1 = new BigDecimal(Double.MIN_VALUE); // Setting up region this.region = initHRegion(tableName, method, CONF, fam1); @@ -1831,6 +1855,25 @@ public class TestHRegion { res = region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BinaryComparator(val1), delete, true); assertEquals(true, res); + + // Putting data in key + put = new Put(row1); + put.addColumn(fam1, qf1, Bytes.toBytes(bd1)); + region.put(put); + + // checkAndPut with correct value + res = + region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BigDecimalComparator( + bd1), put, true); + assertEquals(true, res); + + // checkAndDelete with correct value + delete = new Delete(row1); + delete.addColumn(fam1, qf1); + res = + region.checkAndMutate(row1, fam1, qf1, CompareOperator.EQUAL, new BigDecimalComparator( + bd1), delete, true); + assertEquals(true, res); } finally { HBaseTestingUtility.closeRegionAndWAL(this.region); this.region = null; -- 1.9.1