From 9e8e6419ee0483ef5ffc9d66c15414ae0453cbce Mon Sep 17 00:00:00 2001 From: Zhong Date: Sun, 6 May 2018 16:43:41 +0800 Subject: [PATCH] KYLIN-3364 make it consistent with hive for BigDecimalSumAggregator dealing with null --- .../apache/kylin/measure/basic/BigDecimalSumAggregator.java | 9 +++++++-- .../apache/kylin/metadata/datatype/BigDecimalSerializer.java | 10 ++++++++++ .../kylin/metadata/datatype/BigDecimalSerializerTest.java | 10 ++++++++++ .../java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java | 9 +++++---- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java index 9f6ffc2..fa59c54 100644 --- a/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java +++ b/core-metadata/src/main/java/org/apache/kylin/measure/basic/BigDecimalSumAggregator.java @@ -27,15 +27,20 @@ import org.apache.kylin.measure.MeasureAggregator; @SuppressWarnings("serial") public class BigDecimalSumAggregator extends MeasureAggregator { - BigDecimal sum = new BigDecimal(0); + BigDecimal sum = null; @Override public void reset() { - sum = new BigDecimal(0); + sum = null; } @Override public void aggregate(BigDecimal value) { + if (value == null) + return; + if (sum == null) { + sum = new BigDecimal(0); + } sum = sum.add(value); } diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java index ba1c4ff..967c00d 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/BigDecimalSerializer.java @@ -46,6 +46,12 @@ public class BigDecimalSerializer extends DataTypeSerializer { @Override public void serialize(BigDecimal value, ByteBuffer out) { + if (value == null) { + BytesUtil.writeVInt(0, out); + BytesUtil.writeVInt(-1, out); + return; + } + if (value.scale() > type.getScale()) { if (avoidVerbose++ % 10000 == 0) { logger.warn("value's scale has exceeded the " + type.getScale() + ", cut it off, to ensure encoded value do not exceed maxLength " + maxLength + " times:" + (avoidVerbose)); @@ -67,6 +73,10 @@ public class BigDecimalSerializer extends DataTypeSerializer { int scale = BytesUtil.readVInt(in); int n = BytesUtil.readVInt(in); + if (n < 0) { + return null; + } + byte[] bytes = new byte[n]; in.get(bytes); diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java index 5be5806..a21b08e 100644 --- a/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java +++ b/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/BigDecimalSerializerTest.java @@ -74,4 +74,14 @@ public class BigDecimalSerializerTest extends LocalFileMetadataTestCase { bigDecimalSerializer.serialize(input, buffer); } + @Test + public void testNull() { + BigDecimal input = null; + ByteBuffer buffer = ByteBuffer.allocate(256); + buffer.mark(); + bigDecimalSerializer.serialize(input, buffer); + buffer.reset(); + BigDecimal output = bigDecimalSerializer.deserialize(buffer); + assertEquals(input, output); + } } diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java index 7616df2..f90a644 100644 --- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java +++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/CubeReducerTest.java @@ -148,9 +148,9 @@ public class CubeReducerTest extends LocalFileMetadataTestCase { List> result = reduceDriver.run(); - Pair p1 = new Pair(new Text("72010ustech"), newValueText(codec, "0", "10", "20.34", 3, 600)); - Pair p2 = new Pair(new Text("1tech"), newValueText(codec, "0", "15.09", "20.34", 2, 1500)); - Pair p3 = new Pair(new Text("0"), newValueText(codec, "0", "146.52", "146.52", 0, 0)); + Pair p1 = new Pair<>(new Text("72010ustech"), newValueText(codec, null, "10", "20.34", 3, 600)); + Pair p2 = new Pair<>(new Text("1tech"), newValueText(codec, null, "15.09", "20.34", 2, 1500)); + Pair p3 = new Pair<>(new Text("0"), newValueText(codec, null, "146.52", "146.52", 0, 0)); assertEquals(3, result.size()); @@ -160,7 +160,8 @@ public class CubeReducerTest extends LocalFileMetadataTestCase { } private Text newValueText(BufferedMeasureCodec codec, String sum, String min, String max, int count, int item_count) { - Object[] values = new Object[] { new BigDecimal(sum), new BigDecimal(min), new BigDecimal(max), new Long(count), new Long(item_count) }; + Object[] values = new Object[] { sum == null ? null : new BigDecimal(sum), // + new BigDecimal(min), new BigDecimal(max), new Long(count), new Long(item_count) }; ByteBuffer buf = codec.encode(values); -- 2.5.4 (Apple Git-61)