diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java index c2bf6d7..a8215f2 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java @@ -128,8 +128,25 @@ public int scale() { return bd.scale(); } + /** + * Returns the number of digits (integer and fractional) in the number, which is equivalent + * to SQL decimal precision. Note that this is different from BigDecimal.precision(), + * which returns the precision of the unscaled value (BigDecimal.valueOf(0.01).precision() = 1, + * whereas HiveDecimal.create("0.01").precision() = 2). + * If you want the BigDecimal precision, use HiveDecimal.bigDecimalValue().precision() + * @return + */ public int precision() { - return bd.precision(); + int bdPrecision = bd.precision(); + int bdScale = bd.scale(); + + if (bdPrecision < bdScale) { + // This can happen for numbers less than 0.1 + // For 0.001234: bdPrecision=4, bdScale=6 + // In this case, we'll set the type to have the same precision as the scale. + return bdScale; + } + return bdPrecision; } public int intValue() { diff --git a/ql/src/test/queries/clientpositive/decimal_precision2.q b/ql/src/test/queries/clientpositive/decimal_precision2.q new file mode 100644 index 0000000..b5f6f6b --- /dev/null +++ b/ql/src/test/queries/clientpositive/decimal_precision2.q @@ -0,0 +1,15 @@ + +explain select 100.001BD; + +explain select 100.000BD; + +explain select 0.000BD; + +explain select 0.100BD; + +explain select 0.010BD; + +explain select cast(0.010 as decimal(6,3)); + +explain select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD; +select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD; diff --git a/ql/src/test/results/clientpositive/decimal_precision2.q.out b/ql/src/test/results/clientpositive/decimal_precision2.q.out new file mode 100644 index 0000000..16765db --- /dev/null +++ b/ql/src/test/results/clientpositive/decimal_precision2.q.out @@ -0,0 +1,163 @@ +PREHOOK: query: explain select 100.001BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 100.001BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 100.001 (type: decimal(6,3)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 100.000BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 100.000BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 100.000 (type: decimal(3,0)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 0.000BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 0.000BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 0.000 (type: decimal(1,0)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 0.100BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 0.100BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 0.100 (type: decimal(1,1)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 0.010BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 0.010BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 0.010 (type: decimal(2,2)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select cast(0.010 as decimal(6,3)) +PREHOOK: type: QUERY +POSTHOOK: query: explain select cast(0.010 as decimal(6,3)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 0.01 (type: decimal(6,3)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 69.0212249755859375 (type: decimal(27,20)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +69.0212249755859375 diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java index a6ab7a7..3df3eb0 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java @@ -813,7 +813,9 @@ static void serialize(ByteStream.Output buffer, Object o, ObjectInspector oi, dec = dec.abs(); // get the scale factor to turn big decimal into a decimal < 1 - int factor = dec.precision() - dec.scale(); + // This relies on the BigDecimal precision value, which as of HIVE-10270 + // is now different from HiveDecimal.precision() + int factor = dec.bigDecimalValue().precision() - dec.bigDecimalValue().scale(); factor = sign == 1 ? factor : -factor; // convert the absolute big decimal to string diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java index 370e857..6bebabc 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java @@ -96,4 +96,97 @@ public int randomFill(Random r, ExtraTypeInfo extraTypeInfo) { return result; } + public void nonRandomFill(int idx) { + myByte = (Byte) getNonRandValue(nrByte, idx); + myShort = (Short) getNonRandValue(nrShort, idx); + myInt = (Integer) getNonRandValue(nrInt, idx); + myLong = (Long) getNonRandValue(nrLong, idx); + myFloat = (Float) getNonRandValue(nrFloat, idx); + myDouble = (Double) getNonRandValue(nrDouble, idx); + myString = (String) getNonRandValue(nrString, idx); + myDecimal = (HiveDecimal) getNonRandValue(nrDecimal, idx); + myDate = (Date) getNonRandValue(nrDate, idx); + myStruct = null; + myList = null; + } + + public static Object getNonRandValue(Object[] nrArray, int index) { + return nrArray[index % nrArray.length]; + } + + static Object[] nrByte = { + Byte.valueOf((byte) 1) + }; + + static Object[] nrShort = { + Short.valueOf((short) 1) + }; + + static Object[] nrInt = { + Integer.valueOf(1) + }; + + static Object[] nrLong = { + Long.valueOf(1) + }; + + static Object[] nrFloat = { + Float.valueOf(1.0f) + }; + + static Object[] nrDouble = { + Double.valueOf(1.0) + }; + + static Object[] nrDecimal = { + HiveDecimal.create("100"), + HiveDecimal.create("10"), + HiveDecimal.create("1"), + HiveDecimal.create("0"), + HiveDecimal.create("0.1"), + HiveDecimal.create("0.01"), + HiveDecimal.create("0.001"), + HiveDecimal.create("-100"), + HiveDecimal.create("-10"), + HiveDecimal.create("-1"), + HiveDecimal.create("-0.1"), + HiveDecimal.create("-0.01"), + HiveDecimal.create("-0.001"), + HiveDecimal.create("12345678900"), + HiveDecimal.create("1234567890"), + HiveDecimal.create("123456789"), + HiveDecimal.create("12345678.9"), + HiveDecimal.create("1234567.89"), + HiveDecimal.create("123456.789"), + HiveDecimal.create("12345.6789"), + HiveDecimal.create("1234.56789"), + HiveDecimal.create("123.456789"), + HiveDecimal.create("1.23456789"), + HiveDecimal.create("0.123456789"), + HiveDecimal.create("0.0123456789"), + HiveDecimal.create("0.00123456789"), + HiveDecimal.create("0.000123456789"), + HiveDecimal.create("-12345678900"), + HiveDecimal.create("-1234567890"), + HiveDecimal.create("-123456789"), + HiveDecimal.create("-12345678.9"), + HiveDecimal.create("-1234567.89"), + HiveDecimal.create("-123456.789"), + HiveDecimal.create("-12345.6789"), + HiveDecimal.create("-1234.56789"), + HiveDecimal.create("-123.456789"), + HiveDecimal.create("-1.23456789"), + HiveDecimal.create("-0.123456789"), + HiveDecimal.create("-0.0123456789"), + HiveDecimal.create("-0.00123456789"), + HiveDecimal.create("-0.000123456789"), + }; + + static Object[] nrString = { + "abcdefg" + }; + + static Object[] nrDate = { + Date.valueOf("2001-01-01") + }; } diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java index b3fb3be..af47e6f 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java @@ -147,7 +147,15 @@ public void testBinarySortableSerDe() throws Throwable { Random r = new Random(1234); MyTestClass rows[] = new MyTestClass[num]; - for (int i = 0; i < num; i++) { + int i; + // First try non-random values + for (i = 0; i < MyTestClass.nrDecimal.length; i++) { + MyTestClass t = new MyTestClass(); + t.nonRandomFill(i); + rows[i] = t; + } + + for ( ; i < num; i++) { MyTestClass t = new MyTestClass(); ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo(); t.randomFill(r, extraTypeInfo);