diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java index c2bf6d7..a8215f2 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java @@ -128,8 +128,25 @@ public int scale() { return bd.scale(); } + /** + * Returns the number of digits (integer and fractional) in the number, which is equivalent + * to SQL decimal precision. Note that this is different from BigDecimal.precision(), + * which returns the precision of the unscaled value (BigDecimal.valueOf(0.01).precision() = 1, + * whereas HiveDecimal.create("0.01").precision() = 2). + * If you want the BigDecimal precision, use HiveDecimal.bigDecimalValue().precision() + * @return + */ public int precision() { - return bd.precision(); + int bdPrecision = bd.precision(); + int bdScale = bd.scale(); + + if (bdPrecision < bdScale) { + // This can happen for numbers less than 0.1 + // For 0.001234: bdPrecision=4, bdScale=6 + // In this case, we'll set the type to have the same precision as the scale. + return bdScale; + } + return bdPrecision; } public int intValue() { diff --git a/ql/src/test/queries/clientpositive/decimal_precision2.q b/ql/src/test/queries/clientpositive/decimal_precision2.q new file mode 100644 index 0000000..b5f6f6b --- /dev/null +++ b/ql/src/test/queries/clientpositive/decimal_precision2.q @@ -0,0 +1,15 @@ + +explain select 100.001BD; + +explain select 100.000BD; + +explain select 0.000BD; + +explain select 0.100BD; + +explain select 0.010BD; + +explain select cast(0.010 as decimal(6,3)); + +explain select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD; +select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD; diff --git a/ql/src/test/results/clientpositive/decimal_precision2.q.out b/ql/src/test/results/clientpositive/decimal_precision2.q.out new file mode 100644 index 0000000..16765db --- /dev/null +++ b/ql/src/test/results/clientpositive/decimal_precision2.q.out @@ -0,0 +1,163 @@ +PREHOOK: query: explain select 100.001BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 100.001BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 100.001 (type: decimal(6,3)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 100.000BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 100.000BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 100.000 (type: decimal(3,0)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 0.000BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 0.000BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 0.000 (type: decimal(1,0)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 0.100BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 0.100BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 0.100 (type: decimal(1,1)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 0.010BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 0.010BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 0.010 (type: decimal(2,2)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select cast(0.010 as decimal(6,3)) +PREHOOK: type: QUERY +POSTHOOK: query: explain select cast(0.010 as decimal(6,3)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 0.01 (type: decimal(6,3)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 69.0212249755859375 (type: decimal(27,20)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +69.0212249755859375 diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java index ccdb820..37c23f9 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java @@ -813,7 +813,9 @@ static void serialize(ByteStream.Output buffer, Object o, ObjectInspector oi, dec = dec.abs(); // get the scale factor to turn big decimal into a decimal < 1 - int factor = dec.precision() - dec.scale(); + // This relies on the BigDecimal precision value, which as of HIVE-10270 + // is now different from HiveDecimal.precision() + int factor = dec.bigDecimalValue().precision() - dec.bigDecimalValue().scale(); factor = sign == 1 ? factor : -factor; // convert the absolute big decimal to string diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java index cefb72e..883a162 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java @@ -196,16 +196,114 @@ public static String getRandString(Random r, String characters, int length) { return bytes; } + public static Object getNonRandValue(Object[] nrArray, int index) { + return nrArray[index % nrArray.length]; + } + + Object[] nrByte = { + Byte.valueOf((byte) 1) + }; + + Object[] nrShort = { + Short.valueOf((short) 1) + }; + + Object[] nrInt = { + Integer.valueOf(1) + }; + + Object[] nrLong = { + Long.valueOf(1) + }; + + Object[] nrFloat = { + Float.valueOf(1.0f) + }; + + Object[] nrDouble = { + Double.valueOf(1.0) + }; + + Object[] nrDecimal = { + HiveDecimal.create("100"), + HiveDecimal.create("10"), + HiveDecimal.create("1"), + HiveDecimal.create("0"), + HiveDecimal.create("0.1"), + HiveDecimal.create("0.01"), + HiveDecimal.create("0.001"), + HiveDecimal.create("-100"), + HiveDecimal.create("-10"), + HiveDecimal.create("-1"), + HiveDecimal.create("-0.1"), + HiveDecimal.create("-0.01"), + HiveDecimal.create("-0.001"), + HiveDecimal.create("12345678900"), + HiveDecimal.create("1234567890"), + HiveDecimal.create("123456789"), + HiveDecimal.create("12345678.9"), + HiveDecimal.create("1234567.89"), + HiveDecimal.create("123456.789"), + HiveDecimal.create("12345.6789"), + HiveDecimal.create("1234.56789"), + HiveDecimal.create("123.456789"), + HiveDecimal.create("1.23456789"), + HiveDecimal.create("0.123456789"), + HiveDecimal.create("0.0123456789"), + HiveDecimal.create("0.00123456789"), + HiveDecimal.create("0.000123456789"), + HiveDecimal.create("-12345678900"), + HiveDecimal.create("-1234567890"), + HiveDecimal.create("-123456789"), + HiveDecimal.create("-12345678.9"), + HiveDecimal.create("-1234567.89"), + HiveDecimal.create("-123456.789"), + HiveDecimal.create("-12345.6789"), + HiveDecimal.create("-1234.56789"), + HiveDecimal.create("-123.456789"), + HiveDecimal.create("-1.23456789"), + HiveDecimal.create("-0.123456789"), + HiveDecimal.create("-0.0123456789"), + HiveDecimal.create("-0.00123456789"), + HiveDecimal.create("-0.000123456789"), + }; + + Object[] nrString = { + "abcdefg" + }; + + Object[] nrDate = { + Date.valueOf("2001-01-01") + }; + public void testBinarySortableSerDe() throws Throwable { try { System.out.println("Beginning Test testBinarySortableSerDe:"); - int num = 1000; + int num = 1000 + nrDecimal.length; Random r = new Random(1234); MyTestClass rows[] = new MyTestClass[num]; + int i; + // First try non-random values + for (i = 0; i < nrDecimal.length; i++) { + MyTestClass t = new MyTestClass(); + t.myByte = (Byte) getNonRandValue(nrByte, i); + t.myShort = (Short) getNonRandValue(nrShort, i); + t.myInt = (Integer) getNonRandValue(nrInt, i); + t.myLong = (Long) getNonRandValue(nrLong, i); + t.myFloat = (Float) getNonRandValue(nrFloat, i); + t.myDouble = (Double) getNonRandValue(nrDouble, i); + t.myString = (String) getNonRandValue(nrString, i); + t.myDecimal = (HiveDecimal) getNonRandValue(nrDecimal, i); + t.myDate = (Date) getNonRandValue(nrDate, i); + t.myStruct = null; + t.myList = null; + t.myBA = null; + rows[i] = t; + } - for (int i = 0; i < num; i++) { + for (; i < num; i++) { int randField = r.nextInt(11); MyTestClass t = new MyTestClass(); t.myByte = randField > 0 ? null : Byte.valueOf((byte) r.nextInt());