diff --git storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java index 7fa9fdf..4484ed2 100644 --- storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java +++ storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java @@ -273,6 +273,24 @@ protected boolean fastSerializationUtilsWrite(OutputStream outputStream, scratchLongs); } + /* + * Deserializes 64-bit decimals up to the maximum 64-bit precision (18 decimal digits). + */ + protected void fastDeserialize64(long decimalLong, int scale) { + FastHiveDecimalImpl.fastDeserialize64( + decimalLong, scale, this); + } + + /* + * Serializes decimal64 up to the maximum 64-bit precision (18 decimal digits). + */ + protected long fastSerialize64(int scale) { + return + FastHiveDecimalImpl.fastSerialize64( + scale, + fastSignum, fast1, fast0, fastScale); + } + // The fastBigIntegerBytes method returns 3 56 bit (7 byte) words and a possible sign byte. // However, the fastBigIntegerBytes can take on trailing zeroes -- so make it larger. protected static final int FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES = 1 + 48; diff --git storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java index ef9cbcf..84ff1c6 100644 --- storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java +++ storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java @@ -136,7 +136,9 @@ 10000000000000L, 100000000000000L, 1000000000000000L, - 10000000000000000L // 16 + 10000000000000000L, // 16 + 100000000000000000L, + 1000000000000000000L, // 18 }; public static final int MAX_DECIMAL_DIGITS = 38; @@ -154,6 +156,9 @@ private static final long MAX_LONGWORD_DECIMAL = powerOfTenTable[LONGWORD_DECIMAL_DIGITS] - 1; private static final long MULTIPLER_LONGWORD_DECIMAL = powerOfTenTable[LONGWORD_DECIMAL_DIGITS]; + public static final int DECIMAL64_DECIMAL_DIGITS = 18; + public static final long MAX_ABS_DECIMAL64 = 999999999999999999L; // 18 9's -- quite reliable! + private static final int TWO_X_LONGWORD_DECIMAL_DIGITS = 2 * LONGWORD_DECIMAL_DIGITS; private static final int THREE_X_LONGWORD_DECIMAL_DIGITS = 3 * LONGWORD_DECIMAL_DIGITS; private static final int FOUR_X_LONGWORD_DECIMAL_DIGITS = 4 * LONGWORD_DECIMAL_DIGITS; @@ -2138,6 +2143,70 @@ public static boolean fastSerializationUtilsWrite(OutputStream outputStream, throw new RuntimeException("Unexpected"); } + public static long getDecimal64AbsMax(int precision) { + return powerOfTenTable[precision] - 1; + } + + /* + * Deserializes 64-bit decimals up to the maximum 64-bit precision (18 decimal digits). + * + * NOTE: Major assumption: the input decimal64 has already been bounds checked and a least + * has a precision <= DECIMAL64_DECIMAL_DIGITS. We do not bounds check here for better + * performance. + */ + public static void fastDeserialize64( + final long inputDecimal64Long, final int inputScale, + FastHiveDecimal fastResult) { + + long decimal64Long; + if (inputDecimal64Long == 0) { + fastResult.fastReset(); + return; + } else if (inputDecimal64Long > 0) { + fastResult.fastSignum = 1; + decimal64Long = inputDecimal64Long; + } else { + fastResult.fastSignum = -1; + decimal64Long = -inputDecimal64Long; + } + + // Trim trailing zeroes -- but only below the decimal point. + int trimScale = inputScale; + while (trimScale > 0 && decimal64Long % 10 == 0) { + decimal64Long /= 10; + trimScale--; + } + + fastResult.fast2 = 0; + fastResult.fast1 = decimal64Long / MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast0 = decimal64Long % MULTIPLER_LONGWORD_DECIMAL; + + fastResult.fastScale = trimScale; + + fastResult.fastIntegerDigitCount = + Math.max(0, fastRawPrecision(fastResult) - fastResult.fastScale); + } + + /* + * Serializes decimal64 up to the maximum 64-bit precision (18 decimal digits). + * + * NOTE: Major assumption: the fast decimal has already been bounds checked and a least + * has a precision <= DECIMAL64_DECIMAL_DIGITS. We do not bounds check here for better + * performance. + */ + public static long fastSerialize64( + int scale, + int fastSignum, long fast1, long fast0, int fastScale) { + + if (fastSignum == 0) { + return 0; + } else if (fastSignum == 1) { + return (fast1 * MULTIPLER_LONGWORD_DECIMAL + fast0) * powerOfTenTable[scale - fastScale]; + } else { + return -(fast1 * MULTIPLER_LONGWORD_DECIMAL + fast0) * powerOfTenTable[scale - fastScale]; + } + } + //************************************************************************************************ // Emulate BigInteger deserialization used by LazyBinary and others. diff --git storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java index ffbe31a..617fb99 100644 --- storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java +++ storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java @@ -488,6 +488,64 @@ public boolean serializationUtilsWrite( scratchLongs); } + /* + * Maximum number of decimal digits in a decimal64 long. + */ + @HiveDecimalWritableVersionV2 + public static final int DECIMAL64_DECIMAL_DIGITS = FastHiveDecimalImpl.DECIMAL64_DECIMAL_DIGITS; + + /* + * Test whether a precision will fit within a decimal64 (64-bit signed long with <= 18 decimal + * digits). + */ + @HiveDecimalWritableVersionV2 + public static boolean isPrecisionDecimal64(int precision) { + return (precision <= DECIMAL64_DECIMAL_DIGITS); + } + + /* + * Return the maximum absolute decimal64 value for a precision. + */ + @HiveDecimalWritableVersionV2 + public static long getDecimal64AbsMax(int precision) { + return FastHiveDecimalImpl.getDecimal64AbsMax(precision); + } + + /* + * Deserializes 64-bit decimals up to the maximum 64-bit precision (18 decimal digits). + * + * NOTE: Major assumption: the input decimal64 has already been bounds checked and a least + * has a precision <= DECIMAL64_DECIMAL_DIGITS. We do not bounds check here for better + * performance. You can bounds check beforehand with: + * Math.abs(decimal64Long) <= getDecimal64AbsMax(precision) + */ + @HiveDecimalWritableVersionV2 + public void deserialize64( + long decimal64Long, int scale) { + fastDeserialize64(decimal64Long, scale); + isSet = true; + } + + /* + * Serializes decimal64 up to the maximum 64-bit precision (18 decimal digits). + * + * NOTE: Major assumption: the fast decimal has already been bounds checked and a least + * has a precision <= DECIMAL64_DECIMAL_DIGITS. We do not bounds check here for better + * performance. + */ + @HiveDecimalWritableVersionV2 + public long serialize64(int scale) { + return fastSerialize64(scale); + } + + @HiveDecimalWritableVersionV2 + public boolean isValid() { + if (!isSet) { + return false; + } + return FastHiveDecimalImpl.fastIsValid(this); + } + /** * Returns the length of the decimal converted to bytes. * Call bigIntegerBytesBuffer() to get a reference to the converted bytes. diff --git storage-api/src/test/org/apache/hadoop/hive/common/type/HiveDecimalTestBase.java storage-api/src/test/org/apache/hadoop/hive/common/type/HiveDecimalTestBase.java index 553c456..9550b94 100644 --- storage-api/src/test/org/apache/hadoop/hive/common/type/HiveDecimalTestBase.java +++ storage-api/src/test/org/apache/hadoop/hive/common/type/HiveDecimalTestBase.java @@ -305,8 +305,14 @@ public boolean isTenPowerBug(String string) { "-9999999999999999", "10000000000000000", // 10^16 "-10000000000000000", + "99999999999999999", // 10^17 - 1 + "-99999999999999999", "100000000000000000", "-100000000000000000", + "999999999999999999", // 10^18 - 1 + "-999999999999999999", + "123456789012345678", + "-123456789012345678", "1000000000000000000", "-1000000000000000000", "9223372036854775807", // Long.MAX_VALUE @@ -369,6 +375,10 @@ public boolean isTenPowerBug(String string) { "-0.9999999999999999", "0.00000000000000001", // 10^-16 "-0.00000000000000001", + "0.99999999999999999", + "-0.99999999999999999", + "0.999999999999999999", // 10^-18 + "-0.999999999999999999", "0.00000000000000000000000000000001", // 10^-31 "-0.00000000000000000000000000000001", "0.99999999999999999999999999999999", // 10^-32 + 1 diff --git storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java index f8a36e5..d11f41c 100644 --- storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java +++ storage-api/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java @@ -3394,6 +3394,79 @@ private void doTestMultiply(Random r, BigDecimal bigDecimal, BigDecimal bigDecim } } + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomDecimal64() { + Random r = new Random(2497); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomDecimal64(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomDecimal64(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomDecimal64(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestDecimal64(r, bigDecimal); + } + } + + @Test + public void testDecimal64Special() { + Random r = new Random(198); + for (BigDecimal bigDecimal : specialBigDecimals) { + int precision = Math.min(bigDecimal.precision(), HiveDecimalWritable.DECIMAL64_DECIMAL_DIGITS); + int scale = Math.min(bigDecimal.scale(), precision); + doTestDecimal64(r, bigDecimal, precision, scale); + } + } + + private void doTestDecimal64(Random r, BigDecimal inputBigDecimal) { + final int precision = 1 + r.nextInt(HiveDecimalWritable.DECIMAL64_DECIMAL_DIGITS); + assertTrue(HiveDecimalWritable.isPrecisionDecimal64(precision)); + final int scale = r.nextInt(precision + 1); + + doTestDecimal64(r, inputBigDecimal, precision, scale); + } + + private void doTestDecimal64(Random r, BigDecimal inputBigDecimal, int precision, int scale) { + + BigDecimal bigDecimal = inputBigDecimal; + + if (!bigDecimal.equals(BigDecimal.ZERO)) { + while (true) { + bigDecimal = bigDecimal.remainder(BigDecimal.valueOf(10).pow(precision - scale)); + bigDecimal = bigDecimal.setScale(scale, BigDecimal.ROUND_DOWN); + if (!bigDecimal.unscaledValue().equals(BigInteger.ZERO)) { + break; + } + bigDecimal = randHiveBigDecimalNormalRange(r, standardAlphabet); + } + } + + HiveDecimal dec = HiveDecimal.create(bigDecimal); + assertTrue(dec != null); + dec.validate(); + + HiveDecimalWritable decWritable = new HiveDecimalWritable(dec); + + final long decimal64Long = decWritable.serialize64(scale); + assertTrue(decimal64Long <= HiveDecimalWritable.getDecimal64AbsMax(precision)); + HiveDecimalWritable resultWritable = new HiveDecimalWritable(0); + resultWritable.deserialize64(decimal64Long, scale); + + assertEquals(dec, resultWritable.getHiveDecimal()); + } + + //------------------------------------------------------------------------------------------------ + public static String displayBytes(byte[] bytes, int start, int length) { StringBuilder sb = new StringBuilder(); for (int i = start; i < start + length; i++) {