diff --git common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java index f68842c..ddf8b88 100644 --- common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java +++ common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java @@ -17,80 +17,354 @@ */ package org.apache.hadoop.hive.common.type; +import java.sql.Timestamp; +import java.util.Random; +import java.util.Arrays; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; +import org.apache.hadoop.hive.serde2.io.OldHiveDecimalWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.orc.impl.SerializationUtils; +import org.apache.hadoop.hive.common.type.RandomTypeUtil; +import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; +import org.apache.hadoop.hive.ql.util.TimestampUtils; + import com.google.code.tempusfugit.concurrency.annotations.*; import com.google.code.tempusfugit.concurrency.*; + import org.junit.*; + import static org.junit.Assert.*; public class TestHiveDecimal { + // HiveDecimal is a CPU beast with no threading concerns. The test methods can safely be + // run concurrently. @Rule public ConcurrentRule concurrentRule = new ConcurrentRule(); - @Rule public RepeatingRule repeatingRule = new RepeatingRule(); @Test @Concurrent(count=4) - @Repeating(repetition=100) - public void testPrecisionScaleEnforcement() { - String decStr = "1786135888657847525803324040144343378.09799306448796128931113691624"; - HiveDecimal dec = HiveDecimal.create(decStr); + public void testVariousCases() { + + OldHiveDecimal oldDec; + OldHiveDecimal resultOldDec; + HiveDecimal dec; + HiveDecimal resultDec; + + BigDecimal bigDecimal = new BigDecimal("-99999999999999999999999999999999999999.99999999999999999"); + + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(bigDecimal); + Assert.assertEquals("-100000000000000000000000000000000000000", oldDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(bigDecimal); + Assert.assertTrue(dec == null); + + // One less integer digit... + bigDecimal = new BigDecimal("-9999999999999999999999999999999999999.99999999999999999"); + + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(bigDecimal); + Assert.assertEquals("-10000000000000000000000000000000000000", oldDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(bigDecimal); + Assert.assertEquals("-10000000000000000000000000000000000000", dec.toString()); + + /* + //--------------------------------------------------- + oldDec = OldHiveDecimal.create("101"); + resultOldDec = OldHiveDecimal.enforcePrecisionScale(oldDec, 10, 0); + Assert.assertEquals("101", resultOldDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create("101"); + resultDec = HiveDecimal.enforcePrecisionScale(dec, 10, 0); + Assert.assertEquals("101", resultDec.toString()); + + //--------------------------------------------------- + oldDec = OldHiveDecimal.create("1"); + resultOldDec = oldDec.scaleByPowerOfTen(-99); + Assert.assertEquals("0", resultOldDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create("1"); + resultDec = dec.scaleByPowerOfTen(-99); + Assert.assertEquals("0", resultDec.toString()); + */ + } + + @Test + @Concurrent(count=4) + public void testCreateFromBigIntegerRounding() { + + BigInteger bigInt; + OldHiveDecimal oldDec; + HiveDecimal dec; + + // 1786135888657847525803324040144343378.09799306448796128931113691624 + bigInt = new BigInteger( + "178613588865784752580332404014434337809799306448796128931113691624"); + Assert.assertEquals("178613588865784752580332404014434337809799306448796128931113691624", bigInt.toString()); + // 12345678901234567890123456789012345678 + // 1 2 3 + // 12345678901234567890123456789 + dec = HiveDecimal.create(bigInt, 29); Assert.assertEquals("1786135888657847525803324040144343378.1", dec.toString()); - Assert.assertTrue("Decimal precision should not go above maximum", - dec.precision() <= HiveDecimal.MAX_PRECISION); - Assert.assertTrue("Decimal scale should not go above maximum", dec.scale() <= HiveDecimal.MAX_SCALE); - - decStr = "57847525803324040144343378.09799306448796128931113691624"; - HiveDecimal bd = HiveDecimal.create(decStr); - HiveDecimal bd1 = HiveDecimal.enforcePrecisionScale(bd, 20, 5); - Assert.assertNull(bd1); - bd1 = HiveDecimal.enforcePrecisionScale(bd, 35, 5); - Assert.assertEquals("57847525803324040144343378.09799", bd1.toString()); - bd1 = HiveDecimal.enforcePrecisionScale(bd, 45, 20); - Assert.assertNull(bd1); - - dec = HiveDecimal.create(new BigDecimal(decStr), false); - Assert.assertNull(dec); - dec = HiveDecimal.create("-1786135888657847525803324040144343378.09799306448796128931113691624"); - Assert.assertEquals("-1786135888657847525803324040144343378.1", dec.toString()); + // 8.090000000000000000000000000000000000000123456 + bigInt = new BigInteger( + "8090000000000000000000000000000000000000123456"); + // 123456789012345678901234567890123456789012345 + // 1 2 3 4 + Assert.assertEquals("8090000000000000000000000000000000000000123456", bigInt.toString()); + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(bigInt, 45); + Assert.assertEquals("8.09", oldDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(bigInt, 45); + Assert.assertEquals("8.09", dec.toString()); + + // 99999999.99999999999999999999999999999949999 + // MAX_DECIMAL 9's WITH NO ROUND (longer than 38 digits) + bigInt = new BigInteger( + "9999999999999999999999999999999999999949999"); + // 12345678901234567890123456789012345678 + // 1 2 3 + // 99999999.99999999999999999999999999999949999 + Assert.assertEquals("9999999999999999999999999999999999999949999", bigInt.toString()); + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(bigInt, 35); + Assert.assertEquals("99999999.999999999999999999999999999999", oldDec.toString()); + //--------------------------------------------------- + // Without the round, this conversion fails. + dec = HiveDecimal.create(bigInt, 35); + Assert.assertEquals("99999999.999999999999999999999999999999", dec.toString()); + + // MAX_DECIMAL 9's WITH ROUND. + bigInt = new BigInteger( + "9999999999999999999999999999999999999979999"); + // 12346678.901234667890123466789012346678 + // 1 2 3 + Assert.assertEquals("9999999999999999999999999999999999999979999", bigInt.toString()); + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(bigInt, 35); + Assert.assertEquals("100000000", oldDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(bigInt, 35); + Assert.assertEquals("100000000", dec.toString()); + } + + @Test + @Concurrent(count=4) + public void testCreateFromBigDecimal() { + + BigDecimal bigDec; + OldHiveDecimal oldDec; + HiveDecimal dec; + + bigDec = new BigDecimal("0"); + Assert.assertEquals("0", bigDec.toString()); + dec = HiveDecimal.create(bigDec); + Assert.assertEquals("0", dec.toString()); + + bigDec = new BigDecimal("1"); + Assert.assertEquals("1", bigDec.toString()); + dec = HiveDecimal.create(bigDec); + Assert.assertEquals("1", dec.toString()); + + bigDec = new BigDecimal("0.999"); + Assert.assertEquals("0.999", bigDec.toString()); + dec = HiveDecimal.create(bigDec); + Assert.assertEquals("0.999", dec.toString()); + + // HiveDecimal suppresses trailing zeroes. + bigDec = new BigDecimal("0.9990"); + Assert.assertEquals("0.9990", bigDec.toString()); + dec = HiveDecimal.create(bigDec); + Assert.assertEquals("0.999", dec.toString()); + } + + @Test + @Concurrent(count=4) + public void testCreateFromBigDecimalRounding() { + + BigDecimal bigDec; + OldHiveDecimal oldDec; + HiveDecimal dec; + + bigDec = new BigDecimal( + "1786135888657847525803324040144343378.09799306448796128931113691624"); + Assert.assertEquals("1786135888657847525803324040144343378.09799306448796128931113691624", bigDec.toString()); + // 1234567890123456789012345678901234567.8 + // 1 2 3 + // Without the round, this conversion fails. + dec = HiveDecimal.create(bigDec, false); + Assert.assertTrue(dec == null); + dec = HiveDecimal.create(bigDec, true); + Assert.assertEquals("1786135888657847525803324040144343378.1", dec.toString()); + + bigDec = new BigDecimal( + "8.090000000000000000000000000000000000000123456"); + // 1.23456789012345678901234567890123456789012345 + // 1 2 3 4 + Assert.assertEquals("8.090000000000000000000000000000000000000123456", bigDec.toString()); + //--------------------------------------------------- + OldHiveDecimal oldDec4 = OldHiveDecimal.create(bigDec, false); + Assert.assertTrue(oldDec4 == null); + oldDec4 = OldHiveDecimal.create(bigDec, true); + Assert.assertEquals("8.09", oldDec4.toString()); + //--------------------------------------------------- + // Without the round, this conversion fails. + dec = HiveDecimal.create(bigDec, false); + Assert.assertTrue(dec == null); + dec = HiveDecimal.create(bigDec, true); + Assert.assertEquals("8.09", dec.toString()); + + // MAX_DECIMAL 9's WITH NO ROUND (longer than 38 digits) + bigDec = new BigDecimal( + "99999999.99999999999999999999999999999949999"); + // 12345678.901234567890123456789012345678 + // 1 2 3 + Assert.assertEquals("99999999.99999999999999999999999999999949999", bigDec.toString()); + //--------------------------------------------------- + OldHiveDecimal oldDec5 = OldHiveDecimal.create(bigDec, false); + Assert.assertTrue(oldDec5 == null); + oldDec5 = OldHiveDecimal.create(bigDec, true); + Assert.assertEquals("99999999.999999999999999999999999999999", oldDec5.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(bigDec, false); + Assert.assertTrue(dec == null); + dec = HiveDecimal.create(bigDec, true); + Assert.assertEquals("99999999.999999999999999999999999999999", dec.toString()); + + // MAX_DECIMAL 9's WITH ROUND. + bigDec = new BigDecimal( + "99999999.99999999999999999999999999999979999"); + // 12346678.901234667890123466789012346678 + // 1 2 3 + Assert.assertEquals("99999999.99999999999999999999999999999979999", bigDec.toString()); + //--------------------------------------------------- + OldHiveDecimal oldDec6 = OldHiveDecimal.create(bigDec, false); + Assert.assertTrue(oldDec6 == null); + oldDec6 = OldHiveDecimal.create(bigDec, true); + Assert.assertEquals("100000000", oldDec6.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(bigDec, false); + Assert.assertTrue(dec == null); + dec = HiveDecimal.create(bigDec, true); + Assert.assertEquals("100000000", dec.toString()); + } + + + @Test + @Concurrent(count=4) + public void testPrecisionScaleEnforcement() { + + OldHiveDecimal oldDec; + OldHiveDecimal oldResultDec; + + HiveDecimal dec; + HiveDecimal resultDec; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create("0.02538461538461538461538461538461538462"); + Assert.assertEquals("0.02538461538461538461538", + OldHiveDecimal.enforcePrecisionScale(oldDec, 38, 23).toString()); + //--------------------------------------------------- + dec = HiveDecimal.create("0.02538461538461538461538461538461538462"); + Assert.assertEquals("0.02538461538461538461538", + HiveDecimal.enforcePrecisionScale(dec, 38, 23).toString()); + + //--------------------------------------------------- + oldDec = OldHiveDecimal.create("005.34000"); + Assert.assertEquals(oldDec.precision(), 3); // 1 integer digit; 2 fraction digits. + Assert.assertEquals(oldDec.scale(), 2); // Trailing zeroes are suppressed. + //--------------------------------------------------- dec = HiveDecimal.create("005.34000"); - Assert.assertEquals(dec.precision(), 3); - Assert.assertEquals(dec.scale(), 2); + Assert.assertEquals(dec.sqlPrecision(), 3); // 1 integer digit; 2 fraction digits. + Assert.assertEquals(dec.scale(), 2); // Trailing zeroes are suppressed. dec = HiveDecimal.create("178613588865784752580332404014434337809799306448796128931113691624"); Assert.assertNull(dec); // Rounding numbers that increase int digits + //--------------------------------------------------- + oldDec = OldHiveDecimal.create("9.5"); + Assert.assertEquals("10", + OldHiveDecimal.enforcePrecisionScale(oldDec, 2, 0).toString()); + Assert.assertNull( + OldHiveDecimal.enforcePrecisionScale(oldDec, 1, 0)); + oldDec = OldHiveDecimal.create("9.4"); + Assert.assertEquals("9", + OldHiveDecimal.enforcePrecisionScale(oldDec, 1, 0).toString()); + //--------------------------------------------------- + dec = HiveDecimal.create("9.5"); Assert.assertEquals("10", - HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.5"), 2, 0).toString()); - Assert.assertNull(HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.5"), 1, 0)); + HiveDecimal.enforcePrecisionScale(dec, 2, 0).toString()); + Assert.assertNull( + HiveDecimal.enforcePrecisionScale(dec, 1, 0)); + dec = HiveDecimal.create("9.4"); Assert.assertEquals("9", - HiveDecimal.enforcePrecisionScale(HiveDecimal.create("9.4"), 1, 0).toString()); + HiveDecimal.enforcePrecisionScale(dec, 1, 0).toString()); + } + + @Test + @Concurrent(count=4) + public void testPrecisionScaleEnforcementEdgeCond() { + + // Since HiveDecimal now uses FastHiveDecimal which stores 16 decimal digits per long, + // lets test edge conditions here. + + HiveDecimal fifteenFractionalNinesDec = HiveDecimal.create("0.999999999999999"); + Assert.assertNotNull(fifteenFractionalNinesDec); + Assert.assertEquals("0.999999999999999", + HiveDecimal.enforcePrecisionScale(fifteenFractionalNinesDec, 15, 15).toString()); + + HiveDecimal sixteenFractionalNines = HiveDecimal.create("0.9999999999999999"); + Assert.assertNotNull(sixteenFractionalNines); + Assert.assertEquals("0.9999999999999999", + HiveDecimal.enforcePrecisionScale(sixteenFractionalNines, 16, 16).toString()); + + HiveDecimal seventeenFractionalNines = HiveDecimal.create("0.99999999999999999"); + Assert.assertNotNull(seventeenFractionalNines); + Assert.assertEquals("0.99999999999999999", + HiveDecimal.enforcePrecisionScale(seventeenFractionalNines, 17, 17).toString()); + } @Test @Concurrent(count=4) - @Repeating(repetition=100) public void testTrailingZeroRemovalAfterEnforcement() { String decStr = "8.090000000000000000000000000000000000000123456"; + // 123456789012345678901234567890123456789012345 + // 1 2 3 4 HiveDecimal dec = HiveDecimal.create(decStr); Assert.assertEquals("8.09", dec.toString()); } @Test @Concurrent(count=4) - @Repeating(repetition=100) public void testMultiply() { + + // This multiply produces more than 38 digits --> overflow. + //--------------------------------------------------- + OldHiveDecimal oldDec1 = OldHiveDecimal.create("0.00001786135888657847525803"); + OldHiveDecimal oldDec2 = OldHiveDecimal.create("3.0000123456789"); + OldHiveDecimal oldResult = oldDec1.multiply(oldDec2); + Assert.assertTrue(oldResult == null); + //--------------------------------------------------- HiveDecimal dec1 = HiveDecimal.create("0.00001786135888657847525803"); HiveDecimal dec2 = HiveDecimal.create("3.0000123456789"); - Assert.assertNull(dec1.multiply(dec2)); + HiveDecimal result = dec1.multiply(dec2); + Assert.assertTrue(result == null); dec1 = HiveDecimal.create("178613588865784752580323232232323444.4"); dec2 = HiveDecimal.create("178613588865784752580302323232.3"); - Assert.assertNull(dec1.multiply(dec2)); + Assert.assertNull(dec1.multiply(dec2)); // i.e. Overflow. dec1 = HiveDecimal.create("47.324"); dec2 = HiveDecimal.create("9232.309"); @@ -107,22 +381,282 @@ public void testMultiply() { @Test @Concurrent(count=4) - @Repeating(repetition=100) + public void testMultiply2() { + // 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD + HiveDecimal dec1 = HiveDecimal.create("0.09765625"); + HiveDecimal dec2 = HiveDecimal.create("0.09765625"); + HiveDecimal dec3 = HiveDecimal.create("0.0125"); + HiveDecimal dec4 = HiveDecimal.create("578992"); + HiveDecimal result1 = dec1.multiply(dec2); + Assert.assertNotNull(result1); + HiveDecimal result2 = result1.multiply(dec3); + Assert.assertNotNull(result2); + HiveDecimal result = result2.multiply(dec4); + Assert.assertNotNull(result); + Assert.assertEquals("69.0212249755859375", result.toString()); + } + + @Test + @Concurrent(count=4) public void testPow() { - HiveDecimal dec = HiveDecimal.create("3.00001415926"); - Assert.assertEquals(dec.pow(2), dec.multiply(dec)); - HiveDecimal dec1 = HiveDecimal.create("0.000017861358882"); - dec1 = dec1.pow(3); - Assert.assertNull(dec1); + HiveDecimal dec; - dec1 = HiveDecimal.create("3.140"); - Assert.assertEquals("9.8596", dec1.pow(2).toString()); + dec = HiveDecimal.create("3.00001415926"); + HiveDecimal decPow2 = dec.pow(2); + HiveDecimal decMultiplyTwice = dec.multiply(dec); + Assert.assertEquals(decPow2, decMultiplyTwice); + + dec = HiveDecimal.create("0.000017861358882"); + dec = dec.pow(3); + Assert.assertNull(dec); + + dec = HiveDecimal.create("3.140"); + Assert.assertEquals("9.8596", dec.pow(2).toString()); + } + + @Test + @Concurrent(count=4) + public void testScaleByPowerOfTen() { + + OldHiveDecimal oldDec; + HiveDecimal dec; + OldHiveDecimal oldResultDec; + HiveDecimal resultDec; + + //********************************************************************************************** + + //--------------------------------------------------- + oldDec = OldHiveDecimal.create( + "1"); + Assert.assertEquals(0, oldDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(2); + Assert.assertEquals( + "100", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create( + "1"); + Assert.assertEquals(0, dec.scale()); + // resultDec = dec.scaleByPowerOfTen(2); + // Assert.assertEquals( + // "100", resultDec.toString()); + + //--------------------------------------------------- + oldDec = OldHiveDecimal.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, oldDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(2); + Assert.assertEquals( + "0.000000000000000000000000000000000001", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, dec.scale()); + resultDec = dec.scaleByPowerOfTen(2); + Assert.assertEquals( + "0.000000000000000000000000000000000001", resultDec.toString()); + + //--------------------------------------------------- + oldDec = OldHiveDecimal.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, oldDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(38); + Assert.assertEquals( + "1", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, dec.scale()); + resultDec = dec.scaleByPowerOfTen(38); + Assert.assertEquals( + "1", resultDec.toString()); + + //--------------------------------------------------- + oldDec = OldHiveDecimal.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, oldDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(2 * 38 - 1); + Assert.assertEquals( + "10000000000000000000000000000000000000", oldResultDec.toString()); + Assert.assertEquals(0, oldResultDec.scale()); + //--------------------------------------------------- + dec = HiveDecimal.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, dec.scale()); + resultDec = dec.scaleByPowerOfTen(2 * 38 - 1); + Assert.assertEquals( + "10000000000000000000000000000000000000", resultDec.toString()); + Assert.assertEquals(0, resultDec.scale()); + + //--------------------------------------------------- + oldDec = OldHiveDecimal.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, oldDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(2 * 38); + Assert.assertTrue(oldResultDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create( + "0.00000000000000000000000000000000000001"); + Assert.assertEquals(38, dec.scale()); + resultDec = dec.scaleByPowerOfTen(2 * 38); + Assert.assertTrue(resultDec == null); + + + //--------------------------------------------------- + oldDec = OldHiveDecimal.create( + "0.00000000000000000000000000000000000022"); + Assert.assertEquals(38, oldDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(38); + Assert.assertEquals( + "22", oldResultDec.toString()); + Assert.assertEquals(0, oldResultDec.scale()); + //--------------------------------------------------- + dec = HiveDecimal.create( + "0.00000000000000000000000000000000000022"); + Assert.assertEquals(38, dec.scale()); + resultDec = dec.scaleByPowerOfTen(38); + Assert.assertEquals( + "22", resultDec.toString()); + Assert.assertEquals(0, resultDec.scale()); + + //--------------------------------------------------- + oldDec = OldHiveDecimal.create("3.00001415926"); + Assert.assertEquals(11, oldDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(2); + Assert.assertEquals("300.001415926", oldResultDec.toString()); + Assert.assertEquals(9, oldResultDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(5); + Assert.assertEquals("300001.415926", oldResultDec.toString()); + Assert.assertEquals(6, oldResultDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(18); + Assert.assertEquals("3000014159260000000", oldResultDec.toString()); + Assert.assertEquals(0, oldResultDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(35); + Assert.assertEquals("300001415926000000000000000000000000", oldResultDec.toString()); + Assert.assertEquals(0, oldResultDec.scale()); + oldResultDec = oldDec.scaleByPowerOfTen(37); + Assert.assertEquals("30000141592600000000000000000000000000", oldResultDec.toString()); + Assert.assertEquals(0, oldResultDec.scale()); + //--------------------------------------------------- + dec = HiveDecimal.create("3.00001415926"); + Assert.assertEquals(11, dec.scale()); + Assert.assertEquals(1, dec.integerDigitCount()); + resultDec = dec.scaleByPowerOfTen(2); + Assert.assertEquals("300.001415926", resultDec.toString()); + Assert.assertEquals(9, resultDec.scale()); + Assert.assertEquals(3, resultDec.integerDigitCount()); + resultDec = dec.scaleByPowerOfTen(5); + Assert.assertEquals("300001.415926", resultDec.toString()); + Assert.assertEquals(6, resultDec.scale()); + Assert.assertEquals(6, resultDec.integerDigitCount()); + resultDec = dec.scaleByPowerOfTen(18); + Assert.assertEquals("3000014159260000000", resultDec.toString()); + Assert.assertEquals(0, resultDec.scale()); + Assert.assertEquals(19, resultDec.integerDigitCount()); + resultDec = dec.scaleByPowerOfTen(35); + Assert.assertEquals("300001415926000000000000000000000000", resultDec.toString()); + Assert.assertEquals(0, resultDec.scale()); + Assert.assertEquals(36, resultDec.integerDigitCount()); + resultDec = dec.scaleByPowerOfTen(37); + Assert.assertEquals("30000141592600000000000000000000000000", resultDec.toString()); + Assert.assertEquals(0, resultDec.scale()); + Assert.assertEquals(38, resultDec.integerDigitCount()); } @Test @Concurrent(count=4) - @Repeating(repetition=100) + public void testSingleWordDivision() { + + OldHiveDecimal oldDec1; + OldHiveDecimal oldDec2; + OldHiveDecimal oldResultDec; + + HiveDecimal dec1; + HiveDecimal dec2; + HiveDecimal resultDec; + + //--------------------------------------------------- + oldDec1 = OldHiveDecimal.create("839293"); + oldDec2 = OldHiveDecimal.create("8"); + oldResultDec = oldDec1.divide(oldDec2); + Assert.assertEquals("104911.625", oldResultDec.toString()); + //--------------------------------------------------- + dec1 = HiveDecimal.create("839293"); + dec2 = HiveDecimal.create("8"); + resultDec = dec1.divide(dec2); + Assert.assertEquals("104911.625", resultDec.toString()); // UNDONE + + //--------------------------------------------------- + oldDec1 = OldHiveDecimal.create("1"); + oldDec2 = OldHiveDecimal.create("3"); + oldResultDec = oldDec1.divide(oldDec2); + Assert.assertEquals("0.33333333333333333333333333333333333333", oldResultDec.toString()); + //--------------------------------------------------- + dec1 = HiveDecimal.create("1"); + dec2 = HiveDecimal.create("3"); + resultDec = dec1.divide(dec2); + Assert.assertEquals("0.33333333333333333333333333333333333333", resultDec.toString()); // UNDONE + + //--------------------------------------------------- + oldDec1 = OldHiveDecimal.create("1"); + oldDec2 = OldHiveDecimal.create("9"); + oldResultDec = oldDec1.divide(oldDec2); + Assert.assertEquals("0.11111111111111111111111111111111111111", oldResultDec.toString()); + //--------------------------------------------------- + dec1 = HiveDecimal.create("1"); + dec2 = HiveDecimal.create("9"); + resultDec = dec1.divide(dec2); + Assert.assertEquals("0.11111111111111111111111111111111111111", resultDec.toString()); // UNDONE + + //--------------------------------------------------- + oldDec1 = OldHiveDecimal.create("22"); + oldDec2 = OldHiveDecimal.create("7"); + oldResultDec = oldDec1.divide(oldDec2); + Assert.assertEquals("3.1428571428571428571428571428571428571", oldResultDec.toString()); + //--------------------------------------------------- + dec1 = HiveDecimal.create("22"); + dec2 = HiveDecimal.create("7"); + resultDec = dec1.divide(dec2); + Assert.assertEquals("3.1428571428571428571428571428571428571", resultDec.toString()); // UNDONE + + //--------------------------------------------------- + oldDec1 = OldHiveDecimal.create("1"); + oldDec2 = OldHiveDecimal.create("81"); + oldResultDec = oldDec1.divide(oldDec2); + Assert.assertEquals("0.01234567901234567901234567901234567901", oldResultDec.toString()); + //--------------------------------------------------- + dec1 = HiveDecimal.create("1"); + dec2 = HiveDecimal.create("81"); + resultDec = dec1.divide(dec2); + Assert.assertEquals("0.01234567901234567901234567901234567901", resultDec.toString()); // UNDONE + + //--------------------------------------------------- + oldDec1 = OldHiveDecimal.create("425"); + oldDec2 = OldHiveDecimal.create("1000000000000000"); + oldResultDec = oldDec1.divide(oldDec2); + Assert.assertEquals("0.000000000000425", oldResultDec.toString()); + //--------------------------------------------------- + dec1 = HiveDecimal.create("425"); + dec2 = HiveDecimal.create("1000000000000000"); + resultDec = dec1.divide(dec2); + Assert.assertEquals("0.000000000000425", resultDec.toString()); // UNDONE + + //--------------------------------------------------- + oldDec1 = OldHiveDecimal.create("0.000000000088"); + oldDec2 = OldHiveDecimal.create("1000000000000000"); + oldResultDec = oldDec1.divide(oldDec2); + Assert.assertEquals("0.000000000000000000000000088", oldResultDec.toString()); + Assert.assertEquals(27, oldResultDec.scale()); + //--------------------------------------------------- + dec1 = HiveDecimal.create("0.000000000088"); + dec2 = HiveDecimal.create("1000000000000000"); + resultDec = dec1.divide(dec2); + Assert.assertEquals("0.000000000000000000000000088", resultDec.toString()); // UNDONE + Assert.assertEquals(27, resultDec.scale()); + } + + @Test + @Concurrent(count=4) public void testDivide() { HiveDecimal dec1 = HiveDecimal.create("3.14"); HiveDecimal dec2 = HiveDecimal.create("3"); @@ -139,40 +673,583 @@ public void testDivide() { @Test @Concurrent(count=4) - @Repeating(repetition=100) public void testPlus() { - HiveDecimal dec1 = HiveDecimal.create("99999999999999999999999999999999999"); - HiveDecimal dec2 = HiveDecimal.create("1"); - Assert.assertNotNull(dec1.add(dec2)); + + OldHiveDecimal oldDec; + OldHiveDecimal oldDec2; + OldHiveDecimal oldResultDec; + + HiveDecimal dec; + HiveDecimal dec1; + HiveDecimal dec2; + HiveDecimal resultDec; + + String decStr; + String decStr2; dec1 = HiveDecimal.create("3.140"); + dec1.validate(); dec2 = HiveDecimal.create("1.00"); - Assert.assertEquals("4.14", dec1.add(dec2).toString()); + dec2.validate(); + resultDec = dec1.add(dec2); + resultDec.validate(); + Assert.assertEquals("4.14", resultDec.toString()); + + decStr = "3.140"; + decStr2 = "1.00"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("4.14", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("4.14", resultDec.toString()); + Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "3.140"; + decStr2 = "1.00000008733"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("4.14000008733", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("4.14000008733", resultDec.toString()); + Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "3.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("4.14000000000000000008733", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("4.14000000000000000008733", resultDec.toString()); + Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "30000000000.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("30000000001.14000000000000000008733", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("30000000001.14000000000000000008733", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "300000000000000.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("300000000000001.14000000000000000008733", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("300000000000001.14000000000000000008733", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + // Edge case? + decStr = "3000000000000000.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("3000000000000001.1400000000000000000873", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("3000000000000001.1400000000000000000873", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "300000000000000000000000.14"; + decStr2 = "0.0000055555555550008733"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("300000000000000000000000.14000555555556", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("300000000000000000000000.14000555555556", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "300000000000000000000000.14"; + decStr2 = "0.000005555555555000873355"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("300000000000000000000000.14000555555556", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("300000000000000000000000.14000555555556", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + + + // Example from HiveDecimal.add header comments. + decStr = "598575157855521918987423259.94094"; + decStr2 = "0.0000000000006711991169422033"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("598575157855521918987423259.94094", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("598575157855521918987423259.94094", resultDec.toString()); + Assert.assertEquals(27, resultDec.integerDigitCount()); + + decStr = "598575157855521918987423259.94094"; + decStr2 = "0.5555555555556711991169422033"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("598575157855521918987423260.49649555556", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("598575157855521918987423260.49649555556", resultDec.toString()); + Assert.assertEquals(27, resultDec.integerDigitCount()); + + decStr = "199999999.99995"; + decStr2 = "100000000.00005"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("300000000", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("300000000", resultDec.toString()); + Assert.assertEquals(9, resultDec.integerDigitCount()); + + dec1 = HiveDecimal.create("99999999999999999999999999999999999999"); + dec1.validate(); + Assert.assertEquals(38, dec1.integerDigitCount()); + dec2 = HiveDecimal.create("1"); + dec2.validate(); + Assert.assertNull(dec1.add(dec2)); } + @Test + @Concurrent(count=4) + public void testAdd() { + + OldHiveDecimal oldDec; + OldHiveDecimal oldDec2; + OldHiveDecimal oldResultDec; + + HiveDecimal dec; + HiveDecimal dec2; + HiveDecimal resultDec; + + // Use the example from HIVE-13423 where the integer digits of the result exceed the + // enforced precision/scale. + //--------------------------------------------------- + oldDec = OldHiveDecimal.create("98765432109876543210.12345"); + oldResultDec = oldDec.add(oldDec); + Assert.assertEquals("197530864219753086420.2469", oldResultDec.toString()); + oldResultDec = OldHiveDecimal.enforcePrecisionScale(oldResultDec, 38, 18); + Assert.assertTrue(oldResultDec == null); + //--------------------------------------------------- + dec = HiveDecimal.create("98765432109876543210.12345"); + assertTrue(dec != null); + dec.validate(); + resultDec = dec.add(dec); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("197530864219753086420.2469", resultDec.toString()); + // Assert.assertEquals(21, resultDec.integerDigitCount()); + resultDec = HiveDecimal.enforcePrecisionScale(resultDec, 38, 18); + Assert.assertTrue(resultDec == null); + + // Make sure zero trimming doesn't extend into the integer digits. + //--------------------------------------------------- + oldDec = OldHiveDecimal.create("199999999.99995"); + oldDec2 = OldHiveDecimal.create("100000000.00005"); + oldResultDec = oldDec.add(oldDec2); + Assert.assertEquals("300000000", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create("199999999.99995"); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create("100000000.00005"); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.add(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("300000000", resultDec.toString()); + Assert.assertEquals(9, resultDec.integerDigitCount()); + } + + @Test + @Concurrent(count=4) + public void testMinus() { + + OldHiveDecimal oldDec; + OldHiveDecimal oldDec2; + OldHiveDecimal oldResultDec; + + HiveDecimal dec; + HiveDecimal dec1; + HiveDecimal dec2; + HiveDecimal resultDec; + + String decStr; + String decStr2; + + dec1 = HiveDecimal.create("3.140"); + dec1.validate(); + dec2 = HiveDecimal.create("1.00"); + dec2.validate(); + resultDec = dec1.add(dec2); + resultDec.validate(); + Assert.assertEquals("4.14", resultDec.toString()); + + decStr = "3.140"; + decStr2 = "1.00"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("2.14", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("2.14", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "3.140"; + decStr2 = "1.00000008733"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("2.13999991267", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("2.13999991267", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "3.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("2.13999999999999999991267", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("2.13999999999999999991267", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "30000000000.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("29999999999.13999999999999999991267", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("29999999999.13999999999999999991267", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "300000000000000.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("299999999999999.13999999999999999991267", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("299999999999999.13999999999999999991267", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + // Edge case? + decStr = "3000000000000000.140"; + decStr2 = "1.00000000000000000008733"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("2999999999999999.1399999999999999999127", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("2999999999999999.1399999999999999999127", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "300000000000000000000000.14"; + decStr2 = "0.0000055555555550008733"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("300000000000000000000000.13999444444444", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("300000000000000000000000.13999444444444", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "300000000000000000000000.14"; + decStr2 = "0.000005555555555000873355"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("300000000000000000000000.13999444444444", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("300000000000000000000000.13999444444444", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + // Example from HiveDecimal.subtract header comments. + decStr = "598575157855521918987423259.94094"; + decStr2 = "0.0000000000006711991169422033"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("598575157855521918987423259.94094", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("598575157855521918987423259.94094", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + + decStr = "598575157855521918987423259.94094"; + decStr2 = "0.5555555555556711991169422033"; + //--------------------------------------------------- + oldDec = OldHiveDecimal.create(decStr); + oldDec2 = OldHiveDecimal.create(decStr2); + oldResultDec = oldDec.subtract(oldDec2); + Assert.assertEquals("598575157855521918987423259.38538444444", oldResultDec.toString()); + //--------------------------------------------------- + dec = HiveDecimal.create(decStr); + assertTrue(dec != null); + dec.validate(); + dec2 = HiveDecimal.create(decStr2); + assertTrue(dec2 != null); + dec2.validate(); + resultDec = dec.subtract(dec2); + assertTrue(resultDec != null); + resultDec.validate(); + Assert.assertEquals("598575157855521918987423259.38538444444", resultDec.toString()); + // Assert.assertEquals(1, resultDec.integerDigitCount()); + } @Test @Concurrent(count=4) - @Repeating(repetition=100) public void testSubtract() { - HiveDecimal dec1 = HiveDecimal.create("3.140"); - HiveDecimal dec2 = HiveDecimal.create("1.00"); - Assert.assertEquals("2.14", dec1.subtract(dec2).toString()); + HiveDecimal dec1 = HiveDecimal.create("3.140"); + assertTrue(dec1 != null); + dec1.validate(); + HiveDecimal dec2 = HiveDecimal.create("1.00"); + assertTrue(dec2 != null); + dec2.validate(); + HiveDecimal result = dec1.subtract(dec2); + assertTrue(result != null); + result.validate(); + Assert.assertEquals("2.14", result.toString()); + + dec1 = HiveDecimal.create("0.00001786135888657847525803"); + assertTrue(dec1 != null); + dec1.validate(); + dec2 = HiveDecimal.create("3.0000123456789"); + assertTrue(dec2 != null); + dec2.validate(); + result = dec1.subtract(dec2); + assertTrue(result != null); + result.validate(); + // System.out.println("TEST_HIVE_DECIMAL_SUBTRACT dec1 " + dec1.toString() + " dec2 " + dec2.toString() + " result " + result.toString()); + Assert.assertEquals("-2.99999448432001342152474197", result.toString()); } @Test @Concurrent(count=4) - @Repeating(repetition=100) public void testPosMod() { HiveDecimal hd1 = HiveDecimal.create("-100.91"); + assertTrue(hd1 != null); + hd1.validate(); HiveDecimal hd2 = HiveDecimal.create("9.8"); + assertTrue(hd2 != null); + hd2.validate(); HiveDecimal dec = hd1.remainder(hd2).add(hd2).remainder(hd2); + assertTrue(dec != null); + dec.validate(); Assert.assertEquals("6.89", dec.toString()); } @Test @Concurrent(count=4) - @Repeating(repetition=100) public void testHashCode() { Assert.assertEquals(HiveDecimal.create("9").hashCode(), HiveDecimal.create("9.00").hashCode()); Assert.assertEquals(HiveDecimal.create("0").hashCode(), HiveDecimal.create("0.00").hashCode()); @@ -180,7 +1257,6 @@ public void testHashCode() { @Test @Concurrent(count=4) - @Repeating(repetition=100) public void testException() { HiveDecimal dec = HiveDecimal.create("3.1415.926"); Assert.assertNull(dec); @@ -190,7 +1266,6 @@ public void testException() { @Test @Concurrent(count=4) - @Repeating(repetition=100) public void testBinaryConversion() { testBinaryConversion("0.00"); testBinaryConversion("-12.25"); @@ -198,19 +1273,3161 @@ public void testBinaryConversion() { } private void testBinaryConversion(String num) { - HiveDecimal dec = HiveDecimal.create(num); - int scale = 2; - byte[] d = dec.setScale(2).unscaledValue().toByteArray(); - Assert.assertEquals(dec, HiveDecimal.create(new BigInteger(d), scale)); + + byte[] d; + byte[] res; + int prec = 5; int len = (int) Math.ceil((Math.log(Math.pow(10, prec) - 1) / Math.log(2) + 1) / 8); - byte[] res = new byte[len]; - if ( dec.signum() == -1) - for (int i = 0; i < len; i++) - res[i] |= 0xFF; - System.arraycopy(d, 0, res, len-d.length, d.length); // Padding leading zeros. - Assert.assertEquals(dec, HiveDecimal.create(new BigInteger(res), scale)); + int scale = 2; + } + + @Test + @Concurrent(count=4) + public void testSerializationUtilsWriteRead() { + testSerializationUtilsWriteRead("0.00"); + testSerializationUtilsWriteRead("1"); + testSerializationUtilsWriteRead("234.79"); + testSerializationUtilsWriteRead("-12.25"); + testSerializationUtilsWriteRead("99999999999999999999999999999999"); + testSerializationUtilsWriteRead("-99999999999999999999999999999999"); + testSerializationUtilsWriteRead("99999999999999999999999999999999999999"); + // 12345678901234567890123456789012345678 + testSerializationUtilsWriteRead("-99999999999999999999999999999999999999"); + testSerializationUtilsWriteRead("999999999999.99999999999999999999"); + testSerializationUtilsWriteRead("-999999.99999999999999999999999999"); + testSerializationUtilsWriteRead("9999999999999999999999.9999999999999999"); + testSerializationUtilsWriteRead("-9999999999999999999999999999999.9999999"); + + testSerializationUtilsWriteRead("4611686018427387903"); // 2^62 - 1 + testSerializationUtilsWriteRead("-4611686018427387903"); + testSerializationUtilsWriteRead("4611686018427387904"); // 2^62 + testSerializationUtilsWriteRead("-4611686018427387904"); + + testSerializationUtilsWriteRead("42535295865117307932921825928971026431"); // 2^62*2^63 - 1 + testSerializationUtilsWriteRead("-42535295865117307932921825928971026431"); + testSerializationUtilsWriteRead("42535295865117307932921825928971026432"); // 2^62*2^63 + testSerializationUtilsWriteRead("-42535295865117307932921825928971026432"); + + testSerializationUtilsWriteRead("54216721532321902598.70"); + testSerializationUtilsWriteRead("-906.62545207002374150309544832320"); + } + + private void testSerializationUtilsWriteRead(String string) { + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ START ~~~~~~~~~~~~~~~~~"); + + HiveDecimal dec = HiveDecimal.create(string); + assertTrue(dec != null); + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER dec " + dec.toString()); + + BigInteger bigInteger = dec.unscaledValue(); + int scale = dec.scale(); + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER bigInteger " + bigInteger.toString()); + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER scale " + scale); + + //--------------------------------------------------- + OldHiveDecimal oldDec = OldHiveDecimal.create(string); + assertTrue(oldDec != null); + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER oldDec " + oldDec.toString()); + + BigInteger oldBigInteger = oldDec.unscaledValue(); + int oldScale = oldDec.scale(); + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER oldBigInteger " + oldBigInteger.toString()); + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER oldScale " + oldScale); + //--------------------------------------------------- + + long[] scratchLongs = new long[HiveDecimal.SCRATCH_LONGS_LEN]; + + int which = 0; + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + if (!dec.serializationUtilsWrite( + outputStream, scratchLongs)) { + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER serializationUtilsWrite failed"); + assertTrue(false); + } + byte[] bytes = outputStream.toByteArray(); + + ByteArrayOutputStream outputStreamExpected = new ByteArrayOutputStream(); + SerializationUtils.writeBigInteger(outputStreamExpected, bigInteger); + byte[] bytesExpected = outputStreamExpected.toByteArray(); + + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER check streams"); + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER bytes1 " + displayBytes(bytes, 0, bytes.length)); + if (!StringExpr.equal(bytes, 0, bytes.length, bytesExpected, 0, bytesExpected.length)) { + // Tailing zeroes difference ok. + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER streams not equal"); + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER bytesExpected " + displayBytes(bytesExpected, 0, bytesExpected.length)); + } + // Deserialize and check... + which = 1; + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); + BigInteger deserializedBigInteger = SerializationUtils.readBigInteger(byteArrayInputStream); + + which = 2; + ByteArrayInputStream byteArrayInputStreamExpected = new ByteArrayInputStream(bytesExpected); + BigInteger deserializedBigIntegerExpected = SerializationUtils.readBigInteger(byteArrayInputStreamExpected); + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER deserialized equals " + + // deserializedBigInteger.equals(deserializedBigIntegerExpected)); + if (!deserializedBigInteger.equals(deserializedBigIntegerExpected)) { + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER deserializedBigInteger " + deserializedBigInteger.toString() + + // " deserializedBigIntegerExpected " + deserializedBigIntegerExpected.toString()); + assertTrue(false); + } + + which = 3; + ByteArrayInputStream byteArrayInputStreamRead = new ByteArrayInputStream(bytes); + byte[] scratchBytes = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ]; + HiveDecimal readHiveDecimal = + HiveDecimal.serializationUtilsRead(byteArrayInputStreamRead, scale, scratchBytes); + assertTrue(readHiveDecimal != null); + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER read readHiveDecimal " + readHiveDecimal.toString() + + // " dec " + dec.toString() + " (scale parameter " + scale + ")"); + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER read toString equals " + + // readHiveDecimal.toString().equals(dec.toString())); + assertEquals(readHiveDecimal.toString(), dec.toString()); + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER read equals " + + // readHiveDecimal.equals(dec)); + assertEquals(readHiveDecimal, dec); + } catch (IOException e) { + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER " + e + " which " + which); + assertTrue(false); + } + // System.out.println("TEST_FAST_SERIALIZATION_UTILS_WRITE_BIG_INTEGER ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ END ~~~~~~~~~~~~~~~~~"); + + } + + public static enum BigDecimalFlavor { + NORMAL_RANGE, + FRACTIONS_ONLY, + NEGATIVE_SCALE, + LONG_TAIL + } + + public static enum BigDecimalPairFlavor { + RANDOM, + NEAR, + INVERSE + } + + private BigDecimal randHiveBigDecimal(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + switch (bigDecimalFlavor) { + case NORMAL_RANGE: + return randHiveBigDecimalNormalRange(r, digitAlphabet); + case FRACTIONS_ONLY: + return randHiveBigDecimalFractionsOnly(r, digitAlphabet); + case NEGATIVE_SCALE: + return randHiveBigDecimalNegativeScale(r, digitAlphabet); + case LONG_TAIL: + return randHiveBigDecimalLongTail(r, digitAlphabet); + default: + throw new RuntimeException("Unexpected big decimal flavor " + bigDecimalFlavor); + } + } + + private BigDecimal[] randHiveBigDecimalPair(Random r, String digitAlphabet, + BigDecimalFlavor bigDecimalFlavor, BigDecimalPairFlavor bigDecimalPairFlavor) { + BigDecimal[] pair = new BigDecimal[2]; + BigDecimal bigDecimal1 = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + pair[0] = bigDecimal1; + + BigDecimal bigDecimal2; + switch (bigDecimalPairFlavor) { + case RANDOM: + bigDecimal2 = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + break; + case NEAR: + bigDecimal2 = randHiveBigDecimalNear(r, bigDecimal1); + break; + case INVERSE: + bigDecimal2 = randHiveBigDecimalNear(r, bigDecimal1); + break; + default: + throw new RuntimeException("Unexpected big decimal pair flavor " + bigDecimalPairFlavor); + } + pair[1] = bigDecimal2; + return pair; + } + + private BigDecimal randHiveBigDecimalNormalRange(Random r, String digitAlphabet) { + String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(38)); + BigInteger bigInteger = new BigInteger(digits); + boolean negated = false; + if (r.nextBoolean()) { + bigInteger = bigInteger.negate(); + negated = true; + } + int scale = 0 + r.nextInt(38 + 1); + return new BigDecimal(bigInteger, scale); + } + + private BigDecimal randHiveBigDecimalNegativeScale(Random r, String digitAlphabet) { + String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(38)); + BigInteger bigInteger = new BigInteger(digits); + boolean negated = false; + if (r.nextBoolean()) { + bigInteger = bigInteger.negate(); + negated = true; + } + int scale = 0 + (r.nextBoolean() ? 0 : r.nextInt(38 + 1)); + if (r.nextBoolean()) { + scale = -scale; + } + return new BigDecimal(bigInteger, scale); + } + + private BigDecimal randHiveBigDecimalLongTail(Random r, String digitAlphabet) { + int scale = 0 + r.nextInt(38 + 20); + final int maxDigits = 38 + (scale == 0 ? 0 : 20); + String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(maxDigits)); + BigInteger bigInteger = new BigInteger(digits); + boolean negated = false; + if (r.nextBoolean()) { + bigInteger = bigInteger.negate(); + negated = true; + } + return new BigDecimal(bigInteger, scale); + } + + private BigDecimal randHiveBigDecimalFractionsOnly(Random r, String digitAlphabet) { + int scale = 1 + r.nextInt(38 + 1); + String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(scale)); + BigInteger bigInteger = new BigInteger(digits); + boolean negated = false; + if (r.nextBoolean()) { + bigInteger = bigInteger.negate(); + negated = true; + } + return new BigDecimal(bigInteger, scale); + } + + private BigDecimal randHiveBigDecimalNear(Random r, BigDecimal bigDecimal) { + + int scale = bigDecimal.scale(); + int delta = r.nextInt(10); + if (r.nextBoolean()) { + return bigDecimal.add(new BigDecimal(BigInteger.valueOf(delta), scale)); + } else { + return bigDecimal.subtract(new BigDecimal(BigInteger.valueOf(delta), scale)); + } + } + + private BigDecimal randHiveBigDecimalInverse(Random r, BigDecimal bigDecimal) { + if (bigDecimal.signum() == 0) { + return bigDecimal; + } + return BigDecimal.ONE.divide(bigDecimal); + } + + private BigInteger randHiveBigInteger(Random r, String digitAlphabet) { + String digits = RandomTypeUtil.getRandString(r, digitAlphabet, 1 + r.nextInt(38)); + BigInteger bigInteger = new BigInteger(digits); + boolean negated = false; + if (r.nextBoolean()) { + bigInteger = bigInteger.negate(); + negated = true; + } + return bigInteger; + } + + private boolean isTenPowerBug(String string) { + // // System.out.println("TEST_IS_TEN_TO_38_STRING isTenPowerBug " + string); + if (string.charAt(0) == '-') { + string = string.substring(1); + } + int index = string.indexOf('.'); + if (index != -1) { + if (index == 0) { + string = string.substring(1); + } else { + string = string.substring(0, index) + string.substring(index + 1); + } + } + // // System.out.println("TEST_IS_TEN_TO_38_STRING isTenPowerBug " + string); + return string.equals("100000000000000000000000000000000000000"); + } + + //------------------------------------------------------------------------------------------------ + + private static int POUND_FACTOR = 1000; + + //------------------------------------------------------------------------------------------------ + + private static String[] specialDecimalStrings = new String[] { + "0", + "1", + "-1", + "10", + "-10", + "100", + "-100", + "127", // Byte.MAX_VALUE + "127.1", + "127.0008", + "127.49", + "127.5", + "127.9999999999999999999", + "-127", + "-127.1", + "-127.0008", + "-127.49", + "-127.5", + "-127.999999", + "128", + "128.1", + "128.0008", + "128.49", + "128.5", + "128.9999999999999999999", + "-128", // Byte.MIN_VALUE + "-128.1", + "-128.0008", + "-128.49", + "-128.5", + "-128.999", + "129", + "129.1", + "-129", + "-129.1", + "1000", + "-1000", + "10000", + "-10000", + "32767", // Short.MAX_VALUE + "32767.1", + "32767.0008", + "32767.49", + "32767.5", + "32767.99999999999", + "-32767", + "-32767.1", + "-32767.0008", + "-32767.49", + "-32767.5", + "-32767.9", + "32768", + "32768.1", + "32768.0008", + "32768.49", + "32768.5", + "32768.9999999999", + "-32768", // Short.MIN_VALUE + "-32768.1", + "-32768.0008", + "-32768.49", + "-32768.5", + "-32768.9999999", + "32769", + "32769.1", + "-32769", + "-32769.1", + "100000", + "-100000", + "1000000", + "-1000000", + "10000000", + "-10000000", + "100000000", + "99999999", // 10^8 - 1 + "-99999999", + "-100000000", + "1000000000", + "-1000000000", + "2147483647", // Integer.MAX_VALUE + "2147483647.1", + "2147483647.0008", + "2147483647.49", + "2147483647.5", + "2147483647.9999999999", + "-2147483647", + "-2147483647.1", + "-2147483647.0008", + "-2147483647.49", + "-2147483647.5", + "-2147483647.9999999999999999999", + "2147483648", + "2147483648.1", + "2147483648.0008", + "2147483648.49", + "2147483648.5", + "2147483648.9", + "-2147483648", // Integer.MIN_VALUE + "-2147483648.1", + "-2147483648.0008", + "-2147483648.49", + "-2147483648.5", + "-2147483648.999", + "2147483649", + "2147483649.1", + "-2147483649", + "-2147483649.1", + "10000000000", + "-10000000000", + "100000000000", + "-100000000000", + "1000000000000", + "-1000000000000", + "10000000000000", + "-10000000000000", + "100000000000000", + "-100000000000000", + "999999999999999", + "-999999999999999", + "1000000000000000", // 10^15 + "-1000000000000000", + "9999999999999999", // 10^16 - 1 + "-9999999999999999", + "10000000000000000", // 10^16 + "-10000000000000000", + "100000000000000000", + "-100000000000000000", + "1000000000000000000", + "-1000000000000000000", + "9223372036854775807", // Long.MAX_VALUE + "9223372036854775807.1", + "9223372036854775807.0008", + "9223372036854775807.49", + "9223372036854775807.5", + "9223372036854775807.9", + "-9223372036854775807", + "-9223372036854775807.1", + "-9223372036854775807.0008", + "-9223372036854775807.49", + "-9223372036854775807.5", + "-9223372036854775807.9999999999999999999", + "-9223372036854775808", + "-9223372036854775808.1", + "9223372036854775808", + "9223372036854775808.1", + "9223372036854775808.0008", + "9223372036854775808.49", + "9223372036854775808.5", + "9223372036854775808.9", + "9223372036854775809", + "9223372036854775809.1", + "-9223372036854775808", // Long.MIN_VALUE + "-9223372036854775808.1", + "-9223372036854775808.0008", + "-9223372036854775808.49", + "-9223372036854775808.5", + "-9223372036854775808.9999999", + "9223372036854775809", + "9223372036854775809.1", + "-9223372036854775809", + "-9223372036854775809.1", + "10000000000000000000000000000000", // 10^31 + "-10000000000000000000000000000000", + "99999999999999999999999999999999", // 10^32 - 1 + "-99999999999999999999999999999999", + "100000000000000000000000000000000", // 10^32 + "-100000000000000000000000000000000", + "10000000000000000000000000000000000000", // 10^37 + "-10000000000000000000000000000000000000", + "99999999999999999999999999999999999999", // 10^38 - 1 + "-99999999999999999999999999999999999999", + "100000000000000000000000000000000000000", // 10^38 + "-100000000000000000000000000000000000000", + "1000000000000000000000000000000000000000", // 10^39 + "-1000000000000000000000000000000000000000", + + "18446744073709551616", // Unsigned 64 max. + "-18446744073709551616", + "340282366920938463463374607431768211455", // 2^128 - 1 + "-340282366920938463463374607431768211455", + + "0.999999999999999", + "-0.999999999999999", + "0.0000000000000001", // 10^-15 + "-0.0000000000000001", + "0.9999999999999999", + "-0.9999999999999999", + "0.00000000000000001", // 10^-16 + "-0.00000000000000001", + "0.00000000000000000000000000000001", // 10^-31 + "-0.00000000000000000000000000000001", + "0.99999999999999999999999999999999", // 10^-32 + 1 + "-0.99999999999999999999999999999999", + "0.000000000000000000000000000000001", // 10^-32 + "-0.000000000000000000000000000000001", + "0.00000000000000000000000000000000000001", // 10^-37 + "-0.00000000000000000000000000000000000001", + "0.99999999999999999999999999999999999999", // 10^-38 + 1 + "-0.99999999999999999999999999999999999999", + "0.000000000000000000000000000000000000001", // 10^-38 + "-0.000000000000000000000000000000000000001", + "0.0000000000000000000000000000000000000001", // 10^-39 + "-0.0000000000000000000000000000000000000001", + "0.0000000000000000000000000000000000000005", // 10^-39 (rounds) + "-0.0000000000000000000000000000000000000005", + "0.340282366920938463463374607431768211455", // (2^128 - 1) * 10^-39 + "-0.340282366920938463463374607431768211455", + "0.000000000000000000000000000000000000001", // 10^-38 + "-0.000000000000000000000000000000000000001", + "0.000000000000000000000000000000000000005", // 10^-38 + "-0.000000000000000000000000000000000000005", + + "234.79", + "342348.343", + "12.25", + "-12.25", + "72057594037927935", // 2^56 - 1 + "-72057594037927935", + "72057594037927936", // 2^56 + "-72057594037927936", + "5192296858534827628530496329220095", // 2^56 * 2^56 - 1 + "-5192296858534827628530496329220095", + "5192296858534827628530496329220096", // 2^56 * 2^56 + "-5192296858534827628530496329220096", + + "54216721532321902598.70", + "-906.62545207002374150309544832320", + "-0.0709351061072", + "1460849063411925.53", + "8.809130E-33", + "-4.0786300706013636202E-20", + "-3.8823936518E-1", + "-3.8823936518E-28", + "-3.8823936518E-29", + "598575157855521918987423259.94094", + "299999448432.001342152474197", + "1786135888657847525803324040144343378.09799306448796128931113691624", // More than 38 digits. + "-1786135888657847525803324040144343378.09799306448796128931113691624", + "57847525803324040144343378.09799306448796128931113691624", + "0.999999999999999999990000", + "005.34000", + "1E-90", + + "0.4", + "-0.4", + "0.5", + "-0.5", + "0.6", + "-0.6", + "1.4", + "-1.4", + "1.5", + "-1.5", + "1.6", + "-1.6", + "2.4", + "-2.4", + "2.49", + "-2.49", + "2.5", + "-2.5", + "2.51", + "-2.51", + "-2.5", + "2.6", + "-2.6", + "3.00001415926" + }; + + private static BigDecimal[] specialBigDecimals = stringArrayToBigDecimals(specialDecimalStrings); + + // decimal_1_1.txt + private static String[] decimal_1_1_txt = { + "0.0", + "0.0000", + ".0", + "0.1", + "0.15", + "0.9", + "0.94", + "0.99", + "0.345", + "1.0", + "1", + "0", + "00", + "22", + "1E-9", + "-0.0", + "-0.0000", + "-.0", + "-0.1", + "-0.15", + "-0.9", + "-0.94", + "-0.99", + "-0.345", + "-1.0", + "-1", + "-0", + "-00", + "-22", + "-1E-9" + }; + + // kv7.txt KEYS + private static String[] kv7_txt_keys = { + "-4400", + "1E+99", + "1E-99", + "0", + "100", + "10", + "1", + "0.1", + "0.01", + "200", + "20", + "2", + "0", + "0.2", + "0.02", + "0.3", + "0.33", + "0.333", + "-0.3", + "-0.33", + "-0.333", + "1.0", + "2", + "3.14", + "-1.12", + "-1.12", + "-1.122", + "1.12", + "1.122", + "124.00", + "125.2", + "-1255.49", + "3.14", + "3.14", + "3.140", + "0.9999999999999999999999999", + "-1234567890.1234567890", + "1234567890.1234567800" + }; + + private static String standardAlphabet = "0123456789"; + + private static String[] sparseAlphabets = new String[] { + + "0000000000000000000000000000000000000003", + "0000000000000000000000000000000000000009", + "0000000000000000000000000000000000000001", + "0000000000000000000003", + "0000000000000000000009", + "0000000000000000000001", + "0000000000091", + "000000000005", + "9", + "5555555555999999999000000000000001111111", + "24680", + "1" + }; + + private static BigDecimal[] stringArrayToBigDecimals(String[] strings) { + BigDecimal[] result = new BigDecimal[strings.length]; + for (int i = 0; i < strings.length; i++) { + result[i] = new BigDecimal(strings[i]); + } + return result; + } + +//------------------------------------------------------------------------------------------------ + + @Test + public void testDecimalsWithOneOne() { + doTestDecimalsWithPrecisionScale(decimal_1_1_txt, 1, 1); + } + + @Test + public void testDecimalsWithKv7Keys() { + doTestDecimalsWithPrecisionScale(kv7_txt_keys, 38, 18); + } + + public void doTestDecimalsWithPrecisionScale(String[] decStrings, int precision, int scale) { + + OldHiveDecimal oldSum = OldHiveDecimal.create(0); + HiveDecimalWritable sum = new HiveDecimalWritable(0); + + for (int i = 0; i < decStrings.length; i++) { + // System.out.println("TEST_DECIMALS_WITH_PRECISION_SCALE >>>>>>>>>>>>"); + + String string = decStrings[i]; + // System.out.println("TEST_DECIMALS_WITH_PRECISION_SCALE string[" + i + "] " + string); + + OldHiveDecimal oldDec = OldHiveDecimal.create(string); + // System.out.println("TEST_DECIMALS_WITH_PRECISION_SCALE oldDec " + oldDec); + + OldHiveDecimal resultOldDec; + if (oldDec == null) { + resultOldDec = null; + } else { + resultOldDec = OldHiveDecimal.enforcePrecisionScale(oldDec, precision, scale); + } + // System.out.println("TEST_DECIMALS_WITH_PRECISION_SCALE resultOldDec " + resultOldDec); + + HiveDecimal dec = HiveDecimal.create(string); + // System.out.println("TEST_DECIMALS_WITH_PRECISION_SCALE dec " + dec); + + if (oldDec == null) { + Assert.assertTrue(dec == null); + continue; + } + HiveDecimal resultDec = HiveDecimal.enforcePrecisionScale(dec, precision, scale); + // System.out.println("TEST_DECIMALS_WITH_PRECISION_SCALE resultDec " + resultDec); + if (resultOldDec == null) { + Assert.assertTrue(resultDec == null); + continue; + } + + Assert.assertEquals(resultOldDec.toString(), resultDec.toString()); + Assert.assertEquals(resultOldDec.toFormatString(scale), resultDec.toFormatString(scale)); + + oldSum = oldSum.add(resultOldDec); + sum.mutateAdd(resultDec); + } + + Assert.assertEquals(oldSum.toString(), sum.toString()); + // System.out.println("TEST_DECIMALS_WITH_PRECISION_SCALE oldSum " + oldSum); + // System.out.println("TEST_DECIMALS_WITH_PRECISION_SCALE sum " + sum); + } + +//------------------------------------------------------------------------------------------------ + + @Test + public void testDecimalsWithOneOneWritable() { + doTestDecimalsWithPrecisionScaleWritable(decimal_1_1_txt, 1, 1); + } + + @Test + public void testDecimalsWithKv7KeysWritable() { + doTestDecimalsWithPrecisionScaleWritable(kv7_txt_keys, 38, 18); + } + + public void doTestDecimalsWithPrecisionScaleWritable(String[] decStrings, int precision, int scale) { + + OldHiveDecimal oldSum = OldHiveDecimal.create(0); + HiveDecimalWritable sum = new HiveDecimalWritable(0); + + for (int i = 0; i < decStrings.length; i++) { + String string = decStrings[i]; + + OldHiveDecimal oldDec = OldHiveDecimal.create(string); + OldHiveDecimal resultOldDec; + if (oldDec == null) { + resultOldDec = null; + } else { + resultOldDec = OldHiveDecimal.enforcePrecisionScale(oldDec, precision, scale); + } + + HiveDecimalWritable decWritable = new HiveDecimalWritable(string); + if (oldDec == null) { + Assert.assertTrue(!decWritable.isSet()); + continue; + } + decWritable.mutateEnforcePrecisionScale(precision, scale);; + if (resultOldDec == null) { + Assert.assertTrue(!decWritable.isSet()); + continue; + } + + Assert.assertEquals(resultOldDec.toString(), decWritable.toString()); + Assert.assertEquals(resultOldDec.toFormatString(scale), decWritable.toFormatString(scale)); + // System.out.println("TEST_DECIMALS_WITH_PRECISION_SCALE_WRITABLE oldDec " + oldDec + " resultOldDec " + resultOldDec); + // System.out.println("TEST_DECIMALS_WITH_PRECISION_SCALE_WRITABLE dec " + decWritable); + + oldSum = oldSum.add(resultOldDec); + sum.mutateAdd(decWritable); + } + + Assert.assertEquals(oldSum.toString(), sum.toString()); + // System.out.println("TEST_DECIMALS_WITH_PRECISION_SCALE_WRITABLE oldSum " + oldSum); + // System.out.println("TEST_DECIMALS_WITH_PRECISION_SCALE_WRITABLE sum " + sum); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testSort() { + doTestSort(decimal_1_1_txt); + } + + @Test + public void testSortSpecial() { + doTestSort(specialDecimalStrings); + } + + @Test + public void testSortRandom() { + Random r = new Random(14434); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestSortRandom(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestSortRandom(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + public void doTestSortRandom(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + String[] randomStrings = new String[POUND_FACTOR]; + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + randomStrings[i] = bigDecimal.toString(); + } + + doTestSort(randomStrings); } + public void doTestSort(String[] decStrings) { + + OldHiveDecimal[] oldDecSortArray = new OldHiveDecimal[decStrings.length]; + HiveDecimal[] decSortArray = new HiveDecimal[decStrings.length]; + + int count = 0; + for (int i = 0; i < decStrings.length; i++) { + String string = decStrings[i]; + + OldHiveDecimal oldDec = OldHiveDecimal.create(string); + if (oldDec == null) { + continue; + } + if (isTenPowerBug(oldDec.toString())) { + continue; + } + oldDecSortArray[count] = oldDec; + + HiveDecimal dec = HiveDecimal.create(string); + if (dec == null) { + // System.out.println("DO_TEST_SORT new decimal can't convert string " + string); + // System.out.println("DO_TEST_SORT oldDec " + oldDec); + Assert.assertTrue(false); + } + decSortArray[count] = dec; + count++; + } + + oldDecSortArray = Arrays.copyOf(oldDecSortArray, count); + decSortArray = Arrays.copyOf(decSortArray, count); + + Arrays.sort(oldDecSortArray); + Arrays.sort(decSortArray); + + for (int i = 0; i < count; i++) { + String oldDecString = oldDecSortArray[i].toString(); + String decString = decSortArray[i].toString(); + + if (!oldDecString.equals(decString)) { + // System.out.println("DO_TEST_SORT oldDecString " + oldDecString); + // System.out.println("DO_TEST_SORT decString " + decString); + Assert.assertTrue(false); + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCreateFromBigDecimal() { + Random r = new Random(14434); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomCreateFromBigDecimal(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomCreateFromBigDecimal(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomCreateFromBigDecimal(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestCreateFromBigDecimal(bigDecimal); + } + } + + @Test + public void testCreateFromBigDecimalSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestCreateFromBigDecimal(bigDecimal); + } + } + + private void doTestCreateFromBigDecimal(BigDecimal bigDecimal) { + + // System.out.println("TEST_CREATE_FROM_BIG_INTEGER bigDecimal " + bigDecimal + " scale " + bigDecimal.scale()); + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_CREATE_FROM_BIG_INTEGER oldDec " + oldDec); + // System.out.println("TEST_CREATE_FROM_BIG_INTEGER dec " + dec); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCreateFromBigDecimalNoRound() { + Random r = new Random(14434); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomCreateFromBigDecimalNoRound(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomCreateFromBigDecimalNoRound(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomCreateFromBigDecimalNoRound(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestCreateFromBigDecimalNoRound(bigDecimal); + } + } + + @Test + public void testCreateFromBigDecimalNoRoundSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestCreateFromBigDecimalNoRound(bigDecimal); + } + } + + private void doTestCreateFromBigDecimalNoRound(BigDecimal bigDecimal) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal, /* allowRounding */ false); + HiveDecimal dec = HiveDecimal.create(bigDecimal, /* allowRounding */ false); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + if (dec == null) { + // System.out.println("TEST_CREATE_FROM_BIG_INTEGER_NO_ROUND not equal oldDec have result but dec is null " + oldDec.toString() + + // " scale " + oldDec.scale() + " bigDecimal " + bigDecimal); + Assert.assertTrue(false); + } + dec.validate(); + // System.out.println("TEST_CREATE_FROM_BIG_INTEGER_NO_ROUND oldDec " + oldDec); + // System.out.println("TEST_CREATE_FROM_BIG_INTEGER_NO_ROUND dec " + dec); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testCreateFromBigDecimalNegativeScaleSpecial() { + Random r = new Random(223965); + for (BigDecimal bigDecimal : specialBigDecimals) { + int negativeScale = -(0 + r.nextInt(38 + 1)); + bigDecimal = bigDecimal.setScale(negativeScale, BigDecimal.ROUND_HALF_UP); + doTestCreateFromBigDecimalNegativeScale(bigDecimal); + } + } + + private void doTestCreateFromBigDecimalNegativeScale(BigDecimal bigDecimal) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_CREATE_FROM_BIG_INTEGER_NEGATIVE_SCALE oldDec " + oldDec); + // System.out.println("TEST_CREATE_FROM_BIG_INTEGER_NEGATIVE_SCALE dec " + dec); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCreateFromBigInteger() { + doTestRandomCreateFromBigInteger(standardAlphabet); + } + + @Test + public void testRandomCreateFromBigIntegerSparse() { + for (String digitAlphabet : sparseAlphabets) { + doTestRandomCreateFromBigInteger(digitAlphabet); + } + } + + private void doTestRandomCreateFromBigInteger(String digitAlphabet) { + + Random r = new Random(11241); + for (int i = 0; i < POUND_FACTOR; i++) { + BigInteger bigInteger = randHiveBigInteger(r, digitAlphabet); + + doTestCreateFromBigInteger(bigInteger); + } + } + + @Test + public void testCreateFromBigIntegerSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestCreateFromBigInteger(bigDecimal.unscaledValue()); + } + } + + private void doTestCreateFromBigInteger(BigInteger bigInteger) { + + // System.out.println("TEST_CREATE_FROM_BIG_INTEGER bigInteger " + bigInteger); + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigInteger); + HiveDecimal dec = HiveDecimal.create(bigInteger); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_CREATE_FROM_BIG_INTEGER oldDec " + oldDec); + // System.out.println("TEST_CREATE_FROM_BIG_INTEGER dec " + dec); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCreateFromBigIntegerScale() { + doTestRandomCreateFromBigIntegerScale(standardAlphabet, false); + } + + @Test + public void testRandomCreateFromBigIntegerScaleFractionsOnly() { + doTestRandomCreateFromBigIntegerScale(standardAlphabet, true); + } + + @Test + public void testRandomCreateFromBigIntegerScaleSparse() { + for (String digitAlphabet : sparseAlphabets) { + doTestRandomCreateFromBigIntegerScale(digitAlphabet, false); + } + } + + private void doTestRandomCreateFromBigIntegerScale(String digitAlphabet, boolean fractionsOnly) { + + Random r = new Random(4448); + for (int i = 0; i < POUND_FACTOR; i++) { + BigInteger bigInteger = randHiveBigInteger(r, digitAlphabet); + + int scale; + if (fractionsOnly) { + scale = 1 + r.nextInt(38); + } else { + scale = 0 + r.nextInt(38 + 1); + } + + doTestCreateFromBigIntegerScale(bigInteger, scale); + } + } + + @Test + public void testCreateFromBigIntegerScaleSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestCreateFromBigIntegerScale(bigDecimal.unscaledValue(), bigDecimal.scale()); + } + } + + private void doTestCreateFromBigIntegerScale(BigInteger bigInteger, int scale) { + + // System.out.println("TEST_CREATE_FROM_BIG_INTEGER_SCALE bigInteger " + bigInteger + " scale " + scale); + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigInteger, scale); + HiveDecimal dec = HiveDecimal.create(bigInteger, scale); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_CREATE_FROM_BIG_INTEGER oldDec " + oldDec); + // System.out.println("TEST_CREATE_FROM_BIG_INTEGER dec " + dec); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomSetFromDouble() { + Random r = new Random(14434); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomSetFromDouble(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomSetFromDouble(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomSetFromDouble(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestSetFromDouble(bigDecimal.doubleValue()); + } + } + + private void doTestRandomSetFromDouble() { + + Random r = new Random(94762); + for (int i = 0; i < POUND_FACTOR; i++) { + double randomDouble = r.nextDouble(); + + doTestSetFromDouble(randomDouble); + } + } + + @Test + public void testSetFromDoubleSpecial() { + + for (String specialString : specialDecimalStrings) { + double specialDouble = Double.valueOf(specialString); + doTestSetFromDouble(specialDouble); + } + } + + private void doTestSetFromDouble(double doubleValue) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(Double.toString(doubleValue)); + if (oldDec == null) { + return; + } + // System.out.println("TEST_SET_FROM_DOUBLE oldDec " + oldDec.toString()); + HiveDecimal dec = HiveDecimal.create(doubleValue); + if (dec == null) { + // System.out.println("TEST_SET_FROM_DOUBLE not equal oldDec have result but dec is null " + oldDec.toString() + " scale " + oldDec.scale()); + // Assert.assertTrue(false); + } + dec.validate(); + // System.out.println("TEST_SET_FROM_DOUBLE dec " + dec); + if (!oldDec.toString().equals(dec.toString())) { + // System.out.println("TEST_SET_FROM_DOUBLE not equal oldDec " + oldDec.toString() + " dec " + dec.toString()); + BigDecimal bigDecimal = new BigDecimal(dec.toString()); + for (int i = 16; i < 18;i++) { + BigDecimal trial = bigDecimal.setScale(i, HiveDecimal.ROUND_HALF_UP); + // System.out.println("TEST_SET_FROM_DOUBLE not equal i " + i + " " + trial + " " + trial.toString().equals(oldDec.toString())); + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCreateFromString() { + Random r = new Random(1221); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomCreateFromString(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomCreateFromString(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomCreateFromString(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestCreateFromString(bigDecimal); + } + } + + @Test + public void testCreateFromStringSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestCreateFromString(bigDecimal); + } + } + + private void doTestCreateFromString(BigDecimal bigDecimal) { + + String decString = bigDecimal.toPlainString(); + + OldHiveDecimal oldDec = OldHiveDecimal.create(decString); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(decString); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_CREATE_FROM_STRING oldDec " + oldDec); + // System.out.println("TEST_CREATE_FROM_STRING dec " + dec); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCreateFromStringPadded() { + Random r = new Random(9774); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomCreateFromStringPadded(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomCreateFromStringPadded(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomCreateFromStringPadded(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestCreateFromStringPadded(bigDecimal); + } + } + + @Test + public void testCreateFromStringPaddedSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestCreateFromStringPadded(bigDecimal); + } + } + + private void doTestCreateFromStringPadded(BigDecimal bigDecimal) { + + String decString = bigDecimal.toPlainString(); + String decString1 = " " + decString; + String decString2 = decString + " "; + String decString3 = " " + decString + " "; + String decString4 = " " + decString; + String decString5 = decString + " "; + String decString6 = " " + decString + " "; + + OldHiveDecimal oldDec; + HiveDecimal dec; + + oldDec = OldHiveDecimal.create(decString); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + // System.out.println("TEST_CREATE_FROM_STRING oldDec " + oldDec); + + dec = HiveDecimal.create(decString1, true); + if (oldDec == null) { + assertTrue(dec == null); + } else { + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_CREATE_FROM_STRING dec " + dec); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + dec = HiveDecimal.create(decString2, true); + if (oldDec == null) { + assertTrue(dec == null); + } else { + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_CREATE_FROM_STRING dec " + dec); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + dec = HiveDecimal.create(decString3, true); + if (oldDec == null) { + assertTrue(dec == null); + } else { + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_CREATE_FROM_STRING dec " + dec); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + dec = HiveDecimal.create(decString4, true); + if (oldDec == null) { + assertTrue(dec == null); + } else { + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_CREATE_FROM_STRING dec " + dec); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + dec = HiveDecimal.create(decString5, true); + if (oldDec == null) { + assertTrue(dec == null); + } else { + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_CREATE_FROM_STRING dec " + dec); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + dec = HiveDecimal.create(decString6, true); + if (oldDec == null) { + assertTrue(dec == null); + } else { + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_CREATE_FROM_STRING dec " + dec); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCreateFromStringExponent() { + Random r = new Random(297111); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomCreateFromStringPadded(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomCreateFromStringPadded(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomCreateFromStringExponent(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestCreateFromStringExponent(bigDecimal); + } + } + + @Test + public void testCreateFromStringExponentSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestCreateFromStringExponent(bigDecimal); + } + } + + private void doTestCreateFromStringExponent(BigDecimal bigDecimal) { + + // Use toString which will have exponents instead of toPlainString. + String decString = bigDecimal.toString(); + + // System.out.println("TEST_CREATE_FROM_STRING decString " + decString); + + OldHiveDecimal oldDec = OldHiveDecimal.create(decString); + HiveDecimal dec = HiveDecimal.create(decString); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_CREATE_FROM_STRING oldDec " + oldDec); + // System.out.println("TEST_CREATE_FROM_STRING dec " + dec); + + Assert.assertEquals(oldDec.toString(), dec.toString()); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomLongValue() { + Random r = new Random(73293); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomLongValue(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomLongValue(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomLongValue(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestLongValue(bigDecimal); + } + } + + @Test + public void testLongValueSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestLongValue(bigDecimal); + } + } + + private void doTestLongValue(BigDecimal bigDecimal) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_LONG_VALUE oldDec " + oldDec); + // System.out.println("TEST_LONG_VALUE dec " + dec); + + BigDecimal bigDecimalOldDec = oldDec.bigDecimalValue(); + BigDecimal bigDecimalDec = dec.bigDecimalValue(); + Assert.assertEquals(bigDecimalOldDec, bigDecimalDec); + + BigDecimal bigDecimalFloor = bigDecimalDec.setScale(0, BigDecimal.ROUND_DOWN); + long longValueBigDecimalFloor = bigDecimalFloor.longValue(); + boolean isLongExpected = + bigDecimalFloor.equals(bigDecimalDec.valueOf(longValueBigDecimalFloor)); + + boolean decIsLong = dec.isLong(); + long oldDecLong = oldDec.longValue(); + long decLong = dec.longValue(); + if (isLongExpected != decIsLong) { + // System.out.println("TEST_LONG_VALUE isLong mismatch"); + // System.out.println("TEST_LONG_VALUE bigDecimalFloor " + bigDecimalFloor); + // System.out.println("TEST_LONG_VALUE isLongExpected " + isLongExpected); + // System.out.println("TEST_LONG_VALUE decIsLong " + decIsLong); + // System.out.println("TEST_LONG_VALUE oldDecLong " + oldDecLong); + // System.out.println("TEST_LONG_VALUE decLong " + decLong); + Assert.assertTrue(false); + } + + if (decIsLong) { + if (oldDecLong != decLong) { + // System.out.println("TEST_LONG_VALUE longValue mismatch"); + // System.out.println("TEST_LONG_VALUE oldDecLong " + oldDecLong); + // System.out.println("TEST_LONG_VALUE decLong " + decLong); + Assert.assertTrue(false); + } + } else { + // Not sure whether corrupted values will be equal... + // System.out.println("TEST_LONG_VALUE corrupt oldDec " + oldDec); + // System.out.println("TEST_LONG_VALUE corrupt dec " + dec); + // System.out.println("TEST_LONG_VALUE corrupt oldDecLong " + oldDecLong); + // System.out.println("TEST_LONG_VALUE corrupt decLong " + decLong); + + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomIntValue() { + Random r = new Random(98333); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomIntValue(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomIntValue(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomIntValue(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestIntValue(bigDecimal); + } + } + + @Test + public void testIntValueSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestIntValue(bigDecimal); + } + } + + private void doTestIntValue(BigDecimal bigDecimal) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_INT_VALUE oldDec " + oldDec); + // System.out.println("TEST_INT_VALUE dec " + dec); + + BigDecimal bigDecimalOldDec = oldDec.bigDecimalValue(); + BigDecimal bigDecimalDec = dec.bigDecimalValue(); + Assert.assertEquals(bigDecimalOldDec, bigDecimalDec); + + BigDecimal bigDecimalFloor = bigDecimalDec.setScale(0, BigDecimal.ROUND_DOWN); + int intValueBigDecimalFloor = bigDecimalFloor.intValue(); + boolean isIntExpected = + bigDecimalFloor.equals(bigDecimalDec.valueOf(intValueBigDecimalFloor)); + + boolean decIsInt = dec.isInt(); + int oldDecInt = oldDec.intValue(); + int decInt = dec.intValue(); + if (isIntExpected != decIsInt) { + // System.out.println("TEST_INT_VALUE isInt mismatch"); + // System.out.println("TEST_INT_VALUE bigDecimalFloor " + bigDecimalFloor); + // System.out.println("TEST_INT_VALUE isIntExpected " + isIntExpected); + // System.out.println("TEST_INT_VALUE decIsInt " + decIsInt); + // System.out.println("TEST_INT_VALUE oldDecInt " + oldDecInt); + // System.out.println("TEST_INT_VALUE decInt " + decInt); + Assert.assertTrue(false); + } + + if (decIsInt) { + if (oldDecInt != decInt) { + // System.out.println("TEST_INT_VALUE intValue mismatch"); + // System.out.println("TEST_INT_VALUE oldDecInt " + oldDecInt); + // System.out.println("TEST_INT_VALUE decInt " + decInt); + Assert.assertTrue(false); + } + } else { + // Not sure whether corrupted values will be equal... + // System.out.println("TEST_INT_VALUE corrupt oldDec " + oldDec); + // System.out.println("TEST_INT_VALUE corrupt dec " + dec); + // System.out.println("TEST_INT_VALUE corrupt oldDecInt " + oldDecInt); + // System.out.println("TEST_INT_VALUE corrupt decInt " + decInt); + + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomShortValue() { + Random r = new Random(15); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomShortValue(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomShortValue(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomShortValue(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestShortValue(bigDecimal); + } + } + + @Test + public void testShortValueSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestShortValue(bigDecimal); + } + } + + private void doTestShortValue(BigDecimal bigDecimal) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_INT_VALUE oldDec " + oldDec); + // System.out.println("TEST_INT_VALUE dec " + dec); + + BigDecimal bigDecimalOldDec = oldDec.bigDecimalValue(); + BigDecimal bigDecimalDec = dec.bigDecimalValue(); + Assert.assertEquals(bigDecimalOldDec, bigDecimalDec); + + BigDecimal bigDecimalFloor = bigDecimalDec.setScale(0, BigDecimal.ROUND_DOWN); + short shortValueBigDecimalFloor = bigDecimalFloor.shortValue(); + boolean isShortExpected = + bigDecimalFloor.equals(bigDecimalDec.valueOf(shortValueBigDecimalFloor)); + + boolean decIsShort = dec.isShort(); + short oldDecShort = oldDec.shortValue(); + short decShort = dec.shortValue(); + if (isShortExpected != decIsShort) { + // System.out.println("TEST_INT_VALUE isShort mismatch"); + // System.out.println("TEST_INT_VALUE bigDecimalFloor " + bigDecimalFloor); + // System.out.println("TEST_INT_VALUE isShortExpected " + isShortExpected); + // System.out.println("TEST_INT_VALUE decIsShort " + decIsShort); + // System.out.println("TEST_INT_VALUE oldDecShort " + oldDecShort); + // System.out.println("TEST_INT_VALUE decShort " + decShort); + Assert.assertTrue(false); + } + + if (decIsShort) { + if (oldDecShort != decShort) { + // System.out.println("TEST_INT_VALUE shortValue mismatch"); + // System.out.println("TEST_INT_VALUE oldDecShort " + oldDecShort); + // System.out.println("TEST_INT_VALUE decShort " + decShort); + Assert.assertTrue(false); + } + } else { + // Not sure whether corrupted values will be equal... + // System.out.println("TEST_INT_VALUE corrupt oldDec " + oldDec); + // System.out.println("TEST_INT_VALUE corrupt dec " + dec); + // System.out.println("TEST_INT_VALUE corrupt oldDecShort " + oldDecShort); + // System.out.println("TEST_INT_VALUE corrupt decShort " + decShort); + + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomByteValue() { + Random r = new Random(9292); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomByteValue(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomByteValue(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomByteValue(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestByteValue(bigDecimal); + } + } + + @Test + public void testByteValueSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestByteValue(bigDecimal); + } + } + + private void doTestByteValue(BigDecimal bigDecimal) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_INT_VALUE oldDec " + oldDec); + // System.out.println("TEST_INT_VALUE dec " + dec); + + BigDecimal bigDecimalOldDec = oldDec.bigDecimalValue(); + BigDecimal bigDecimalDec = dec.bigDecimalValue(); + Assert.assertEquals(bigDecimalOldDec, bigDecimalDec); + + BigDecimal bigDecimalFloor = bigDecimalDec.setScale(0, BigDecimal.ROUND_DOWN); + byte byteValueBigDecimalFloor = bigDecimalFloor.byteValue(); + boolean isByteExpected = + bigDecimalFloor.equals(bigDecimalDec.valueOf(byteValueBigDecimalFloor)); + + boolean decIsByte = dec.isByte(); + byte oldDecByte = oldDec.byteValue(); + byte decByte = dec.byteValue(); + if (isByteExpected != decIsByte) { + // System.out.println("TEST_INT_VALUE isByte mismatch"); + // System.out.println("TEST_INT_VALUE bigDecimalFloor " + bigDecimalFloor); + // System.out.println("TEST_INT_VALUE isByteExpected " + isByteExpected); + // System.out.println("TEST_INT_VALUE decIsByte " + decIsByte); + // System.out.println("TEST_INT_VALUE oldDecByte " + oldDecByte); + // System.out.println("TEST_INT_VALUE decByte " + decByte); + Assert.assertTrue(false); + } + + if (decIsByte) { + if (oldDecByte != decByte) { + // System.out.println("TEST_INT_VALUE byteValue mismatch"); + // System.out.println("TEST_INT_VALUE oldDecByte " + oldDecByte); + // System.out.println("TEST_INT_VALUE decByte " + decByte); + Assert.assertTrue(false); + } + } else { + // Not sure whether corrupted values will be equal... + // System.out.println("TEST_INT_VALUE corrupt oldDec " + oldDec); + // System.out.println("TEST_INT_VALUE corrupt dec " + dec); + // System.out.println("TEST_INT_VALUE corrupt oldDecByte " + oldDecByte); + // System.out.println("TEST_INT_VALUE corrupt decByte " + decByte); + + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomTimestamp() { + Random r = new Random(5476); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomTimestamp(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomTimestamp(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomTimestamp(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestTimestamp(bigDecimal); + } + } + + @Test + public void testTimestampSpecial() { + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestTimestamp(bigDecimal); + } + } + + private void doTestTimestamp(BigDecimal bigDecimal) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_TIMESTAMP oldDec " + oldDec); + // System.out.println("TEST_TIMESTAMP dec " + dec); + + Timestamp timestampOldDec = TimestampUtils.decimalToTimestamp(oldDec); + Timestamp timestampDec = TimestampUtils.decimalToTimestamp(dec); + if (timestampOldDec == null) { + Assert.assertTrue(timestampDec == null); + return; + } + if (timestampDec == null) { + // System.out.println("TEST_TIMESTAMP timestampDec is NULL"); + return; + } + // System.out.println("TEST_TIMESTAMP timestampOldDec " + timestampOldDec); + // System.out.println("TEST_TIMESTAMP timestampDec " + timestampDec); + Assert.assertEquals(timestampOldDec, timestampDec); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomSerializationUtilsRead() + throws IOException { + doTestRandomSerializationUtilsRead(standardAlphabet); + } + + @Test + public void testRandomSerializationUtilsReadSparse() + throws IOException { + for (String digitAlphabet : sparseAlphabets) { + doTestRandomSerializationUtilsRead(digitAlphabet); + } + } + + private void doTestRandomSerializationUtilsRead(String digitAlphabet) + throws IOException { + + Random r = new Random(2389); + for (int i = 0; i < POUND_FACTOR; i++) { + BigInteger bigInteger = randHiveBigInteger(r, digitAlphabet); + + doTestSerializationUtilsRead(r, bigInteger); + } + } + + @Test + public void testSerializationUtilsReadSpecial() + throws IOException { + Random r = new Random(9923); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestSerializationUtilsRead(r, bigDecimal.unscaledValue()); + } + } + + private void doTestSerializationUtilsRead(Random r, BigInteger bigInteger) + throws IOException { + + // System.out.println("TEST_SERIALIZATION_UTILS_READ bigInteger " + bigInteger); + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigInteger); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigInteger); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_SERIALIZATION_UTILS_READ oldDec " + oldDec); + // System.out.println("TEST_SERIALIZATION_UTILS_READ dec " + dec); + + Assert.assertEquals(bigInteger, oldDec.unscaledValue()); + Assert.assertEquals(bigInteger, dec.unscaledValue()); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + SerializationUtils.writeBigInteger(outputStream, bigInteger); + byte[] bytes = outputStream.toByteArray(); + + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); + BigInteger deserializedBigInteger = + SerializationUtils.readBigInteger(byteArrayInputStream); + + // Verify SerializationUtils first. + Assert.assertEquals(bigInteger, deserializedBigInteger); + + // Now HiveDecimal + byte[] scratchBytes = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ]; + + byteArrayInputStream = new ByteArrayInputStream(bytes); + HiveDecimal resultDec = + dec.serializationUtilsRead( + byteArrayInputStream, dec.scale(), + scratchBytes); + assertTrue(resultDec != null); + resultDec.validate(); + + Assert.assertEquals(dec.toString(), resultDec.toString()); + + //---------------------------------------------------------------------------------------------- + + // Add scale. + + int scale = 0 + r.nextInt(38 + 1); + BigDecimal bigDecimal = new BigDecimal(bigInteger, scale); + + oldDec = OldHiveDecimal.create(bigDecimal); + dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_SERIALIZATION_UTILS_READ with scale oldDec " + oldDec); + // System.out.println("TEST_SERIALIZATION_UTILS_READ with scale dec " + dec); + + outputStream = new ByteArrayOutputStream(); + SerializationUtils.writeBigInteger(outputStream, dec.unscaledValue()); + bytes = outputStream.toByteArray(); + + // Now HiveDecimal + byteArrayInputStream = new ByteArrayInputStream(bytes); + resultDec = + dec.serializationUtilsRead( + byteArrayInputStream, dec.scale(), + scratchBytes); + assertTrue(resultDec != null); + resultDec.validate(); + + Assert.assertEquals(dec.toString(), resultDec.toString()); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomSerializationUtilsWrite() + throws IOException { + doTestRandomSerializationUtilsWrite(standardAlphabet, false); + } + + @Test + public void testRandomSerializationUtilsWriteFractionsOnly() + throws IOException { + doTestRandomSerializationUtilsWrite(standardAlphabet, true); + } + + @Test + public void testRandomSerializationUtilsWriteSparse() + throws IOException { + for (String digitAlphabet : sparseAlphabets) { + doTestRandomSerializationUtilsWrite(digitAlphabet, false); + } + } + + private void doTestRandomSerializationUtilsWrite(String digitAlphabet, boolean fractionsOnly) + throws IOException { + + Random r = new Random(823); + for (int i = 0; i < POUND_FACTOR; i++) { + BigInteger bigInteger = randHiveBigInteger(r, digitAlphabet); + + doTestSerializationUtilsWrite(r, bigInteger); + } + } + + @Test + public void testSerializationUtilsWriteSpecial() + throws IOException { + Random r = new Random(998737); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestSerializationUtilsWrite(r, bigDecimal.unscaledValue()); + } + } + + private void doTestSerializationUtilsWrite(Random r, BigInteger bigInteger) + throws IOException { + + // System.out.println("TEST_SERIALIZATION_UTILS_WRITE bigInteger " + bigInteger); + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigInteger); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigInteger); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_SERIALIZATION_UTILS_WRITE oldDec " + oldDec); + // System.out.println("TEST_SERIALIZATION_UTILS_WRITE dec " + dec); + + Assert.assertEquals(bigInteger, oldDec.unscaledValue()); + Assert.assertEquals(bigInteger, dec.unscaledValue()); + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + SerializationUtils.writeBigInteger(outputStream, bigInteger); + byte[] bytes = outputStream.toByteArray(); + + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); + BigInteger deserializedBigInteger = + SerializationUtils.readBigInteger(byteArrayInputStream); + + // Verify SerializationUtils first. + Assert.assertEquals(bigInteger, deserializedBigInteger); + + ByteArrayOutputStream decOutputStream = new ByteArrayOutputStream(); + + long[] scratchLongs = new long[HiveDecimal.SCRATCH_LONGS_LEN]; + + boolean successful = + dec.serializationUtilsWrite( + decOutputStream, scratchLongs); + Assert.assertTrue(successful); + byte[] decBytes = decOutputStream.toByteArray(); + + if (!StringExpr.equal(bytes, 0, bytes.length, decBytes, 0, decBytes.length)) { + // Tailing zeroes difference ok... + // System.out.println("TEST_SERIALIZATION_UTILS_WRITE streams not equal"); + // System.out.println("TEST_SERIALIZATION_UTILS_WRITE bytes " + displayBytes(bytes, 0, bytes.length)); + // System.out.println("TEST_SERIALIZATION_UTILS_WRITE decBytes " + displayBytes(decBytes, 0, decBytes.length)); + } + + ByteArrayInputStream decByteArrayInputStream = new ByteArrayInputStream(decBytes); + BigInteger decDeserializedBigInteger = + SerializationUtils.readBigInteger(decByteArrayInputStream); + + Assert.assertEquals(bigInteger, decDeserializedBigInteger); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomBigIntegerBytes() { + Random r = new Random(1050); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomBigIntegerBytes(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomBigIntegerBytes(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomBigIntegerBytes(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestBigIntegerBytes(bigDecimal); + } + } + + @Test + public void testBigIntegerBytesSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + int negativeScale = -(0 + r.nextInt(38 + 1)); + bigDecimal = bigDecimal.setScale(negativeScale, BigDecimal.ROUND_HALF_UP); + doTestBigIntegerBytes(bigDecimal); + } + } + + private void doTestBigIntegerBytes(BigDecimal bigDecimal) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + if (oldDec == null) { + return; + } + // System.out.println("TEST_BIG_INTEGER_BYTES oldDec " + oldDec.toString()); + HiveDecimal dec = HiveDecimal.create(bigDecimal); + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_BIG_INTEGER_BYTES dec " + dec); + + //--------------------------------------------------- + BigInteger oldBigInteger = oldDec.unscaledValue(); + int oldScale = oldDec.scale(); + // System.out.println("TEST_BIG_INTEGER_BYTES oldBigInteger " + oldBigInteger.toString()); + // System.out.println("TEST_BIG_INTEGER_BYTES oldScale " + oldScale); + //--------------------------------------------------- + + BigInteger bigInteger = dec.unscaledValue(); + int scale = dec.scale(); + // System.out.println("TEST_BIG_INTEGER_BYTES bigInteger " + bigInteger.toString()); + // System.out.println("TEST_BIG_INTEGER_BYTES scale " + scale); + + long[] scratchLongs = new long[HiveDecimal.SCRATCH_LONGS_LEN]; + byte[] scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; + + int which = 0; + try { + which = 1; + int byteLength = dec.bigIntegerBytes(scratchLongs, scratchBuffer); + byte[] bytes = null; + if (byteLength == 0) { + // System.out.println("TEST_FAST_BIG_INTEGER_BYTES bigIntegerBytes failure"); + } else { + bytes = Arrays.copyOf(scratchBuffer, byteLength); + // System.out.println("TEST_FAST_BIG_INTEGER_BYTES bigIntegerBytes " + displayBytes(bytes, 0, bytes.length)); + } + + which = 2; + byte[] bytesExpected = bigInteger.toByteArray(); + String bytesExpectedString = displayBytes(bytesExpected, 0, bytesExpected.length); + // System.out.println("TEST_FAST_BIG_INTEGER_BYTES bytesExpected " + bytesExpectedString); + + // System.out.println("TEST_FAST_BIG_INTEGER_BYTES dec.unscaledValue() " + dec.unscaledValue()); + // System.out.println("TEST_FAST_BIG_INTEGER_BYTES bigInteger " + bigInteger.toString()); + + if (!StringExpr.equal(bytes, 0, bytes.length, bytesExpected, 0, bytesExpected.length)) { + // System.out.println("TEST_FAST_BIG_INTEGER_BYTES bytes not equal"); + if (bytesExpectedString.substring(0,4).equals("\\254")) { + // System.out.println("TEST_FAST_BIG_INTEGER_BYTES strange \\254 difference"); + } else { + assertTrue(false); + } + } + + which = 3; + HiveDecimal createFromBigIntegerBytesDec = + HiveDecimal.createFromBigIntegerBytesAndScale( + bytes, 0, bytes.length, scale); + if (createFromBigIntegerBytesDec == null) { + // System.out.println("TEST_FAST_BIG_INTEGER_BYTES createFromBigIntegerBytesDec NULL"); + } else { + // System.out.println("TEST_FAST_BIG_INTEGER_BYTES createFromBigIntegerBytesDec " + createFromBigIntegerBytesDec.toString()); + // System.out.println("TEST_FAST_BIG_INTEGER_BYTES dec " + dec.toString()); + // System.out.println("TEST_FAST_BIG_INTEGER_BYTES createFromBigIntegerBytesDec equals " + + // createFromBigIntegerBytesDec.equals(dec)); + if (!createFromBigIntegerBytesDec.equals(dec)) { + } + } + + } catch (Exception e) { + // System.out.println("TEST_FAST_BIG_INTEGER_BYTES " + e + " which " + which); + assertTrue(false); + } + // System.out.println("TEST_BIG_INTEGER_BYTES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ END ~~~~~~~~~~~~~~~~~"); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomToFormatString() { + Random r = new Random(1051); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomToFormatString(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomToFormatString(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomToFormatString(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestToFormatString(r, bigDecimal); + } + } + + @Test + public void testToFormatStringSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestToFormatString(r, bigDecimal); + } + } + + private void doTestToFormatString(Random r, BigDecimal bigDecimal) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec; + if (oldDec == null) { + dec = HiveDecimal.create(bigDecimal); + if (dec != null) { + // System.out.println("TEST_TO_FORMAT_STRING oldDec result null but dec " + dec); + Assert.assertTrue(false); + } + return; + } else { + dec = HiveDecimal.create(bigDecimal); + if (dec == null) { + // System.out.println("TEST_TO_FORMAT_STRING create not equal oldDec have result but dec is null " + oldDec.toString() + " scale " + oldDec.scale()); + if (isTenPowerBug(oldDec.toString())) { + // System.out.println("TEST_TO_FORMAT_STRING OldHiveDecimal 10^38 bug!!!"); + return; + } + Assert.assertTrue(false); + } + } + dec.validate(); + + // System.out.println("TEST_TO_FORMAT_STRING oldDec " + oldDec); + // System.out.println("TEST_TO_FORMAT_STRING dec " + dec); + + // UNDONE: Does this random range need to go as high as 38? + int formatScale = 0 + r.nextInt(38); + // System.out.println("TEST_TO_FORMAT_STRING formatScale " + formatScale); + + String oldDecFormatString = oldDec.toFormatString(formatScale); + String decFormatString; + if (oldDecFormatString == null) { + decFormatString = dec.toFormatString(formatScale); + if (decFormatString != null) { + // System.out.println("TEST_TO_FORMAT_STRING oldDecFormatString result null but decFormatString " + decFormatString); + Assert.assertTrue(false); + } + return; + } else { + decFormatString = dec.toFormatString(formatScale); + if (decFormatString == null) { + // System.out.println("TEST_TO_FORMAT_STRING toFormatString not equal oldDecFormatString have result but decFormatString is null " + oldDecFormatString); + if (isTenPowerBug(oldDecFormatString)) { + // System.out.println("TEST_TO_FORMAT_STRING OldHiveDecimal 10^38 bug!!!"); + return; + } + Assert.assertTrue(false); + } + } + + if (!oldDecFormatString.equals(decFormatString)) { + // System.out.println("TEST_TO_FORMAT_STRING oldDecFormatString " + oldDecFormatString); + // System.out.println("TEST_TO_FORMAT_STRING decFormatString " + decFormatString); + // System.out.println("TEST_TO_FORMAT_STRING not equal"); + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomScaleByPowerOfTen() { + Random r = new Random(1052); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomScaleByPowerOfTen(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomScaleByPowerOfTen(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomScaleByPowerOfTen(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestScaleByPowerOfTen(r, bigDecimal); + } + } + + @Test + public void testScaleByPowerOfTenSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestScaleByPowerOfTen(r, bigDecimal); + } + } + + private void doTestScaleByPowerOfTen(Random r, BigDecimal bigDecimal) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_SCALE_BY_POWER_OF_TEN *********************************************"); + // System.out.println("TEST_SCALE_BY_POWER_OF_TEN oldDec " + oldDec); + // System.out.println("TEST_SCALE_BY_POWER_OF_TEN dec " + dec); + + OldHiveDecimal oldPowerDec; + HiveDecimal powerDec; + + for (int power = -(2 * HiveDecimal.MAX_SCALE + 1); + power <= 2 * HiveDecimal.MAX_SCALE + 1; + power++) { + + // System.out.println("TEST_SCALE_BY_POWER_OF_TEN >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"); + // System.out.println("TEST_SCALE_BY_POWER_OF_TEN power " + power); + + oldPowerDec = oldDec.scaleByPowerOfTen(power); + boolean isEqual; + if (oldPowerDec == null) { + powerDec = dec.scaleByPowerOfTen(power); + if (powerDec != null) { + // System.out.println("TEST_SCALE_BY_POWER_OF_TEN OldHiveDecimal result null but powerDec " + powerDec); + Assert.assertTrue(false); + } + return; + } else { + String oldPowerDecString = oldPowerDec.toString(); + powerDec = dec.scaleByPowerOfTen(power); + if (powerDec == null) { + // System.out.println("TEST_SCALE_BY_POWER_OF_TEN not equal oldPowerDec has result but powerDec is null " + oldPowerDec.toString() + " scale " + oldPowerDec.scale()); + if (isTenPowerBug(oldPowerDec.toString())) { + // System.out.println("TEST_SCALE_BY_POWER_OF_TEN OldHiveDecimal 10^38 bug!!!"); + return; + } + // OldHiveDecimal.scaleByPowerOfTen is broken. + Assert.assertTrue(false); + continue; + } + powerDec.validate(); + String powerDecString = powerDec.toString(); + // System.out.println("TEST_SCALE_BY_POWER_OF_TEN oldPowerDecString " + oldPowerDecString + " scale " + oldPowerDec.scale()); + // System.out.println("TEST_SCALE_BY_POWER_OF_TEN powerDecString " + powerDecString); + isEqual = oldPowerDecString.equals(powerDecString); + if (!isEqual) { + // System.out.println("TEST_SCALE_BY_POWER_OF_TEN not equal"); + // OldHiveDecimal.scaleByPowerOfTen is broken. + if (oldPowerDecString.equals("0.00000000000000000000000000000000000001") || + oldPowerDecString.equals("-0.00000000000000000000000000000000000001")) { + continue; + } + Assert.assertTrue(false); + } else { + // System.out.println("TEST_SCALE_BY_POWER_OF_TEN equal"); + } + } + // System.out.println("TEST_SCALE_BY_POWER_OF_TEN <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"); + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomWriteReadFields() throws Exception { + Random r = new Random(1052); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomWriteReadFields(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomWriteReadFields(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomWriteReadFields(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) throws Exception { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestWriteReadFields(r, bigDecimal); + } + } + + @Test + public void testWriteReadFieldsSpecial() throws Exception { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestWriteReadFields(r, bigDecimal); + } + } + + private void doTestWriteReadFields(Random r, BigDecimal bigDecimal) throws IOException { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_WRITE_READ_FIELDS *********************************************"); + // System.out.println("TEST_WRITE_READ_FIELDS oldDec " + oldDec); + // System.out.println("TEST_WRITE_READ_FIELDS dec " + dec); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(baos); + + HiveDecimalWritable decimalWritableOut = new HiveDecimalWritable(dec); + decimalWritableOut.write(out); + + byte[] valueBytes = baos.toByteArray(); + + ByteArrayInputStream bais = new ByteArrayInputStream(valueBytes); + DataInputStream in = new DataInputStream(bais); + HiveDecimalWritable decimalWritableIn = new HiveDecimalWritable(); + decimalWritableIn.readFields(in); + + Assert.assertEquals(dec, decimalWritableIn.getHiveDecimal()); + + // System.out.println("TEST_WRITE_READ_FIELDS<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomBigIntegerBytesScaled() throws Exception { + Random r = new Random(1052); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomBigIntegerBytesScaled(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomBigIntegerBytesScaled(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomBigIntegerBytesScaled(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) throws Exception { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestBigIntegerBytesScaled(r, bigDecimal); + } + } + + @Test + public void testBigIntegerBytesScaledSpecial() throws Exception { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestBigIntegerBytesScaled(r, bigDecimal); + } + } + + private void doTestBigIntegerBytesScaled(Random r, BigDecimal bigDecimal) throws IOException { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + int scale = oldDec.scale(); + int newScale; + if (scale == HiveDecimal.MAX_SCALE) { + newScale = scale; + } else { + newScale = scale + r.nextInt(HiveDecimal.MAX_SCALE - scale); + } + + OldHiveDecimal oldDecScaled = oldDec.setScale(newScale); + OldHiveDecimalWritable oldDecScaledWritable = new OldHiveDecimalWritable(oldDecScaled); + + byte[] bytesExpected = oldDecScaledWritable.getInternalStorage(); + + byte[] bytes = dec.bigIntegerBytesScaled(newScale); + + // System.out.println("TEST_BIG_INTEGER_BYTES_SCALED *********************************************"); + // System.out.println("TEST_BIG_INTEGER_BYTES_SCALED oldDec " + oldDec); + // System.out.println("TEST_BIG_INTEGER_BYTES_SCALED dec " + dec); + + // System.out.println("TEST_BIG_INTEGER_BYTES_SCALED check serializations"); + // System.out.println("TEST_BIG_INTEGER_BYTES_SCALED bytes " + displayBytes(bytes, 0, bytes.length)); + if (!StringExpr.equal(bytes, 0, bytes.length, bytesExpected, 0, bytesExpected.length)) { + // System.out.println("TEST_BIG_INTEGER_BYTES_SCALED serializations not equal"); + // System.out.println("TEST_BIG_INTEGER_BYTES_SCALED bytesExpected " + displayBytes(bytesExpected, 0, bytesExpected.length)); + Assert.assertTrue(false); + } + + OldHiveDecimalWritable oldDecWritableRetrieve = new OldHiveDecimalWritable(bytesExpected, newScale); + OldHiveDecimal oldDecRetrieve = oldDecWritableRetrieve.getHiveDecimal(); + Assert.assertTrue(oldDecRetrieve != null); + // System.out.println("TEST_BIG_INTEGER_BYTES_SCALED oldDecRetrieve " + oldDecRetrieve); + + HiveDecimal decRetrieve = HiveDecimal.createFromBigIntegerBytesAndScale(bytes, newScale); + Assert.assertTrue(decRetrieve != null); + + Assert.assertEquals(oldDecRetrieve.toString(), decRetrieve.toString()); + + // System.out.println("TEST_BIG_INTEGER_BYTES_SCALED<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomRoundFloor() { + Random r = new Random(1052); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomRound(r, standardAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_FLOOR); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomRound(r, sparseAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_FLOOR); + } + } + } + + private void doTestRandomRound(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor, int roundingMode) { + + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal bigDecimal = randHiveBigDecimal(r, digitAlphabet, bigDecimalFlavor); + + doTestRound(r, bigDecimal, roundingMode); + } + } + + @Test + public void testRoundFloorSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestRound(r, bigDecimal, HiveDecimal.ROUND_FLOOR); + } + } + + // Used by all flavors. + private void doTestRound(Random r, BigDecimal bigDecimal, int roundingMode) { + + // Temporarily.... + bigDecimal = bigDecimal.abs(); + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + // System.out.println("TEST_SET_SCALE_ROUND oldDec " + oldDec); + // System.out.println("TEST_SET_SCALE_ROUND dec " + dec); + + OldHiveDecimal oldScaledDec; + HiveDecimal scaledDec; + + for (int newScale = -(2 * HiveDecimal.MAX_SCALE + 1); + newScale <= 2 * HiveDecimal.MAX_SCALE + 1; + newScale++) { + + // System.out.println("TEST_SET_SCALE_ROUND newScale " + newScale); + + oldScaledDec = oldDec.setScale(newScale, roundingMode); + boolean isEqual; + if (oldScaledDec == null) { + scaledDec = dec.round(newScale, roundingMode); + if (scaledDec != null) { + // System.out.println("TEST_SET_SCALE_ROUND OldHiveDecimal result null but scaledDec " + scaledDec); + Assert.assertTrue(false); + } + return; + } else { + scaledDec = dec.round(newScale, roundingMode); + if (scaledDec == null) { + // System.out.println("TEST_SET_SCALE_ROUND not equal oldScaledDec has result but scaledDec is null " + oldScaledDec.toString() + " scale " + oldScaledDec.scale()); + if (isTenPowerBug(oldScaledDec.toString())) { + // System.out.println("TEST_SET_SCALE_ROUND OldHiveDecimal 10^38 bug!!!"); + continue; + } + /* + if (oldScaledDec.toString().equals("0")) { + // System.out.println("TEST_SET_SCALE_ROUND Assume this is where OldHiveDecimal treats overflow as 0 instead of null"); + continue; + } + */ + Assert.assertTrue(false); + } + scaledDec.validate(); + // System.out.println("TEST_SET_SCALE_ROUND oldScaledDec " + oldScaledDec + " scale " + oldScaledDec.scale()); + // System.out.println("TEST_SET_SCALE_ROUND scaledDec " + scaledDec); + isEqual = oldScaledDec.toString().equals(scaledDec.toString()); + if (!isEqual) { + // System.out.println("TEST_SET_SCALE_ROUND not equal"); + Assert.assertTrue(false); + } else { + // System.out.println("TEST_SET_SCALE_ROUND equal"); + } + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomRoundCeiling() { + Random r = new Random(1053); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomRound(r, standardAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_CEILING); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomRound(r, sparseAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_CEILING); + } + } + } + + @Test + public void testRoundCeilingSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestRound(r, bigDecimal, HiveDecimal.ROUND_CEILING); + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomRoundHalfUp() { + Random r = new Random(1053); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomRound(r, standardAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_HALF_UP); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomRound(r, sparseAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_HALF_UP); + } + } + } + + @Test + public void testRoundHalfUpSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestRound(r, bigDecimal, HiveDecimal.ROUND_HALF_UP); + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomRoundHalfEven() { + Random r = new Random(1053); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomRound(r, standardAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_HALF_EVEN); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomRound(r, sparseAlphabet, bigDecimalFlavor, HiveDecimal.ROUND_HALF_EVEN); + } + } + } + + @Test + public void testRoundHalfEvenSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + doTestRound(r, bigDecimal, HiveDecimal.ROUND_HALF_EVEN); + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomCompareTo() { + Random r = new Random(1054); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomCompareTo(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomCompareTo(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomCompareTo(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (BigDecimalPairFlavor bigDecimalPairFlavor : BigDecimalPairFlavor.values()) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal[] pair = randHiveBigDecimalPair(r, digitAlphabet, bigDecimalFlavor, bigDecimalPairFlavor); + + doTestCompareTo(r, pair[0], pair[1]); + } + } + } + + @Test + public void testCompareToSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + for (BigDecimal bigDecimal2 : specialBigDecimals) { + doTestCompareTo(r, bigDecimal, bigDecimal2); + } + } + } + + private void doTestCompareTo(Random r, BigDecimal bigDecimal, BigDecimal bigDecimal2) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + OldHiveDecimal oldDec2 = OldHiveDecimal.create(bigDecimal2); + if (oldDec2 != null && isTenPowerBug(oldDec2.toString())) { + return; + } + HiveDecimal dec2 = HiveDecimal.create(bigDecimal2); + if (oldDec2 == null) { + assertTrue(dec2 == null); + return; + } + assertTrue(dec2 != null); + dec.validate(); + + // System.out.println("TEST_COMPARE_TO oldDec " + oldDec); + // System.out.println("TEST_COMPARE_TO dec " + dec); + // System.out.println("TEST_COMPARE_TO oldDec2 " + oldDec2); + // System.out.println("TEST_COMPARE_TO dec2 " + dec2); + + // Verify. + Assert.assertEquals(oldDec.toString(), dec.toString()); + Assert.assertEquals(oldDec2.toString(), dec2.toString()); + + int oldCompareTo; + int compareTo; + + // Same object. + oldCompareTo = oldDec.compareTo(oldDec); + Assert.assertEquals(0, oldCompareTo); + compareTo = dec.compareTo(dec); + Assert.assertEquals(0, compareTo); + + // Two objects. + oldCompareTo = oldDec.compareTo(oldDec2); + compareTo = dec.compareTo(dec2); + Assert.assertEquals(oldCompareTo, compareTo); + + int oldCompareToReverse = oldDec2.compareTo(oldDec); + int compareToReverse = dec2.compareTo(dec); + Assert.assertEquals(oldCompareToReverse, compareToReverse); + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomAdd() { + Random r = new Random(1055); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomAdd(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomAdd(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomAdd(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (BigDecimalPairFlavor bigDecimalPairFlavor : BigDecimalPairFlavor.values()) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal[] pair = randHiveBigDecimalPair(r, digitAlphabet, bigDecimalFlavor, bigDecimalPairFlavor); + + doTestAdd(r, pair[0], pair[1]); + } + } + } + + @Test + public void testAddSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + for (BigDecimal bigDecimal2 : specialBigDecimals) { + doTestAdd(r, bigDecimal, bigDecimal2); + } + } + } + + private void doTestAdd(Random r, BigDecimal bigDecimal, BigDecimal bigDecimal2) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + OldHiveDecimal oldDec2 = OldHiveDecimal.create(bigDecimal2); + if (oldDec2 != null && isTenPowerBug(oldDec2.toString())) { + return; + } + HiveDecimal dec2 = HiveDecimal.create(bigDecimal2); + if (oldDec2 == null) { + assertTrue(dec2 == null); + return; + } + assertTrue(dec2 != null); + dec.validate(); + + // System.out.println("DO_TEST_ADD oldDec " + oldDec); + // System.out.println("DO_TEST_ADD dec " + dec); + // System.out.println("DO_TEST_ADD oldDec2 " + oldDec2); + // System.out.println("DO_TEST_ADD dec2 " + dec2); + + // Verify. + Assert.assertEquals(oldDec.toString(), dec.toString()); + Assert.assertEquals(oldDec2.toString(), dec2.toString()); + + // Add to self. + OldHiveDecimal oldAddDec; + HiveDecimal addDec; + + oldAddDec = oldDec.add(oldDec); + boolean isEqual; + if (oldAddDec == null) { + addDec = dec.add(dec); + assertTrue(addDec == null); + return; + } else { + // System.out.println("DO_TEST_ADD oldAddDec " + oldAddDec + " scale " + oldAddDec.scale()); + addDec = dec.add(dec); + if (addDec == null) { + // System.out.println("DO_TEST_ADD not equal oldAddDec has result but addDec is null " + oldAddDec.toString()); + if (isTenPowerBug(oldAddDec.toString())) { + // System.out.println("DO_TEST_ADD_POSITIVE OldHiveDecimal 10^38 bug!!!"); + return; + } + Assert.assertTrue(false); + } + addDec.validate(); + // System.out.println("DO_TEST_ADD addDec " + addDec); + isEqual = oldAddDec.toString().equals(addDec.toString()); + if (!isEqual) { + // System.out.println("DO_TEST_ADD not equal"); + Assert.assertTrue(false); + } else { + // System.out.println("DO_TEST_ADD equal"); + } + } + + // Add two decimals. + // System.out.println("DO_TEST_ADD oldDec " + oldDec); + // System.out.println("DO_TEST_ADD dec " + dec); + // System.out.println("DO_TEST_ADD oldDec2 " + oldDec2); + // System.out.println("DO_TEST_ADD dec2 " + dec2); + oldAddDec = oldDec.add(oldDec2); + if (oldAddDec == null) { + addDec = dec.add(dec2); + assertTrue(addDec == null); + return; + } else { + // System.out.println("DO_TEST_ADD oldAddDec " + oldAddDec + " scale " + oldAddDec.scale()); + addDec = dec.add(dec2); + if (addDec == null) { + // System.out.println("DO_TEST_ADD not equal oldAddDec has result but addDec is null " + oldAddDec.toString()); + if (isTenPowerBug(oldAddDec.toString())) { + // System.out.println("DO_TEST_ADD_POSITIVE OldHiveDecimal 10^38 bug!!!"); + return; + } + Assert.assertTrue(false); + } + addDec.validate(); + // System.out.println("DO_TEST_ADD addDec " + addDec); + isEqual = oldAddDec.toString().equals(addDec.toString()); + if (!isEqual) { + // System.out.println("DO_TEST_ADD not equal"); + Assert.assertTrue(false); + } else { + // System.out.println("DO_TEST_ADD equal"); + } + } + + // Add negative self. + + oldAddDec = oldDec.add(oldDec.negate()); + if (oldAddDec == null) { + addDec = dec.add(dec.negate()); + assertTrue(addDec == null); + return; + } else { + // System.out.println("DO_TEST_ADD oldAddDec " + oldAddDec + " scale " + oldAddDec.scale()); + addDec = dec.add(dec.negate()); + if (addDec == null) { + // System.out.println("DO_TEST_ADD not equal oldAddDec has result but addDec is null " + oldAddDec.toString()); + if (isTenPowerBug(oldAddDec.toString())) { + // System.out.println("DO_TEST_ADD_POSITIVE OldHiveDecimal 10^38 bug!!!"); + return; + } + Assert.assertTrue(false); + } + addDec.validate(); + // System.out.println("DO_TEST_ADD addDec " + addDec); + isEqual = oldAddDec.toString().equals(addDec.toString()); + if (!isEqual) { + // System.out.println("DO_TEST_ADD not equal"); + Assert.assertTrue(false); + } else { + // System.out.println("DO_TEST_ADD equal"); + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomSubtract() { + Random r = new Random(1055); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomSubtract(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomSubtract(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomSubtract(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (BigDecimalPairFlavor bigDecimalPairFlavor : BigDecimalPairFlavor.values()) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal[] pair = randHiveBigDecimalPair(r, digitAlphabet, bigDecimalFlavor, bigDecimalPairFlavor); + + doTestSubtract(r, pair[0], pair[1]); + } + } + } + + @Test + public void testSubtractSpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + for (BigDecimal bigDecimal2 : specialBigDecimals) { + doTestSubtract(r, bigDecimal, bigDecimal2); + } + } + } + + private void doTestSubtract(Random r, BigDecimal bigDecimal, BigDecimal bigDecimal2) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + OldHiveDecimal oldDec2 = OldHiveDecimal.create(bigDecimal2); + if (oldDec2 != null && isTenPowerBug(oldDec2.toString())) { + return; + } + HiveDecimal dec2 = HiveDecimal.create(bigDecimal2); + if (oldDec2 == null) { + assertTrue(dec2 == null); + return; + } + assertTrue(dec2 != null); + dec.validate(); + + // System.out.println("DO_TEST_SUBTRACT oldDec " + oldDec); + // System.out.println("DO_TEST_SUBTRACT dec " + dec); + // System.out.println("DO_TEST_SUBTRACT oldDec2 " + oldDec2); + // System.out.println("DO_TEST_SUBTRACT dec2 " + dec2); + + // Verify. + Assert.assertEquals(oldDec.toString(), dec.toString()); + Assert.assertEquals(oldDec2.toString(), dec2.toString()); + + // Subtract from self. + OldHiveDecimal oldSubtractDec; + HiveDecimal subtractDec; + + oldSubtractDec = oldDec.subtract(oldDec); + Assert.assertEquals(0, oldSubtractDec.signum()); + subtractDec = dec.subtract(dec); + Assert.assertEquals(0, subtractDec.signum()); + + // System.out.println("DO_TEST_SUBTRACT oldDec " + oldDec); + // System.out.println("DO_TEST_SUBTRACT dec " + dec); + // System.out.println("DO_TEST_SUBTRACT oldDec2 " + oldDec2); + // System.out.println("DO_TEST_SUBTRACT dec2 " + dec2); + boolean isEqual; + oldSubtractDec = oldDec.subtract(oldDec2); + if (oldSubtractDec == null) { + subtractDec = dec.subtract(dec2); + assertTrue(subtractDec == null); + return; + } else { + // System.out.println("DO_TEST_SUBTRACT oldSubtractDec " + oldSubtractDec + " scale " + oldSubtractDec.scale()); + subtractDec = dec.subtract(dec2); + if (subtractDec == null) { + // System.out.println("DO_TEST_SUBTRACT not equal oldSubtractDec has result but subtractDec is null " + oldSubtractDec.toString()); + if (isTenPowerBug(oldSubtractDec.toString())) { + // System.out.println("DO_TEST_SUBTRACT OldHiveDecimal 10^38 bug!!!"); + return; + } + Assert.assertTrue(false); + } + subtractDec.validate(); + // System.out.println("DO_TEST_SUBTRACT subtractDec " + subtractDec); + isEqual = oldSubtractDec.toString().equals(subtractDec.toString()); + if (!isEqual) { + // System.out.println("DO_TEST_SUBTRACT not equal"); + Assert.assertTrue(false); + } else { + // System.out.println("DO_TEST_SUBTRACT equal"); + } + } + } + + //------------------------------------------------------------------------------------------------ + + @Test + public void testRandomMultiply() { + Random r = new Random(1056); + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + doTestRandomMultiply(r, standardAlphabet, bigDecimalFlavor); + } + for (BigDecimalFlavor bigDecimalFlavor : BigDecimalFlavor.values()) { + for (String sparseAlphabet : sparseAlphabets) { + doTestRandomMultiply(r, sparseAlphabet, bigDecimalFlavor); + } + } + } + + private void doTestRandomMultiply(Random r, String digitAlphabet, BigDecimalFlavor bigDecimalFlavor) { + + for (BigDecimalPairFlavor bigDecimalPairFlavor : BigDecimalPairFlavor.values()) { + for (int i = 0; i < POUND_FACTOR; i++) { + BigDecimal[] pair = randHiveBigDecimalPair(r, digitAlphabet, bigDecimalFlavor, bigDecimalPairFlavor); + + doTestMultiply(r, pair[0], pair[1]); + } + } + } + + @Test + public void testMultiplySpecial() { + Random r = new Random(1050); + for (BigDecimal bigDecimal : specialBigDecimals) { + for (BigDecimal bigDecimal2 : specialBigDecimals) { + doTestMultiply(r, bigDecimal, bigDecimal2); + } + } + } + + private void doTestMultiply(Random r, BigDecimal bigDecimal, BigDecimal bigDecimal2) { + + OldHiveDecimal oldDec = OldHiveDecimal.create(bigDecimal); + if (oldDec != null && isTenPowerBug(oldDec.toString())) { + return; + } + HiveDecimal dec = HiveDecimal.create(bigDecimal); + if (oldDec == null) { + assertTrue(dec == null); + return; + } + assertTrue(dec != null); + dec.validate(); + + OldHiveDecimal oldDec2 = OldHiveDecimal.create(bigDecimal2); + if (oldDec2 != null && isTenPowerBug(oldDec2.toString())) { + return; + } + HiveDecimal dec2 = HiveDecimal.create(bigDecimal2); + if (oldDec2 == null) { + assertTrue(dec2 == null); + return; + } + assertTrue(dec2 != null); + dec.validate(); + + // System.out.println("DO_TEST_MULTIPLY oldDec " + oldDec); + // System.out.println("DO_TEST_MULTIPLY dec " + dec); + // System.out.println("DO_TEST_MULTIPLY oldDec2 " + oldDec2); + // System.out.println("DO_TEST_MULTIPLY dec2 " + dec2); + + // Verify. + Assert.assertEquals(oldDec.toString(), dec.toString()); + Assert.assertEquals(oldDec2.toString(), dec2.toString()); + + // Multiply by self. + // System.out.println("DO_TEST_MULTIPLY Multiply by self"); + BigDecimal bigDecimalMultiply = bigDecimal.multiply(bigDecimal); + BigDecimal bigDecimalMultiplyAbs = bigDecimalMultiply.abs(); + String bigDecimalMultiplyAbsString = bigDecimalMultiplyAbs.toString(); + int digits = bigDecimalMultiplyAbsString.indexOf('.') != -1 ? bigDecimalMultiplyAbsString.length() - 1: bigDecimalMultiplyAbsString.length(); + // System.out.println("DO_TEST_MULTIPLY bigDecimal*bigDecimal " + bigDecimalMultiply + " digits " + digits); + OldHiveDecimal oldMultiplyDec; + HiveDecimal multiplyDec; + + oldMultiplyDec = oldDec.multiply(oldDec); + boolean isEqual; + if (oldMultiplyDec == null) { + multiplyDec = dec.multiply(dec); + if (multiplyDec != null) { + // System.out.println("DO_TEST_MULTIPLY OldHiveDecimal result null but multiplyDec " + multiplyDec); + Assert.assertTrue(false); + } + return; + } else { + // System.out.println("DO_TEST_MULTIPLY oldMultiplyDec " + oldMultiplyDec + " scale " + oldMultiplyDec.scale()); + multiplyDec = dec.multiply(dec); + if (multiplyDec == null) { + // System.out.println("DO_TEST_MULTIPLY dec*dec not equal oldMultiplyDec has result but multiplyDec is null " + oldMultiplyDec.toString() + " scale " + oldMultiplyDec.scale()); + if (isTenPowerBug(oldMultiplyDec.toString())) { + // System.out.println("DO_TEST_MULTIPLY OldHiveDecimal 10^38 bug!!!"); + return; + } + Assert.assertTrue(false); + } + multiplyDec.validate(); + // System.out.println("DO_TEST_MULTIPLY multiplyDec " + multiplyDec); + isEqual = oldMultiplyDec.toString().equals(multiplyDec.toString()); + if (!isEqual) { + // System.out.println("DO_TEST_MULTIPLY not equal"); + Assert.assertTrue(false); + } else { + // System.out.println("DO_TEST_MULTIPLY equal"); + } + } + + // System.out.println("DO_TEST_MULTIPLY Multiply by other"); + // System.out.println("DO_TEST_MULTIPLY oldDec " + oldDec); + // System.out.println("DO_TEST_MULTIPLY dec " + dec); + // System.out.println("DO_TEST_MULTIPLY oldDec2 " + oldDec2); + // System.out.println("DO_TEST_MULTIPLY dec2 " + dec2); + bigDecimalMultiply = bigDecimal.multiply(bigDecimal2); + bigDecimalMultiplyAbs = bigDecimalMultiply.abs(); + bigDecimalMultiplyAbsString = bigDecimalMultiplyAbs.toString(); + digits = bigDecimalMultiplyAbsString.indexOf('.') != -1 ? bigDecimalMultiplyAbsString.length() - 1: bigDecimalMultiplyAbsString.length(); + // System.out.println("DO_TEST_MULTIPLY bigDecimal*bigDecimal2 " + bigDecimalMultiply + " digits " + digits); + oldMultiplyDec = oldDec.multiply(oldDec2); + if (oldMultiplyDec == null) { + multiplyDec = dec.multiply(dec2); + if (multiplyDec != null) { + // System.out.println("DO_TEST_MULTIPLY OldHiveDecimal result null but multiplyDec " + multiplyDec); + Assert.assertTrue(false); + } + return; + } else { + // System.out.println("DO_TEST_MULTIPLY oldMultiplyDec " + oldMultiplyDec + " scale " + oldMultiplyDec.scale()); + multiplyDec = dec.multiply(dec2); + if (multiplyDec == null) { + // System.out.println("DO_TEST_MULTIPLY dec*dec2 not equal oldMultiplyDec has result but multiplyDec is null " + oldMultiplyDec.toString() + " scale " + oldMultiplyDec.scale()); + if (isTenPowerBug(oldMultiplyDec.toString())) { + // System.out.println("DO_TEST_ADD_POSITIVE OldHiveDecimal 10^38 bug!!!"); + return; + } + Assert.assertTrue(false); + } + multiplyDec.validate(); + // System.out.println("DO_TEST_MULTIPLY multiplyDec " + multiplyDec); + isEqual = oldMultiplyDec.toString().equals(multiplyDec.toString()); + if (!isEqual) { + // System.out.println("DO_TEST_MULTIPLY not equal"); + Assert.assertTrue(false); + } else { + // System.out.println("DO_TEST_MULTIPLY equal"); + } + } + } + + public static String displayBytes(byte[] bytes, int start, int length) { + StringBuilder sb = new StringBuilder(); + for (int i = start; i < start + length; i++) { + sb.append(String.format("\\%03d", (int) (bytes[i] & 0xff))); + } + return sb.toString(); + } } diff --git orc/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java orc/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java index 745ed9a..1118c5c 100644 --- orc/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java +++ orc/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java @@ -20,6 +20,7 @@ import java.sql.Date; import java.sql.Timestamp; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.io.BytesWritable; @@ -573,9 +574,11 @@ public String toString() { private static final class DecimalStatisticsImpl extends ColumnStatisticsImpl implements DecimalColumnStatistics { - private HiveDecimal minimum = null; - private HiveDecimal maximum = null; - private HiveDecimal sum = HiveDecimal.ZERO; + + // These objects are mutable for better performance. + private HiveDecimalWritable minimum = null; + private HiveDecimalWritable maximum = null; + private HiveDecimalWritable sum = new HiveDecimalWritable(0); DecimalStatisticsImpl() { } @@ -584,13 +587,13 @@ public String toString() { super(stats); OrcProto.DecimalStatistics dec = stats.getDecimalStatistics(); if (dec.hasMaximum()) { - maximum = HiveDecimal.create(dec.getMaximum()); + maximum = new HiveDecimalWritable(dec.getMaximum()); } if (dec.hasMinimum()) { - minimum = HiveDecimal.create(dec.getMinimum()); + minimum = new HiveDecimalWritable(dec.getMinimum()); } if (dec.hasSum()) { - sum = HiveDecimal.create(dec.getSum()); + sum = new HiveDecimalWritable(dec.getSum()); } else { sum = null; } @@ -601,21 +604,21 @@ public void reset() { super.reset(); minimum = null; maximum = null; - sum = HiveDecimal.ZERO; + sum = new HiveDecimalWritable(0); } @Override - public void updateDecimal(HiveDecimal value) { + public void updateDecimal(HiveDecimalWritable value) { if (minimum == null) { - minimum = value; - maximum = value; + minimum = new HiveDecimalWritable(value); + maximum = new HiveDecimalWritable(value); } else if (minimum.compareTo(value) > 0) { - minimum = value; + minimum.set(value); } else if (maximum.compareTo(value) < 0) { - maximum = value; + maximum.set(value); } if (sum != null) { - sum = sum.add(value); + sum.mutateAdd(value); } } @@ -624,20 +627,20 @@ public void merge(ColumnStatisticsImpl other) { if (other instanceof DecimalStatisticsImpl) { DecimalStatisticsImpl dec = (DecimalStatisticsImpl) other; if (minimum == null) { - minimum = dec.minimum; - maximum = dec.maximum; + minimum = (dec.minimum != null ? new HiveDecimalWritable(dec.minimum) : null); + maximum = (dec.maximum != null ? new HiveDecimalWritable(dec.maximum) : null); sum = dec.sum; } else if (dec.minimum != null) { if (minimum.compareTo(dec.minimum) > 0) { - minimum = dec.minimum; + minimum.set(dec.minimum); } if (maximum.compareTo(dec.maximum) < 0) { - maximum = dec.maximum; + maximum.set(dec.maximum); } if (sum == null || dec.sum == null) { sum = null; } else { - sum = sum.add(dec.sum); + sum.mutateAdd(dec.sum); } } } else { @@ -657,7 +660,8 @@ public void merge(ColumnStatisticsImpl other) { dec.setMinimum(minimum.toString()); dec.setMaximum(maximum.toString()); } - if (sum != null) { + // Check isSet for overflow. + if (sum != null && sum.isSet()) { dec.setSum(sum.toString()); } result.setDecimalStatistics(dec); @@ -666,17 +670,17 @@ public void merge(ColumnStatisticsImpl other) { @Override public HiveDecimal getMinimum() { - return minimum; + return minimum.getHiveDecimal(); } @Override public HiveDecimal getMaximum() { - return maximum; + return maximum.getHiveDecimal(); } @Override public HiveDecimal getSum() { - return sum; + return sum.getHiveDecimal(); } @Override @@ -987,7 +991,7 @@ public void updateBinary(byte[] bytes, int offset, int length, throw new UnsupportedOperationException("Can't update string"); } - public void updateDecimal(HiveDecimal value) { + public void updateDecimal(HiveDecimalWritable value) { throw new UnsupportedOperationException("Can't update decimal"); } diff --git orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java index 5d5f991..e60075f 100644 --- orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java +++ orc/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java @@ -608,19 +608,55 @@ public void nextVector(ColumnVector previousVector, setConvertTreeReader(decimalTreeReader); } - private static HiveDecimal DECIMAL_MAX_LONG = HiveDecimal.create(Long.MAX_VALUE); - private static HiveDecimal DECIMAL_MIN_LONG = HiveDecimal.create(Long.MIN_VALUE); - @Override public void setConvertVectorElement(int elementNum) throws IOException { - HiveDecimal decimalValue = decimalColVector.vector[elementNum].getHiveDecimal(); - if (decimalValue.compareTo(DECIMAL_MAX_LONG) > 0 || - decimalValue.compareTo(DECIMAL_MIN_LONG) < 0) { + HiveDecimalWritable decWritable = decimalColVector.vector[elementNum]; + long[] vector = longColVector.vector; + Category readerCategory = readerType.getCategory(); + + // Check to see if the decimal will fit in the Hive integer data type. + // If not, set the element to null. + boolean isInRange; + switch (readerCategory) { + case BOOLEAN: + // No data loss for boolean. + vector[elementNum] = decWritable.signum() == 0 ? 0 : 1; + return; + case BYTE: + isInRange = decWritable.isByte(); + break; + case SHORT: + isInRange = decWritable.isShort(); + break; + case INT: + isInRange = decWritable.isInt(); + break; + case LONG: + isInRange = decWritable.isLong(); + break; + default: + throw new RuntimeException("Unexpected type kind " + readerCategory.name()); + } + if (!isInRange) { longColVector.isNull[elementNum] = true; longColVector.noNulls = false; - } else { - // TODO: lossy conversion! - downCastAnyInteger(longColVector, elementNum, decimalValue.longValue(), readerType); + return; + } + switch (readerCategory) { + case BYTE: + vector[elementNum] = decWritable.byteValue(); + break; + case SHORT: + vector[elementNum] = decWritable.shortValue(); + break; + case INT: + vector[elementNum] = decWritable.intValue(); + break; + case LONG: + vector[elementNum] = decWritable.longValue(); + break; + default: + throw new RuntimeException("Unexpected type kind " + readerCategory.name()); } } @@ -828,7 +864,7 @@ public void nextVector(ColumnVector previousVector, @Override public void setConvertVectorElement(int elementNum) throws IOException { doubleColVector.vector[elementNum] = - (float) decimalColVector.vector[elementNum].getHiveDecimal().doubleValue(); + (float) decimalColVector.vector[elementNum].doubleValue(); } @Override @@ -1034,7 +1070,7 @@ public void nextVector(ColumnVector previousVector, @Override public void setConvertVectorElement(int elementNum) throws IOException { doubleColVector.vector[elementNum] = - decimalColVector.vector[elementNum].getHiveDecimal().doubleValue(); + decimalColVector.vector[elementNum].doubleValue(); } @Override @@ -1371,14 +1407,8 @@ public void nextVector(ColumnVector previousVector, @Override public void setConvertVectorElement(int elementNum) throws IOException { - HiveDecimalWritable valueWritable = HiveDecimalWritable.enforcePrecisionScale( - fileDecimalColVector.vector[elementNum], readerPrecision, readerScale); - if (valueWritable != null) { - decimalColVector.set(elementNum, valueWritable); - } else { - decimalColVector.noNulls = false; - decimalColVector.isNull[elementNum] = true; - } + decimalColVector.set(elementNum, fileDecimalColVector.vector[elementNum]); + } @Override @@ -1540,6 +1570,7 @@ public void nextVector(ColumnVector previousVector, private final TypeDescription readerType; private DecimalColumnVector decimalColVector; private BytesColumnVector bytesColVector; + private byte[] scratchBuffer; StringGroupFromDecimalTreeReader(int columnId, TypeDescription fileType, TypeDescription readerType, boolean skipCorrupt) throws IOException { @@ -1549,13 +1580,19 @@ public void nextVector(ColumnVector previousVector, this.readerType = readerType; decimalTreeReader = new DecimalTreeReader(columnId, precision, scale); setConvertTreeReader(decimalTreeReader); + scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES]; } @Override public void setConvertVectorElement(int elementNum) { - String string = decimalColVector.vector[elementNum].getHiveDecimal().toString(); - byte[] bytes = string.getBytes(); - assignStringGroupVectorEntry(bytesColVector, elementNum, readerType, bytes); + HiveDecimalWritable decWritable = decimalColVector.vector[elementNum]; + + // Convert decimal into bytes instead of a String for better performance. + final int byteIndex = decWritable.toBytes(scratchBuffer); + + assignStringGroupVectorEntry( + bytesColVector, elementNum, readerType, + scratchBuffer, byteIndex, HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES - byteIndex); } @Override diff --git orc/src/java/org/apache/orc/impl/TreeReaderFactory.java orc/src/java/org/apache/orc/impl/TreeReaderFactory.java index 484209b..3ddafba 100644 --- orc/src/java/org/apache/orc/impl/TreeReaderFactory.java +++ orc/src/java/org/apache/orc/impl/TreeReaderFactory.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.orc.TypeDescription; import org.apache.orc.OrcProto; @@ -1044,6 +1045,7 @@ void skipRows(long items) throws IOException { protected InStream valueStream; protected IntegerReader scaleReader = null; private int[] scratchScaleVector; + private byte[] scratchBytes; private final int precision; private final int scale; @@ -1060,6 +1062,7 @@ protected DecimalTreeReader(int columnId, int precision, int scale, InStream pre this.scale = scale; this.scratchScaleVector = new int[VectorizedRowBatch.DEFAULT_SIZE]; this.valueStream = valueStream; + this.scratchBytes = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ]; if (scaleStream != null && encoding != null) { checkEncoding(encoding); this.scaleReader = createIntegerReader(encoding.getKind(), scaleStream, true, false); @@ -1112,18 +1115,30 @@ public void nextVector(ColumnVector previousVector, // read the scales scaleReader.nextVector(result, scratchScaleVector, batchSize); // Read value entries based on isNull entries + // Use the fast ORC deserialization method that emulates SerializationUtils.readBigInteger + // provided by HiveDecimalWritable. + HiveDecimalWritable[] vector = result.vector; + HiveDecimalWritable decWritable; if (result.noNulls) { for (int r=0; r < batchSize; ++r) { - BigInteger bInt = SerializationUtils.readBigInteger(valueStream); - HiveDecimal dec = HiveDecimal.create(bInt, scratchScaleVector[r]); - result.set(r, dec); + decWritable = vector[r]; + if (!decWritable.serializationUtilsRead( + valueStream, scratchScaleVector[r], + scratchBytes)) { + result.isNull[r] = true; + result.noNulls = false; + } } } else if (!result.isRepeating || !result.isNull[0]) { for (int r=0; r < batchSize; ++r) { if (!result.isNull[r]) { - BigInteger bInt = SerializationUtils.readBigInteger(valueStream); - HiveDecimal dec = HiveDecimal.create(bInt, scratchScaleVector[r]); - result.set(r, dec); + decWritable = vector[r]; + if (!decWritable.serializationUtilsRead( + valueStream, scratchScaleVector[r], + scratchBytes)) { + result.isNull[r] = true; + result.noNulls = false; + } } } } @@ -1132,8 +1147,9 @@ public void nextVector(ColumnVector previousVector, @Override void skipRows(long items) throws IOException { items = countNonNulls(items); + HiveDecimalWritable scratchDecWritable = new HiveDecimalWritable(); for (int i = 0; i < items; i++) { - SerializationUtils.readBigInteger(valueStream); + scratchDecWritable.serializationUtilsRead(valueStream, 0, scratchBytes); } scaleReader.skip(items); } diff --git orc/src/java/org/apache/orc/impl/WriterImpl.java orc/src/java/org/apache/orc/impl/WriterImpl.java index b17fb41..1c3e8f5 100644 --- orc/src/java/org/apache/orc/impl/WriterImpl.java +++ orc/src/java/org/apache/orc/impl/WriterImpl.java @@ -30,6 +30,7 @@ import java.util.TimeZone; import java.util.TreeMap; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.ql.util.JavaDataModel; import org.apache.orc.BinaryColumnStatistics; import org.apache.orc.BloomFilterIO; @@ -1610,6 +1611,11 @@ void recordPosition(PositionRecorder recorder) throws IOException { private static class DecimalTreeWriter extends TreeWriter { private final PositionedOutputStream valueStream; + + // These scratch buffers allow us to serialize decimals much faster. + private final long[] scratchLongs; + private final byte[] scratchBuffer; + private final IntegerWriter scaleStream; private final boolean isDirectV2; @@ -1620,6 +1626,8 @@ void recordPosition(PositionRecorder recorder) throws IOException { super(columnId, schema, writer, nullable); this.isDirectV2 = isNewWriteFormat(writer); valueStream = writer.createStream(id, OrcProto.Stream.Kind.DATA); + scratchLongs = new long[HiveDecimal.SCRATCH_LONGS_LEN]; + scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES]; this.scaleStream = createIntegerWriter(writer.createStream(id, OrcProto.Stream.Kind.SECONDARY), true, isDirectV2, writer); recordPosition(rowIndexPosition); @@ -1640,29 +1648,41 @@ void writeBatch(ColumnVector vector, int offset, int length) throws IOException { super.writeBatch(vector, offset, length); DecimalColumnVector vec = (DecimalColumnVector) vector; + int byteIndex; if (vector.isRepeating) { if (vector.noNulls || !vector.isNull[0]) { - HiveDecimal value = vec.vector[0].getHiveDecimal(); + HiveDecimalWritable value = vec.vector[0]; indexStatistics.updateDecimal(value); if (createBloomFilter) { - bloomFilter.addString(value.toString()); + + // The decimal toBytes method writes the decimal UTF-8 bytes at the end of the scratch + // buffer and returns the index of the first UTF-8 byte. It is used for good + // performance. + byteIndex = value.toBytes(scratchBuffer); + bloomFilter.addBytes(scratchBuffer, byteIndex, scratchBuffer.length - byteIndex); } for(int i=0; i < length; ++i) { - SerializationUtils.writeBigInteger(valueStream, - value.unscaledValue()); + + // Use the fast ORC serialization method that emulates SerializationUtils.writeBigInteger + // provided by HiveDecimalWritable. + value.serializationUtilsWrite( + valueStream, + scratchLongs); scaleStream.write(value.scale()); } } } else { for(int i=0; i < length; ++i) { if (vec.noNulls || !vec.isNull[i + offset]) { - HiveDecimal value = vec.vector[i + offset].getHiveDecimal(); - SerializationUtils.writeBigInteger(valueStream, - value.unscaledValue()); + HiveDecimalWritable value = vec.vector[i + offset]; + value.serializationUtilsWrite( + valueStream, + scratchLongs); scaleStream.write(value.scale()); indexStatistics.updateDecimal(value); if (createBloomFilter) { - bloomFilter.addString(value.toString()); + byteIndex = value.toBytes(scratchBuffer); + bloomFilter.addBytes(scratchBuffer, byteIndex, scratchBuffer.length - byteIndex); } } } diff --git orc/src/test/org/apache/orc/TestColumnStatistics.java orc/src/test/org/apache/orc/TestColumnStatistics.java index 1837dbb..93d4bdb 100644 --- orc/src/test/org/apache/orc/TestColumnStatistics.java +++ orc/src/test/org/apache/orc/TestColumnStatistics.java @@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; @@ -170,17 +171,17 @@ public void testDecimalMerge() throws Exception { ColumnStatisticsImpl stats1 = ColumnStatisticsImpl.create(schema); ColumnStatisticsImpl stats2 = ColumnStatisticsImpl.create(schema); - stats1.updateDecimal(HiveDecimal.create(10)); - stats1.updateDecimal(HiveDecimal.create(100)); - stats2.updateDecimal(HiveDecimal.create(1)); - stats2.updateDecimal(HiveDecimal.create(1000)); + stats1.updateDecimal(new HiveDecimalWritable(10)); + stats1.updateDecimal(new HiveDecimalWritable(100)); + stats2.updateDecimal(new HiveDecimalWritable(1)); + stats2.updateDecimal(new HiveDecimalWritable(1000)); stats1.merge(stats2); DecimalColumnStatistics typed = (DecimalColumnStatistics) stats1; assertEquals(1, typed.getMinimum().longValue()); assertEquals(1000, typed.getMaximum().longValue()); stats1.reset(); - stats1.updateDecimal(HiveDecimal.create(-10)); - stats1.updateDecimal(HiveDecimal.create(10000)); + stats1.updateDecimal(new HiveDecimalWritable(-10)); + stats1.updateDecimal(new HiveDecimalWritable(10000)); stats1.merge(stats2); assertEquals(-10, typed.getMinimum().longValue()); assertEquals(10000, typed.getMaximum().longValue()); diff --git ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt index 9a48171..b532e2f 100644 --- ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt +++ ql/src/gen/vectorization/UDAFTemplates/VectorUDAFMinMaxDecimal.txt @@ -61,12 +61,11 @@ public class extends VectorAggregateExpression { } public void checkValue(HiveDecimalWritable writable, short scale) { - HiveDecimal value = writable.getHiveDecimal(); if (isNull) { isNull = false; - this.value.set(value); - } else if (this.value.getHiveDecimal().compareTo(value) 0) { - this.value.set(value); + this.value.set(writable); + } else if (this.value.compareTo(writable) 0) { + this.value.set(writable); } } @@ -321,8 +320,7 @@ public class extends VectorAggregateExpression { if (inputVector.noNulls && (myagg.isNull || (myagg.value.compareTo(vector[0]) 0))) { myagg.isNull = false; - HiveDecimal value = vector[0].getHiveDecimal(); - myagg.value.set(value); + myagg.value.set(vector[0]); } return; } @@ -354,13 +352,13 @@ public class extends VectorAggregateExpression { for (int j=0; j< batchSize; ++j) { int i = selected[j]; if (!isNull[i]) { - HiveDecimal value = vector[i].getHiveDecimal(); + HiveDecimalWritable writable = vector[i]; if (myagg.isNull) { myagg.isNull = false; - myagg.value.set(value); + myagg.value.set(writable); } - else if (myagg.value.getHiveDecimal().compareTo(value) 0) { - myagg.value.set(value); + else if (myagg.value.compareTo(writable) 0) { + myagg.value.set(writable); } } } @@ -374,15 +372,14 @@ public class extends VectorAggregateExpression { int[] selected) { if (myagg.isNull) { - HiveDecimal value = vector[selected[0]].getHiveDecimal(); - myagg.value.set(value); + myagg.value.set(vector[selected[0]]); myagg.isNull = false; } for (int i=0; i< batchSize; ++i) { - HiveDecimal value = vector[selected[i]].getHiveDecimal(); - if (myagg.value.getHiveDecimal().compareTo(value) 0) { - myagg.value.set(value); + HiveDecimalWritable writable = vector[selected[i]]; + if (myagg.value.compareTo(writable) 0) { + myagg.value.set(writable); } } } @@ -396,13 +393,13 @@ public class extends VectorAggregateExpression { for(int i=0;i 0) { - myagg.value.set(value); + else if (myagg.value.compareTo(writable) 0) { + myagg.value.set(writable); } } } @@ -414,15 +411,14 @@ public class extends VectorAggregateExpression { short scale, int batchSize) { if (myagg.isNull) { - HiveDecimal value = vector[0].getHiveDecimal(); - myagg.value.set(value); + myagg.value.set(vector[0]); myagg.isNull = false; } for (int i=0;i 0) { - myagg.value.set(value); + HiveDecimalWritable writable = vector[i]; + if (myagg.value.compareTo(writable) 0) { + myagg.value.set(writable); } } } @@ -472,4 +468,3 @@ public class extends VectorAggregateExpression { this.inputExpression = inputExpression; } } - diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java index d31d338..9db9eca 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java @@ -534,8 +534,9 @@ private void storeRowColumn(VectorizedRowBatch batch, int batchIndex, } break; case DECIMAL: + // The DecimalColumnVector set method will quickly copy the fast decimal fields. ((DecimalColumnVector) batch.cols[projectionColumnNum]).set( - batchIndex, deserializeRead.currentHiveDecimalWritable.getHiveDecimal()); + batchIndex, deserializeRead.currentHiveDecimalWritable); break; case INTERVAL_YEAR_MONTH: ((LongColumnVector) batch.cols[projectionColumnNum]).vector[batchIndex] = diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java index e6dc9ec..2ac0f79 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java @@ -310,8 +310,9 @@ public Object extractRowColumn(VectorizedRowBatch batch, int batchIndex, int log return primitiveWritable; } case DECIMAL: + // The HiveDecimalWritable set method will quickly copy the fast decimal fields. ((HiveDecimalWritable) primitiveWritable).set( - ((DecimalColumnVector) batch.cols[projectionColumnNum]).vector[adjustedIndex].getHiveDecimal()); + ((DecimalColumnVector) batch.cols[projectionColumnNum]).vector[adjustedIndex]); return primitiveWritable; case INTERVAL_YEAR_MONTH: ((HiveIntervalYearMonthWritable) primitiveWritable).set( diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java index 8a101a6..ce8c427 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapper.java @@ -109,7 +109,7 @@ public void setHashKey() { Arrays.hashCode(isNull); for (int i = 0; i < decimalValues.length; i++) { - hashcode ^= decimalValues[i].getHiveDecimal().hashCode(); + hashcode ^= decimalValues[i].hashCode(); } for (int i = 0; i < timestampValues.length; i++) { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java index bfd26ae..b4708b5 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java @@ -773,7 +773,7 @@ public Object getWritableKeyValue(VectorHashKeyWrapper kw, int i, } else if (klh.decimalIndex >= 0) { return kw.getIsDecimalNull(klh.decimalIndex)? null : keyOutputWriter.writeValue( - kw.getDecimal(klh.decimalIndex).getHiveDecimal()); + kw.getDecimal(klh.decimalIndex)); } else if (klh.timestampIndex >= 0) { return kw.getIsTimestampNull(klh.timestampIndex)? null : keyOutputWriter.writeValue( diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java index 6af3d99..a95098a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java @@ -497,7 +497,10 @@ boolean apply(VectorizedRowBatch batch, int batchIndex) throws IOException { if (colVector.isRepeating) { if (colVector.noNulls || !colVector.isNull[0]) { - serializeWrite.writeHiveDecimal(colVector.vector[0].getHiveDecimal(), colVector.scale); + // We serialize specifying the HiveDecimalWritable but also the desired + // serialization scale that will be used by text serialization for adding + // trailing fractional zeroes. + serializeWrite.writeHiveDecimal(colVector.vector[0], colVector.scale); return true; } else { serializeWrite.writeNull(); @@ -505,7 +508,7 @@ boolean apply(VectorizedRowBatch batch, int batchIndex) throws IOException { } } else { if (colVector.noNulls || !colVector.isNull[batchIndex]) { - serializeWrite.writeHiveDecimal(colVector.vector[batchIndex].getHiveDecimal(), colVector.scale); + serializeWrite.writeHiveDecimal(colVector.vector[batchIndex], colVector.scale); return true; } else { serializeWrite.writeNull(); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java index d2f5408..2a6d263 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java @@ -383,20 +383,20 @@ protected OutputColumnManager(int initialOutputCol) { //Vectorized row batch for processing. The index in the row batch is //equal to the index in this array plus initialOutputCol. //Start with size 100 and double when needed. - private String [] outputColumnsTypes = new String[100]; + private String [] scratchVectorTypeNames = new String[100]; private final Set usedOutputColumns = new HashSet(); - int allocateOutputColumn(String hiveTypeName) throws HiveException { + int allocateOutputColumn(TypeInfo typeInfo) throws HiveException { if (initialOutputCol < 0) { - // This is a test + // This is a test calling. return 0; } - // We need to differentiate DECIMAL columns by their precision and scale... - String normalizedTypeName = getNormalizedName(hiveTypeName); - int relativeCol = allocateOutputColumnInternal(normalizedTypeName); - // LOG.info("allocateOutputColumn for hiveTypeName " + hiveTypeName + " column " + (initialOutputCol + relativeCol)); + // CONCERN: We currently differentiate DECIMAL columns by their precision and scale..., + // which could lead to a lot of extra unnecessary scratch columns. + String vectorTypeName = getScratchName(typeInfo); + int relativeCol = allocateOutputColumnInternal(vectorTypeName); return initialOutputCol + relativeCol; } @@ -405,7 +405,7 @@ private int allocateOutputColumnInternal(String columnType) { // Re-use an existing, available column of the same required type. if (usedOutputColumns.contains(i) || - !(outputColumnsTypes)[i].equalsIgnoreCase(columnType)) { + !(scratchVectorTypeNames)[i].equalsIgnoreCase(columnType)) { continue; } //Use i @@ -413,16 +413,16 @@ private int allocateOutputColumnInternal(String columnType) { return i; } //Out of allocated columns - if (outputColCount < outputColumnsTypes.length) { + if (outputColCount < scratchVectorTypeNames.length) { int newIndex = outputColCount; - outputColumnsTypes[outputColCount++] = columnType; + scratchVectorTypeNames[outputColCount++] = columnType; usedOutputColumns.add(newIndex); return newIndex; } else { //Expand the array - outputColumnsTypes = Arrays.copyOf(outputColumnsTypes, 2*outputColCount); + scratchVectorTypeNames = Arrays.copyOf(scratchVectorTypeNames, 2*outputColCount); int newIndex = outputColCount; - outputColumnsTypes[outputColCount++] = columnType; + scratchVectorTypeNames[outputColCount++] = columnType; usedOutputColumns.add(newIndex); return newIndex; } @@ -448,8 +448,8 @@ void freeOutputColumn(int index) { } } - public int allocateScratchColumn(String hiveTypeName) throws HiveException { - return ocm.allocateOutputColumn(hiveTypeName); + public int allocateScratchColumn(TypeInfo typeInfo) throws HiveException { + return ocm.allocateOutputColumn(typeInfo); } public int[] currentScratchColumns() { @@ -1044,7 +1044,7 @@ private VectorExpression getConstantVectorExpression(Object constantValue, TypeI } int outCol = -1; if (mode == VectorExpressionDescriptor.Mode.PROJECTION) { - outCol = ocm.allocateOutputColumn(typeName); + outCol = ocm.allocateOutputColumn(typeInfo); } if (constantValue == null) { return new ConstantVectorExpression(outCol, typeName, true); @@ -1286,24 +1286,26 @@ private VectorExpression instantiateExpression(Class vclass, TypeInfo returnT // Additional argument is needed, which is the outputcolumn. Object [] newArgs = null; try { - String outType; - - // Special handling for decimal because decimal types need scale and precision parameter. - // This special handling should be avoided by using returnType uniformly for all cases. - if (returnType != null) { - outType = getNormalizedName(returnType.getTypeName()).toLowerCase(); - if (outType == null) { - throw new HiveException("No vector type for type name " + returnType); + String returnTypeName; + if (returnType == null) { + returnTypeName = ((VectorExpression) vclass.newInstance()).getOutputType().toLowerCase(); + if (returnTypeName.equals("long")) { + returnTypeName = "bigint"; } + returnType = TypeInfoUtils.getTypeInfoFromTypeString(returnTypeName); } else { - outType = ((VectorExpression) vclass.newInstance()).getOutputType(); + returnTypeName = returnType.getTypeName(); } - int outputCol = ocm.allocateOutputColumn(outType); + + // Special handling for decimal because decimal types need scale and precision parameter. + // This special handling should be avoided by using returnType uniformly for all cases. + int outputCol = ocm.allocateOutputColumn(returnType); + newArgs = Arrays.copyOf(args, numParams); newArgs[numParams-1] = outputCol; ve = (VectorExpression) ctor.newInstance(newArgs); - ve.setOutputType(outType); + ve.setOutputType(returnTypeName); } catch (Exception ex) { throw new HiveException("Could not instantiate " + vclass.getSimpleName() + " with arguments " + getNewInstanceArgumentString(newArgs) + ", exception: " + StringUtils.stringifyException(ex)); @@ -1398,7 +1400,7 @@ private VectorExpression getCoalesceExpression(List childExpr, Typ inputColumns[i++] = ve.getOutputColumn(); } - int outColumn = ocm.allocateOutputColumn(returnType.getTypeName()); + int outColumn = ocm.allocateOutputColumn(returnType); VectorCoalesce vectorCoalesce = new VectorCoalesce(inputColumns, outColumn); vectorCoalesce.setOutputType(returnType.getTypeName()); vectorCoalesce.setChildExpressions(vectorChildren); @@ -1425,7 +1427,7 @@ private VectorExpression getEltExpression(List childExpr, TypeInfo inputColumns[i++] = ve.getOutputColumn(); } - int outColumn = ocm.allocateOutputColumn(returnType.getTypeName()); + int outColumn = ocm.allocateOutputColumn(returnType); VectorElt vectorElt = new VectorElt(inputColumns, outColumn); vectorElt.setOutputType(returnType.getTypeName()); vectorElt.setChildExpressions(vectorChildren); @@ -1607,7 +1609,7 @@ private VectorExpression getStructInExpression(List childExpr, Exp // Create a single child representing the scratch column where we will // generate the serialized keys of the batch. - int scratchBytesCol = ocm.allocateOutputColumn("string"); + int scratchBytesCol = ocm.allocateOutputColumn(TypeInfoFactory.stringTypeInfo); Class cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterStructColumnInList.class : StructColumnInList.class); @@ -1854,7 +1856,10 @@ private Long castConstantToLong(Object scalar, TypeInfo type) throws HiveExcepti return ((Number) scalar).longValue(); case DECIMAL: HiveDecimal decimalVal = (HiveDecimal) scalar; - return decimalVal.longValueExact(); + if (!decimalVal.isLong()) { + throw new ArithmeticException("Overflow"); + } + return decimalVal.longValue(); default: throw new HiveException("Unsupported type "+typename+" for cast to Long"); } @@ -2004,7 +2009,7 @@ private VectorExpression getCastToBoolean(List childExpr) VectorExpression lenExpr = createVectorExpression(StringLength.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, null); - int outputCol = ocm.allocateOutputColumn("Long"); + int outputCol = ocm.allocateOutputColumn(TypeInfoFactory.longTypeInfo); VectorExpression lenToBoolExpr = new CastLongToBooleanViaLongToLong(lenExpr.getOutputColumn(), outputCol); lenToBoolExpr.setChildExpressions(new VectorExpression[] {lenExpr}); @@ -2196,12 +2201,10 @@ private VectorExpression getCustomUDFExpression(ExprNodeGenericFuncDesc expr, Ve int outputCol = -1; String resultTypeName = expr.getTypeInfo().getTypeName(); - outputCol = ocm.allocateOutputColumn(resultTypeName); + outputCol = ocm.allocateOutputColumn(expr.getTypeInfo()); // Make vectorized operator - String normalizedName = getNormalizedName(resultTypeName); - - VectorExpression ve = new VectorUDFAdaptor(expr, outputCol, normalizedName, argDescs); + VectorExpression ve = new VectorUDFAdaptor(expr, outputCol, resultTypeName, argDescs); // Set child expressions VectorExpression[] childVEs = null; @@ -2395,36 +2398,15 @@ private Timestamp evaluateCastToTimestamp(ExprNodeDesc expr) throws HiveExceptio } } - static String getNormalizedName(String hiveTypeName) throws HiveException { - VectorExpressionDescriptor.ArgumentType argType = VectorExpressionDescriptor.ArgumentType.fromHiveTypeName(hiveTypeName); - switch (argType) { - case INT_FAMILY: - return "Long"; - case FLOAT_FAMILY: - return "Double"; - case DECIMAL: - //Return the decimal type as is, it includes scale and precision. - return hiveTypeName; - case STRING: - return "String"; - case CHAR: - //Return the CHAR type as is, it includes maximum length - return hiveTypeName; - case VARCHAR: - //Return the VARCHAR type as is, it includes maximum length. - return hiveTypeName; - case BINARY: - return "Binary"; - case DATE: - return "Date"; - case TIMESTAMP: - return "Timestamp"; - case INTERVAL_YEAR_MONTH: - case INTERVAL_DAY_TIME: - return hiveTypeName; - default: - throw new HiveException("Unexpected hive type name " + hiveTypeName); + static String getScratchName(TypeInfo typeInfo) throws HiveException { + // For now, leave DECIMAL precision/scale in the name so DecimalColumnVector scratch columns + // don't need their precision/scale adjusted... + if (typeInfo.getCategory() == Category.PRIMITIVE && + ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory() == PrimitiveCategory.DECIMAL) { + return typeInfo.getTypeName(); } + Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo); + return columnVectorType.name().toLowerCase(); } static String getUndecoratedName(String hiveTypeName) throws HiveException { @@ -2702,9 +2684,16 @@ public int firstOutputColumnIndex() { public String[] getScratchColumnTypeNames() { String[] result = new String[ocm.outputColCount]; for (int i = 0; i < ocm.outputColCount; i++) { - String typeName = ocm.outputColumnsTypes[i]; - if (typeName.equalsIgnoreCase("long")) { - typeName = "bigint"; // Convert our synonym to a real Hive type name. + String vectorTypeName = ocm.scratchVectorTypeNames[i]; + String typeName; + if (vectorTypeName.equalsIgnoreCase("bytes")) { + // Use hive type name. + typeName = "string"; + } else if (vectorTypeName.equalsIgnoreCase("long")) { + // Use hive type name. + typeName = "bigint"; + } else { + typeName = vectorTypeName; } result[i] = typeName; } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToBoolean.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToBoolean.java index 9621cd3..ac52373 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToBoolean.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToBoolean.java @@ -41,6 +41,11 @@ public CastDecimalToBoolean(int inputColumn, int outputColumn) { * Otherwise, return 1 for true. */ protected void func(LongColumnVector outV, DecimalColumnVector inV, int i) { - outV.vector[i] = inV.vector[i].getHiveDecimal().signum() == 0 ? 0 : 1; + outV.vector[i] = inV.vector[i].signum() == 0 ? 0 : 1; + } + + @Override + public String getOutputType() { + return "Boolean"; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToChar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToChar.java index aab3e70..e753a6e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToChar.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToChar.java @@ -37,8 +37,8 @@ public CastDecimalToChar(int inputColumn, int outputColumn) { } @Override - protected void assign(BytesColumnVector outV, int i, byte[] bytes, int length) { - StringExpr.rightTrimAndTruncate(outV, i, bytes, 0, length, maxLength); + protected void assign(BytesColumnVector outV, int i, byte[] bytes, int offset, int length) { + StringExpr.rightTrimAndTruncate(outV, i, bytes, offset, length, maxLength); } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToDouble.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToDouble.java index 63d878d..9cf97f4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToDouble.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToDouble.java @@ -34,6 +34,6 @@ public CastDecimalToDouble(int inputCol, int outputCol) { } protected void func(DoubleColumnVector outV, DecimalColumnVector inV, int i) { - outV.vector[i] = inV.vector[i].getHiveDecimal().doubleValue(); + outV.vector[i] = inV.vector[i].doubleValue(); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToLong.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToLong.java index 2ff6b79..28a2d74 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToLong.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToLong.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; /** * Type cast decimal to long @@ -37,6 +38,47 @@ public CastDecimalToLong(int inputColumn, int outputColumn) { @Override protected void func(LongColumnVector outV, DecimalColumnVector inV, int i) { - outV.vector[i] = inV.vector[i].getHiveDecimal().longValue(); // TODO: lossy conversion! + HiveDecimalWritable decWritable = inV.vector[i]; + + // Check based on the Hive integer type we need to test with isByte, isShort, isInt, isLong + // so we do not use corrupted (truncated) values for the Hive integer type. + boolean isInRange; + switch (integerPrimitiveCategory) { + case BYTE: + isInRange = decWritable.isByte(); + break; + case SHORT: + isInRange = decWritable.isShort(); + break; + case INT: + isInRange = decWritable.isInt(); + break; + case LONG: + isInRange = decWritable.isLong(); + break; + default: + throw new RuntimeException("Unexpected integer primitive category " + integerPrimitiveCategory); + } + if (!isInRange) { + outV.isNull[i] = true; + outV.noNulls = false; + return; + } + switch (integerPrimitiveCategory) { + case BYTE: + outV.vector[i] = decWritable.byteValue(); + break; + case SHORT: + outV.vector[i] = decWritable.shortValue(); + break; + case INT: + outV.vector[i] = decWritable.intValue(); + break; + case LONG: + outV.vector[i] = decWritable.longValue(); + break; + default: + throw new RuntimeException("Unexpected integer primitive category " + integerPrimitiveCategory); + } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToString.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToString.java index 243a807..ca58890 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToString.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToString.java @@ -18,8 +18,10 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; +import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; /** * To support vectorized cast of decimal to string. @@ -28,29 +30,28 @@ private static final long serialVersionUID = 1L; + // We use a scratch buffer with the HiveDecimalWritable toBytes method so + // we don't incur poor performance creating a String result. + private byte[] scratchBuffer; + public CastDecimalToString() { super(); } public CastDecimalToString(int inputColumn, int outputColumn) { super(inputColumn, outputColumn); + scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES]; } // The assign method will be overridden for CHAR and VARCHAR. - protected void assign(BytesColumnVector outV, int i, byte[] bytes, int length) { - outV.setVal(i, bytes, 0, length); + protected void assign(BytesColumnVector outV, int i, byte[] bytes, int offset, int length) { + outV.setVal(i, bytes, offset, length); } @Override protected void func(BytesColumnVector outV, DecimalColumnVector inV, int i) { - String s = inV.vector[i].getHiveDecimal().toString(); - byte[] b = null; - try { - b = s.getBytes("UTF-8"); - } catch (Exception e) { - // This should never happen. If it does, there is a bug. - throw new RuntimeException("Internal error: unable to convert decimal to string", e); - } - assign(outV, i, b, b.length); + HiveDecimalWritable decWritable = inV.vector[i]; + final int byteIndex = decWritable.toBytes(scratchBuffer); + assign(outV, i, scratchBuffer, byteIndex, HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES - byteIndex); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToTimestamp.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToTimestamp.java index 8963449..dfd9802 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToTimestamp.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToTimestamp.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector; import org.apache.hadoop.hive.ql.util.TimestampUtils; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; /** * Type cast decimal to timestamp. The decimal value is interpreted @@ -32,8 +33,13 @@ public class CastDecimalToTimestamp extends FuncDecimalToTimestamp { private static final long serialVersionUID = 1L; + private HiveDecimalWritable scratchHiveDecimalWritable1; + private HiveDecimalWritable scratchHiveDecimalWritable2; + public CastDecimalToTimestamp(int inputColumn, int outputColumn) { super(inputColumn, outputColumn); + scratchHiveDecimalWritable1 = new HiveDecimalWritable(); + scratchHiveDecimalWritable2 = new HiveDecimalWritable(); } public CastDecimalToTimestamp() { @@ -41,7 +47,10 @@ public CastDecimalToTimestamp() { @Override protected void func(TimestampColumnVector outV, DecimalColumnVector inV, int i) { - Timestamp timestamp = TimestampUtils.decimalToTimestamp(inV.vector[i].getHiveDecimal()); + Timestamp timestamp = + TimestampUtils.decimalToTimestamp( + inV.vector[i], + scratchHiveDecimalWritable1, scratchHiveDecimalWritable2); outV.set(i, timestamp); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToVarChar.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToVarChar.java index 267b0b1..3a2c2d0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToVarChar.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDecimalToVarChar.java @@ -37,8 +37,8 @@ public CastDecimalToVarChar(int inputColumn, int outputColumn) { } @Override - protected void assign(BytesColumnVector outV, int i, byte[] bytes, int length) { - StringExpr.truncate(outV, i, bytes, 0, length, maxLength); + protected void assign(BytesColumnVector outV, int i, byte[] bytes, int offset, int length) { + StringExpr.truncate(outV, i, bytes, offset, length, maxLength); } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDoubleToDecimal.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDoubleToDecimal.java index 6d6b588..79478b9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDoubleToDecimal.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastDoubleToDecimal.java @@ -18,9 +18,9 @@ package org.apache.hadoop.hive.ql.exec.vector.expressions; -import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector; import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; /** * Cast input double to a decimal. Get target value scale from output column vector. @@ -39,7 +39,11 @@ public CastDoubleToDecimal(int inputColumn, int outputColumn) { @Override protected void func(DecimalColumnVector outV, DoubleColumnVector inV, int i) { - String s = ((Double) inV.vector[i]).toString(); - outV.vector[i].set(HiveDecimal.create(s)); + HiveDecimalWritable decWritable = outV.vector[i]; + decWritable.setFromDouble(inV.vector[i]); + if (!decWritable.isSet()) { + outV.isNull[i] = true; + outV.noNulls = false; + } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnInList.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnInList.java index 0601c66..d4d8fea 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnInList.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalColumnInList.java @@ -37,7 +37,10 @@ private int outputColumn; // The set object containing the IN list. - private transient HashSet inSet; + // We use a HashSet of HiveDecimalWritable objects instead of HiveDecimal objects so + // we can lookup DecimalColumnVector HiveDecimalWritable quickly without creating + // a HiveDecimal lookup object. + private transient HashSet inSet; public DecimalColumnInList() { super(); @@ -61,9 +64,9 @@ public void evaluate(VectorizedRowBatch batch) { } if (inSet == null) { - inSet = new HashSet(inListValues.length); + inSet = new HashSet(inListValues.length); for (HiveDecimal val : inListValues) { - inSet.add(val); + inSet.add(new HiveDecimalWritable(val)); } } @@ -88,16 +91,16 @@ public void evaluate(VectorizedRowBatch batch) { // All must be selected otherwise size would be zero // Repeating property will not change. - outputVector[0] = inSet.contains(vector[0].getHiveDecimal()) ? 1 : 0; + outputVector[0] = inSet.contains(vector[0]) ? 1 : 0; outputColVector.isRepeating = true; } else if (batch.selectedInUse) { for(int j = 0; j != n; j++) { int i = sel[j]; - outputVector[i] = inSet.contains(vector[i].getHiveDecimal()) ? 1 : 0; + outputVector[i] = inSet.contains(vector[i]) ? 1 : 0; } } else { for(int i = 0; i != n; i++) { - outputVector[i] = inSet.contains(vector[i].getHiveDecimal()) ? 1 : 0; + outputVector[i] = inSet.contains(vector[i]) ? 1 : 0; } } } else { @@ -106,7 +109,7 @@ public void evaluate(VectorizedRowBatch batch) { //All must be selected otherwise size would be zero //Repeating property will not change. if (!nullPos[0]) { - outputVector[0] = inSet.contains(vector[0].getHiveDecimal()) ? 1 : 0; + outputVector[0] = inSet.contains(vector[0]) ? 1 : 0; outNulls[0] = false; } else { outNulls[0] = true; @@ -117,14 +120,14 @@ public void evaluate(VectorizedRowBatch batch) { int i = sel[j]; outNulls[i] = nullPos[i]; if (!nullPos[i]) { - outputVector[i] = inSet.contains(vector[i].getHiveDecimal()) ? 1 : 0; + outputVector[i] = inSet.contains(vector[i]) ? 1 : 0; } } } else { System.arraycopy(nullPos, 0, outNulls, 0, n); for(int i = 0; i != n; i++) { if (!nullPos[i]) { - outputVector[i] = inSet.contains(vector[i].getHiveDecimal()) ? 1 : 0; + outputVector[i] = inSet.contains(vector[i]) ? 1 : 0; } } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalUtil.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalUtil.java index a01f7a2..254044c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalUtil.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/DecimalUtil.java @@ -34,15 +34,17 @@ public static int compare(HiveDecimalWritable writableLeft, HiveDecimal right) { } public static int compare(HiveDecimal left, HiveDecimalWritable writableRight) { - return left.compareTo(writableRight.getHiveDecimal()); + return HiveDecimalWritable.compareTo(left, writableRight); } // Addition with overflow check. Overflow produces NULL output. public static void addChecked(int i, HiveDecimal left, HiveDecimal right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.add(right)); - } catch (ArithmeticException e) { // catch on overflow + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateAdd(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -50,9 +52,11 @@ public static void addChecked(int i, HiveDecimal left, HiveDecimal right, public static void addChecked(int i, HiveDecimalWritable left, HiveDecimalWritable right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.getHiveDecimal().add(right.getHiveDecimal())); - } catch (ArithmeticException e) { // catch on overflow + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateAdd(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -60,9 +64,11 @@ public static void addChecked(int i, HiveDecimalWritable left, HiveDecimalWritab public static void addChecked(int i, HiveDecimalWritable left, HiveDecimal right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.getHiveDecimal().add(right)); - } catch (ArithmeticException e) { // catch on overflow + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateAdd(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -70,9 +76,11 @@ public static void addChecked(int i, HiveDecimalWritable left, HiveDecimal right public static void addChecked(int i, HiveDecimal left, HiveDecimalWritable right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.add(right.getHiveDecimal())); - } catch (ArithmeticException e) { // catch on overflow + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateAdd(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -81,9 +89,11 @@ public static void addChecked(int i, HiveDecimal left, HiveDecimalWritable right // Subtraction with overflow check. Overflow produces NULL output. public static void subtractChecked(int i, HiveDecimal left, HiveDecimal right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.subtract(right)); - } catch (ArithmeticException e) { // catch on overflow + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateSubtract(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -91,9 +101,11 @@ public static void subtractChecked(int i, HiveDecimal left, HiveDecimal right, public static void subtractChecked(int i, HiveDecimalWritable left, HiveDecimalWritable right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.getHiveDecimal().subtract(right.getHiveDecimal())); - } catch (ArithmeticException e) { // catch on overflow + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateSubtract(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -101,9 +113,11 @@ public static void subtractChecked(int i, HiveDecimalWritable left, HiveDecimalW public static void subtractChecked(int i, HiveDecimalWritable left, HiveDecimal right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.getHiveDecimal().subtract(right)); - } catch (ArithmeticException e) { // catch on overflow + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateSubtract(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -111,9 +125,11 @@ public static void subtractChecked(int i, HiveDecimalWritable left, HiveDecimal public static void subtractChecked(int i, HiveDecimal left, HiveDecimalWritable right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.subtract(right.getHiveDecimal())); - } catch (ArithmeticException e) { // catch on overflow + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateSubtract(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -122,9 +138,11 @@ public static void subtractChecked(int i, HiveDecimal left, HiveDecimalWritable // Multiplication with overflow check. Overflow produces NULL output. public static void multiplyChecked(int i, HiveDecimal left, HiveDecimal right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.multiply(right)); - } catch (ArithmeticException e) { // catch on overflow + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateMultiply(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -132,9 +150,11 @@ public static void multiplyChecked(int i, HiveDecimal left, HiveDecimal right, public static void multiplyChecked(int i, HiveDecimalWritable left, HiveDecimalWritable right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.getHiveDecimal().multiply(right.getHiveDecimal())); - } catch (ArithmeticException e) { // catch on overflow + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateMultiply(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -142,9 +162,11 @@ public static void multiplyChecked(int i, HiveDecimalWritable left, HiveDecimalW public static void multiplyChecked(int i, HiveDecimalWritable left, HiveDecimal right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.getHiveDecimal().multiply(right)); - } catch (ArithmeticException e) { // catch on overflow + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateMultiply(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -152,9 +174,11 @@ public static void multiplyChecked(int i, HiveDecimalWritable left, HiveDecimal public static void multiplyChecked(int i, HiveDecimal left, HiveDecimalWritable right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.multiply(right.getHiveDecimal())); - } catch (ArithmeticException e) { // catch on overflow + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateMultiply(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -163,9 +187,11 @@ public static void multiplyChecked(int i, HiveDecimal left, HiveDecimalWritable // Division with overflow/zero-divide check. Error produces NULL output. public static void divideChecked(int i, HiveDecimal left, HiveDecimal right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.divide(right)); - } catch (ArithmeticException e) { // catch on error + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateDivide(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -173,9 +199,11 @@ public static void divideChecked(int i, HiveDecimal left, HiveDecimal right, public static void divideChecked(int i, HiveDecimalWritable left, HiveDecimalWritable right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.getHiveDecimal().divide(right.getHiveDecimal())); - } catch (ArithmeticException e) { // catch on error + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateDivide(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -183,9 +211,11 @@ public static void divideChecked(int i, HiveDecimalWritable left, HiveDecimalWri public static void divideChecked(int i, HiveDecimalWritable left, HiveDecimal right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.getHiveDecimal().divide(right)); - } catch (ArithmeticException e) { // catch on error + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateDivide(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -193,9 +223,11 @@ public static void divideChecked(int i, HiveDecimalWritable left, HiveDecimal ri public static void divideChecked(int i, HiveDecimal left, HiveDecimalWritable right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.divide(right.getHiveDecimal())); - } catch (ArithmeticException e) { // catch on error + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateDivide(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -204,9 +236,11 @@ public static void divideChecked(int i, HiveDecimal left, HiveDecimalWritable ri // Modulo operator with overflow/zero-divide check. public static void moduloChecked(int i, HiveDecimal left, HiveDecimal right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.remainder(right)); - } catch (ArithmeticException e) { // catch on error + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateRemainder(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -214,9 +248,11 @@ public static void moduloChecked(int i, HiveDecimal left, HiveDecimal right, public static void moduloChecked(int i, HiveDecimalWritable left, HiveDecimalWritable right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.getHiveDecimal().remainder(right.getHiveDecimal())); - } catch (ArithmeticException e) { // catch on error + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateRemainder(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -224,9 +260,11 @@ public static void moduloChecked(int i, HiveDecimalWritable left, HiveDecimalWri public static void moduloChecked(int i, HiveDecimalWritable left, HiveDecimal right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.getHiveDecimal().remainder(right)); - } catch (ArithmeticException e) { // catch on error + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateRemainder(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -234,99 +272,122 @@ public static void moduloChecked(int i, HiveDecimalWritable left, HiveDecimal ri public static void moduloChecked(int i, HiveDecimal left, HiveDecimalWritable right, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, left.remainder(right.getHiveDecimal())); - } catch (ArithmeticException e) { // catch on error + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(left); + decWritable.mutateRemainder(right); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } } + // UNDONE: Why don't these methods take decimalPlaces? public static void floor(int i, HiveDecimal input, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, input.setScale(0, HiveDecimal.ROUND_FLOOR)); - } catch (ArithmeticException e) { + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(input); + decWritable.mutateRound(0, HiveDecimal.ROUND_FLOOR); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } } public static void floor(int i, HiveDecimalWritable input, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, input.getHiveDecimal().setScale(0, HiveDecimal.ROUND_FLOOR)); - } catch (ArithmeticException e) { + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(input); + decWritable.mutateRound(0, HiveDecimal.ROUND_FLOOR); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } } public static void ceiling(int i, HiveDecimal input, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, input.setScale(0, HiveDecimal.ROUND_CEILING)); - } catch (ArithmeticException e) { + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(input); + decWritable.mutateRound(0, HiveDecimal.ROUND_CEILING); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } } public static void ceiling(int i, HiveDecimalWritable input, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, input.getHiveDecimal().setScale(0, HiveDecimal.ROUND_CEILING)); - } catch (ArithmeticException e) { + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(input); + decWritable.mutateRound(0, HiveDecimal.ROUND_CEILING); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } } public static void round(int i, HiveDecimal input, int decimalPlaces, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, RoundUtils.round(input, decimalPlaces)); - } catch (ArithmeticException e) { + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(input); + decWritable.mutateRound(decimalPlaces, HiveDecimal.ROUND_HALF_UP); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } } public static void round(int i, HiveDecimalWritable input, int decimalPlaces, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, RoundUtils.round(input.getHiveDecimal(), decimalPlaces)); - } catch (ArithmeticException e) { + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(input); + decWritable.mutateRound(decimalPlaces, HiveDecimal.ROUND_HALF_UP); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } } public static void round(int i, HiveDecimal input, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, RoundUtils.round(input, outputColVector.scale)); - } catch (ArithmeticException e) { + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(input); + decWritable.mutateRound(outputColVector.scale, HiveDecimal.ROUND_HALF_UP); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } } public static void round(int i, HiveDecimalWritable input, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, RoundUtils.round(input.getHiveDecimal(), outputColVector.scale)); - } catch (ArithmeticException e) { + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(input); + decWritable.mutateRound(outputColVector.scale, HiveDecimal.ROUND_HALF_UP); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } } public static void bround(int i, HiveDecimalWritable input, int decimalPlaces, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, RoundUtils.bround(input.getHiveDecimal(), decimalPlaces)); - } catch (ArithmeticException e) { + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(input); + decWritable.mutateRound(decimalPlaces, HiveDecimal.ROUND_HALF_EVEN); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } } public static void bround(int i, HiveDecimalWritable input, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, RoundUtils.bround(input.getHiveDecimal(), outputColVector.scale)); - } catch (ArithmeticException e) { + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(input); + decWritable.mutateRound(outputColVector.scale, HiveDecimal.ROUND_HALF_EVEN); + decWritable.mutateEnforcePrecisionScale(outputColVector.precision, outputColVector.scale); + if (!decWritable.isSet()) { outputColVector.noNulls = false; outputColVector.isNull[i] = true; } @@ -337,42 +398,30 @@ public static void sign(int i, HiveDecimal input, LongColumnVector outputColVect } public static void sign(int i, HiveDecimalWritable input, LongColumnVector outputColVector) { - outputColVector.vector[i] = input.getHiveDecimal().signum(); + outputColVector.vector[i] = input.signum(); } public static void abs(int i, HiveDecimal input, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, input.abs()); - } catch (ArithmeticException e) { - outputColVector.noNulls = false; - outputColVector.isNull[i] = true; - } + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(input); + decWritable.mutateAbs(); } public static void abs(int i, HiveDecimalWritable input, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, input.getHiveDecimal().abs()); - } catch (ArithmeticException e) { - outputColVector.noNulls = false; - outputColVector.isNull[i] = true; - } + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(input); + decWritable.mutateAbs(); } public static void negate(int i, HiveDecimal input, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, input.negate()); - } catch (ArithmeticException e) { - outputColVector.noNulls = false; - outputColVector.isNull[i] = true; - } + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(input); + decWritable.mutateNegate(); } public static void negate(int i, HiveDecimalWritable input, DecimalColumnVector outputColVector) { - try { - outputColVector.set(i, input.getHiveDecimal().negate()); - } catch (ArithmeticException e) { - outputColVector.noNulls = false; - outputColVector.isNull[i] = true; - } + HiveDecimalWritable decWritable = outputColVector.vector[i]; + decWritable.set(input); + decWritable.mutateNegate(); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimalColumnInList.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimalColumnInList.java index a865343..79d3fe3 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimalColumnInList.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterDecimalColumnInList.java @@ -35,7 +35,7 @@ private HiveDecimal[] inListValues; // The set object containing the IN list. - private transient HashSet inSet; + private transient HashSet inSet; public FilterDecimalColumnInList() { super(); @@ -58,9 +58,9 @@ public void evaluate(VectorizedRowBatch batch) { } if (inSet == null) { - inSet = new HashSet(inListValues.length); + inSet = new HashSet(inListValues.length); for (HiveDecimal val : inListValues) { - inSet.add(val); + inSet.add(new HiveDecimalWritable(val)); } } @@ -81,7 +81,7 @@ public void evaluate(VectorizedRowBatch batch) { // All must be selected otherwise size would be zero // Repeating property will not change. - if (!(inSet.contains(vector[0].getHiveDecimal()))) { + if (!(inSet.contains(vector[0]))) { //Entire batch is filtered out. batch.size = 0; } @@ -89,7 +89,7 @@ public void evaluate(VectorizedRowBatch batch) { int newSize = 0; for(int j = 0; j != n; j++) { int i = sel[j]; - if (inSet.contains(vector[i].getHiveDecimal())) { + if (inSet.contains(vector[i])) { sel[newSize++] = i; } } @@ -97,7 +97,7 @@ public void evaluate(VectorizedRowBatch batch) { } else { int newSize = 0; for(int i = 0; i != n; i++) { - if (inSet.contains(vector[i].getHiveDecimal())) { + if (inSet.contains(vector[i])) { sel[newSize++] = i; } } @@ -112,7 +112,7 @@ public void evaluate(VectorizedRowBatch batch) { //All must be selected otherwise size would be zero //Repeating property will not change. if (!nullPos[0]) { - if (!inSet.contains(vector[0].getHiveDecimal())) { + if (!inSet.contains(vector[0])) { //Entire batch is filtered out. batch.size = 0; @@ -125,7 +125,7 @@ public void evaluate(VectorizedRowBatch batch) { for(int j = 0; j != n; j++) { int i = sel[j]; if (!nullPos[i]) { - if (inSet.contains(vector[i].getHiveDecimal())) { + if (inSet.contains(vector[i])) { sel[newSize++] = i; } } @@ -137,7 +137,7 @@ public void evaluate(VectorizedRowBatch batch) { int newSize = 0; for(int i = 0; i != n; i++) { if (!nullPos[i]) { - if (inSet.contains(vector[i].getHiveDecimal())) { + if (inSet.contains(vector[i])) { sel[newSize++] = i; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterStructColumnInList.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterStructColumnInList.java index 70b393c..1e21fea 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterStructColumnInList.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FilterStructColumnInList.java @@ -112,7 +112,7 @@ public void evaluate(VectorizedRowBatch batch) { case DECIMAL: DecimalColumnVector decColVector = ((DecimalColumnVector) colVec); binarySortableSerializeWrite.writeHiveDecimal( - decColVector.vector[adjustedIndex].getHiveDecimal(), decColVector.scale); + decColVector.vector[adjustedIndex], decColVector.scale); break; default: diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FuncDecimalToLong.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FuncDecimalToLong.java index 4691fe1..7f005a1 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FuncDecimalToLong.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/FuncDecimalToLong.java @@ -22,6 +22,10 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; /** * This is a superclass for unary decimal functions and expressions returning integers that @@ -32,6 +36,9 @@ int inputColumn; int outputColumn; + private transient boolean integerPrimitiveCategoryKnown = false; + protected transient PrimitiveCategory integerPrimitiveCategory; + public FuncDecimalToLong(int inputColumn, int outputColumn) { this.inputColumn = inputColumn; this.outputColumn = outputColumn; @@ -50,6 +57,13 @@ public void evaluate(VectorizedRowBatch batch) { super.evaluateChildren(batch); } + if (!integerPrimitiveCategoryKnown) { + String typeName = getOutputType().toLowerCase(); + TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName); + integerPrimitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); + integerPrimitiveCategoryKnown = true; + } + DecimalColumnVector inV = (DecimalColumnVector) batch.cols[inputColumn]; int[] sel = batch.selected; int n = batch.size; @@ -117,11 +131,6 @@ public int getOutputColumn() { } @Override - public String getOutputType() { - return "long"; - } - - @Override public VectorExpressionDescriptor.Descriptor getDescriptor() { VectorExpressionDescriptor.Builder b = new VectorExpressionDescriptor.Builder(); b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StructColumnInList.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StructColumnInList.java index 769c70a..8134108 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StructColumnInList.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/StructColumnInList.java @@ -113,7 +113,7 @@ public void evaluate(VectorizedRowBatch batch) { case DECIMAL: DecimalColumnVector decColVector = ((DecimalColumnVector) colVec); binarySortableSerializeWrite.writeHiveDecimal( - decColVector.vector[adjustedIndex].getHiveDecimal(), decColVector.scale); + decColVector.vector[adjustedIndex], decColVector.scale); break; default: diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgDecimal.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgDecimal.java index d0ff5fa..4f6d652 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgDecimal.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/aggregates/VectorUDAFAvgDecimal.java @@ -60,45 +60,25 @@ transient private long count; transient private boolean isNull; - // We use this to catch overflow. - transient private boolean isOutOfRange; - - public void sumValueWithNullCheck(HiveDecimalWritable writable, short scale) { - if (isOutOfRange) { - return; - } - HiveDecimal value = writable.getHiveDecimal(); + public void sumValueWithNullCheck(HiveDecimalWritable writable) { if (isNull) { - sum.set(value); + // Make a copy since we intend to mutate sum. + sum.set(writable); count = 1; isNull = false; } else { - HiveDecimal result; - try { - result = sum.getHiveDecimal().add(value); - } catch (ArithmeticException e) { // catch on overflow - isOutOfRange = true; - return; - } - sum.set(result); + // Note that if sum is out of range, mutateAdd will ignore the call. + // At the end, sum.isSet() can be checked for null. + sum.mutateAdd(writable); count++; } } - public void sumValueNoNullCheck(HiveDecimalWritable writable, short scale) { - HiveDecimal value = writable.getHiveDecimal(); - HiveDecimal result; - try { - result = sum.getHiveDecimal().add(value); - } catch (ArithmeticException e) { // catch on overflow - isOutOfRange = true; - return; - } - sum.set(result); + public void sumValueNoNullCheck(HiveDecimalWritable writable) { + sum.mutateAdd(writable); count++; } - @Override public int getVariableSize() { throw new UnsupportedOperationException(); @@ -107,9 +87,8 @@ public int getVariableSize() { @Override public void reset() { isNull = true; - isOutOfRange = false; - sum.set(HiveDecimal.ZERO); - count = 0L; + sum.setFromLong(0L); + count = 0; } } @@ -251,7 +230,7 @@ private void iterateNoNullsRepeatingWithAggregationSelection( aggregationBufferSets, bufferIndex, i); - myagg.sumValueWithNullCheck(value, this.sumScale); + myagg.sumValueWithNullCheck(value); } } @@ -267,7 +246,7 @@ private void iterateNoNullsSelectionWithAggregationSelection( aggregationBufferSets, bufferIndex, i); - myagg.sumValueWithNullCheck(values[selection[i]], this.sumScale); + myagg.sumValueWithNullCheck(values[selection[i]]); } } @@ -281,7 +260,7 @@ private void iterateNoNullsWithAggregationSelection( aggregationBufferSets, bufferIndex, i); - myagg.sumValueWithNullCheck(values[i], this.sumScale); + myagg.sumValueWithNullCheck(values[i]); } } @@ -302,7 +281,7 @@ private void iterateHasNullsRepeatingSelectionWithAggregationSelection( aggregationBufferSets, bufferIndex, i); - myagg.sumValueWithNullCheck(value, this.sumScale); + myagg.sumValueWithNullCheck(value); } } @@ -323,7 +302,7 @@ private void iterateHasNullsRepeatingWithAggregationSelection( aggregationBufferSets, bufferIndex, i); - myagg.sumValueWithNullCheck(value, this.sumScale); + myagg.sumValueWithNullCheck(value); } } @@ -342,7 +321,7 @@ private void iterateHasNullsSelectionWithAggregationSelection( aggregationBufferSets, bufferIndex, j); - myagg.sumValueWithNullCheck(values[i], this.sumScale); + myagg.sumValueWithNullCheck(values[i]); } } } @@ -360,7 +339,7 @@ private void iterateHasNullsWithAggregationSelection( aggregationBufferSets, bufferIndex, i); - myagg.sumValueWithNullCheck(values[i], this.sumScale); + myagg.sumValueWithNullCheck(values[i]); } } } @@ -389,25 +368,12 @@ public void aggregateInput(AggregationBuffer agg, VectorizedRowBatch batch) if (inputVector.noNulls) { if (myagg.isNull) { myagg.isNull = false; - myagg.sum.set(HiveDecimal.ZERO); + myagg.sum.setFromLong(0L); myagg.count = 0; } HiveDecimal value = vector[0].getHiveDecimal(); - HiveDecimal multiple; - try { - multiple = value.multiply(HiveDecimal.create(batchSize)); - } catch (ArithmeticException e) { // catch on overflow - myagg.isOutOfRange = true; - return; - } - HiveDecimal result; - try { - result = myagg.sum.getHiveDecimal().add(multiple); - } catch (ArithmeticException e) { // catch on overflow - myagg.isOutOfRange = true; - return; - } - myagg.sum.set(result); + HiveDecimal multiple = value.multiply(HiveDecimal.create(batchSize)); + myagg.sum.mutateAdd(multiple); myagg.count += batchSize; } return; @@ -437,8 +403,7 @@ private void iterateSelectionHasNulls( for (int j=0; j< batchSize; ++j) { int i = selected[j]; if (!isNull[i]) { - HiveDecimalWritable value = vector[i]; - myagg.sumValueWithNullCheck(value, this.sumScale); + myagg.sumValueWithNullCheck(vector[i]); } } } @@ -451,13 +416,12 @@ private void iterateSelectionNoNulls( if (myagg.isNull) { myagg.isNull = false; - myagg.sum.set(HiveDecimal.ZERO); + myagg.sum.setFromLong(0L); myagg.count = 0; } for (int i=0; i< batchSize; ++i) { - HiveDecimalWritable value = vector[selected[i]]; - myagg.sumValueNoNullCheck(value, this.sumScale); + myagg.sumValueNoNullCheck(vector[selected[i]]); } } @@ -469,8 +433,7 @@ private void iterateNoSelectionHasNulls( for(int i=0;i 0) { readRepetitionAndDefinitionLevels(); if (definitionLevel >= maxDefLevel) { - c.vector[rowId].set(dataColumn.readBytes().getBytesUnsafe(), c.scale); + c.vector[rowId].setFromBigIntegerBytesAndScale(dataColumn.readBytes().getBytesUnsafe(), c.scale); c.isNull[rowId] = false; - c.isRepeating = c.isRepeating && (c.vector[0] == c.vector[rowId]); + c.isRepeating = c.isRepeating && c.vector[0].equals(c.vector[rowId]); } else { c.isNull[rowId] = true; c.isRepeating = false; @@ -435,7 +435,7 @@ private void decodeDictionaryIds( decimalColumnVector.scale = (short) type.asPrimitiveType().getDecimalMetadata().getScale(); for (int i = rowId; i < rowId + num; ++i) { decimalColumnVector.vector[i] - .set(dictionary.decodeToBinary((int) dictionaryIds.vector[i]).getBytesUnsafe(), + .setFromBigIntegerBytesAndScale(dictionary.decodeToBinary((int) dictionaryIds.vector[i]).getBytesUnsafe(), decimalColumnVector.scale); } } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java index 1e26c19..6b7b50a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/write/DataWritableWriter.java @@ -517,7 +517,8 @@ public void write(Object value) { private Binary decimalToBinary(final HiveDecimal hiveDecimal, final DecimalTypeInfo decimalTypeInfo) { int prec = decimalTypeInfo.precision(); int scale = decimalTypeInfo.scale(); - byte[] decimalBytes = hiveDecimal.setScale(scale).unscaledValue().toByteArray(); + + byte[] decimalBytes = hiveDecimal.bigIntegerBytesScaled(scale); // Estimated number of bytes needed. int precToBytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1]; diff --git ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java index 9013084..9d900e4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/sarg/ConvertAstToSearchArg.java @@ -141,7 +141,11 @@ private static Object boxLiteral(ExprNodeConstantDesc constantDesc, switch (type) { case LONG: if (lit instanceof HiveDecimal) { - return ((HiveDecimal)lit).longValueExact(); + HiveDecimal dec = (HiveDecimal) lit; + if (!dec.isLong()) { + throw new ArithmeticException("Overflow"); + } + return dec.longValue(); } return ((Number) lit).longValue(); case STRING: diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index ace3eaf..cfe9668 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -337,7 +337,7 @@ public static ExprNodeConstantDesc createDecimal(String strVal, boolean notNull) int prec = 1; int scale = 0; if (hd != null) { - prec = hd.precision(); + prec = hd.sqlPrecision(); scale = hd.scale(); } DecimalTypeInfo typeInfo = TypeInfoFactory.getDecimalTypeInfo(prec, scale); diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java index 31e2878..d214a96 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFLog.java @@ -59,8 +59,8 @@ public DoubleWritable evaluate(HiveDecimalWritable baseWritable, HiveDecimalWrit return null; } - double base = baseWritable.getHiveDecimal().bigDecimalValue().doubleValue(); - double d = writable.getHiveDecimal().bigDecimalValue().doubleValue(); + double base = baseWritable.doubleValue(); + double d = writable.doubleValue(); return log(base, d); } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMath.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMath.java index 8087df1..378b3d7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMath.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMath.java @@ -54,7 +54,7 @@ public final DoubleWritable evaluate(HiveDecimalWritable writable) { return null; } - double d = writable.getHiveDecimal().bigDecimalValue().doubleValue(); + double d = writable.doubleValue(); doubleWritable.set(d); return doEvaluate(doubleWritable); } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSign.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSign.java index 67d62d9..449848a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSign.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSign.java @@ -79,12 +79,12 @@ public DoubleWritable evaluate(LongWritable a) { * * @return -1, 0, or 1 representing the sign of the input decimal */ - public IntWritable evaluate(HiveDecimalWritable dec) { - if (dec == null || dec.getHiveDecimal() == null) { + public IntWritable evaluate(HiveDecimalWritable decWritable) { + if (decWritable == null || !decWritable.isSet()) { return null; } - intWritable.set(dec.getHiveDecimal().signum()); + intWritable.set(decWritable.signum()); return intWritable; } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java index 17b892c..0cc0c9e 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java @@ -191,10 +191,10 @@ public BooleanWritable evaluate(TimestampWritable i) { } public BooleanWritable evaluate(HiveDecimalWritable i) { - if (i == null) { + if (i == null || !i.isSet()) { return null; } else { - booleanWritable.set(HiveDecimal.ZERO.compareTo(i.getHiveDecimal()) != 0); + booleanWritable.set(i.compareTo(HiveDecimal.ZERO) != 0); return booleanWritable; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java index efae82d..df2b42f 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java @@ -192,10 +192,10 @@ public ByteWritable evaluate(TimestampWritable i) { } public ByteWritable evaluate(HiveDecimalWritable i) { - if (i == null) { + if (i == null || !i.isSet() || !i.isByte()) { return null; } else { - byteWritable.set(i.getHiveDecimal().byteValue()); + byteWritable.set(i.byteValue()); return byteWritable; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java index 9cbc114..5fcae42 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java @@ -195,10 +195,10 @@ public DoubleWritable evaluate(TimestampWritable i) { } public DoubleWritable evaluate(HiveDecimalWritable i) { - if (i == null) { + if (i == null || !i.isSet()) { return null; } else { - doubleWritable.set(i.getHiveDecimal().doubleValue()); + doubleWritable.set(i.doubleValue()); return doubleWritable; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java index 5808c90..c8e32f4 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java @@ -196,10 +196,10 @@ public FloatWritable evaluate(TimestampWritable i) { } public FloatWritable evaluate(HiveDecimalWritable i) { - if (i == null) { + if (i == null || !i.isSet()) { return null; } else { - floatWritable.set(i.getHiveDecimal().floatValue()); + floatWritable.set(i.floatValue()); return floatWritable; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java index a7551cb..42972c7 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java @@ -201,10 +201,10 @@ public IntWritable evaluate(TimestampWritable i) { } public IntWritable evaluate(HiveDecimalWritable i) { - if (i == null) { + if (i == null || !i.isSet() || !i.isInt()) { return null; } else { - intWritable.set(i.getHiveDecimal().intValue()); // TODO: lossy conversion! + intWritable.set(i.intValue()); return intWritable; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java index c961d14..847b535 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java @@ -204,10 +204,10 @@ public LongWritable evaluate(TimestampWritable i) { } public LongWritable evaluate(HiveDecimalWritable i) { - if (i == null) { + if (i == null || !i.isSet() || !i.isLong()) { return null; } else { - longWritable.set(i.getHiveDecimal().longValue()); // TODO: lossy conversion! + longWritable.set(i.longValue()); return longWritable; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java index 570408a..0ae0c13 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java @@ -194,10 +194,10 @@ public ShortWritable evaluate(TimestampWritable i) { } public ShortWritable evaluate(HiveDecimalWritable i) { - if (i == null) { + if (i == null || !i.isShort() || !i.isShort()) { return null; } else { - shortWritable.set(i.getHiveDecimal().shortValue()); // TODO: lossy conversion! + shortWritable.set(i.shortValue()); return shortWritable; } } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java index 38269f4..e2cd213 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFSum.java @@ -206,7 +206,7 @@ protected boolean isEligibleValue(SumAgg agg, Object input) { public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveException { assert (parameters.length == 1); super.init(m, parameters); - result = new HiveDecimalWritable(HiveDecimal.ZERO); + result = new HiveDecimalWritable(0); inputOI = (PrimitiveObjectInspector) parameters[0]; // The output precision is 10 greater than the input which should cover at least // 10b rows. The scale is the same as the input. @@ -226,21 +226,21 @@ public ObjectInspector init(Mode m, ObjectInspector[] parameters) throws HiveExc /** class for storing decimal sum value. */ @AggregationType(estimable = false) // hard to know exactly for decimals - static class SumHiveDecimalAgg extends SumAgg { + static class SumHiveDecimalWritableAgg extends SumAgg { } @Override public AggregationBuffer getNewAggregationBuffer() throws HiveException { - SumHiveDecimalAgg agg = new SumHiveDecimalAgg(); + SumHiveDecimalWritableAgg agg = new SumHiveDecimalWritableAgg(); reset(agg); return agg; } @Override public void reset(AggregationBuffer agg) throws HiveException { - SumAgg bdAgg = (SumAgg) agg; + SumAgg bdAgg = (SumAgg) agg; bdAgg.empty = true; - bdAgg.sum = HiveDecimal.ZERO; + bdAgg.sum = new HiveDecimalWritable(0); bdAgg.uniqueObjects = new HashSet(); } @@ -250,9 +250,9 @@ public void reset(AggregationBuffer agg) throws HiveException { public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveException { assert (parameters.length == 1); try { - if (isEligibleValue((SumHiveDecimalAgg) agg, parameters[0])) { - ((SumHiveDecimalAgg)agg).empty = false; - ((SumHiveDecimalAgg)agg).sum = ((SumHiveDecimalAgg)agg).sum.add( + if (isEligibleValue((SumHiveDecimalWritableAgg) agg, parameters[0])) { + ((SumHiveDecimalWritableAgg)agg).empty = false; + ((SumHiveDecimalWritableAgg)agg).sum.mutateAdd( PrimitiveObjectInspectorUtils.getHiveDecimal(parameters[0], inputOI)); } } catch (NumberFormatException e) { @@ -270,8 +270,8 @@ public void iterate(AggregationBuffer agg, Object[] parameters) throws HiveExcep @Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if (partial != null) { - SumHiveDecimalAgg myagg = (SumHiveDecimalAgg) agg; - if (myagg.sum == null) { + SumHiveDecimalWritableAgg myagg = (SumHiveDecimalWritableAgg) agg; + if (myagg.sum == null || !myagg.sum.isSet()) { return; } @@ -279,22 +279,22 @@ public void merge(AggregationBuffer agg, Object partial) throws HiveException { if (isWindowingDistinct()) { throw new HiveException("Distinct windowing UDAF doesn't support merge and terminatePartial"); } else { - myagg.sum = myagg.sum.add(PrimitiveObjectInspectorUtils.getHiveDecimal(partial, inputOI)); + myagg.sum.mutateAdd(PrimitiveObjectInspectorUtils.getHiveDecimal(partial, inputOI)); } } } @Override public Object terminate(AggregationBuffer agg) throws HiveException { - SumHiveDecimalAgg myagg = (SumHiveDecimalAgg) agg; - if (myagg.empty || myagg.sum == null) { + SumHiveDecimalWritableAgg myagg = (SumHiveDecimalWritableAgg) agg; + if (myagg.empty || myagg.sum == null || !myagg.sum.isSet()) { return null; } - if (myagg.sum != null) { - if (HiveDecimalUtils.enforcePrecisionScale(myagg.sum, (DecimalTypeInfo)outputOI.getTypeInfo()) == null) { - LOG.warn("The sum of a column with data type HiveDecimal is out of range"); - return null; - } + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo)outputOI.getTypeInfo(); + myagg.sum.mutateEnforcePrecisionScale(decimalTypeInfo.getPrecision(), decimalTypeInfo.getScale()); + if (!myagg.sum.isSet()) { + LOG.warn("The sum of a column with data type HiveDecimal is out of range"); + return null; } result.set(myagg.sum); @@ -315,8 +315,8 @@ public GenericUDAFEvaluator getWindowingEvaluator(WindowFrameDef wFrameDef) { protected HiveDecimalWritable getNextResult( org.apache.hadoop.hive.ql.udf.generic.GenericUDAFStreamingEvaluator.SumAvgEnhancer.SumAvgStreamingState ss) throws HiveException { - SumHiveDecimalAgg myagg = (SumHiveDecimalAgg) ss.wrappedBuf; - HiveDecimal r = myagg.empty ? null : myagg.sum; + SumHiveDecimalWritableAgg myagg = (SumHiveDecimalWritableAgg) ss.wrappedBuf; + HiveDecimal r = myagg.empty ? null : myagg.sum.getHiveDecimal(); HiveDecimal d = ss.retrieveNextIntermediateValue(); if (d != null ) { r = r == null ? null : r.subtract(d); @@ -329,8 +329,8 @@ protected HiveDecimalWritable getNextResult( protected HiveDecimal getCurrentIntermediateResult( org.apache.hadoop.hive.ql.udf.generic.GenericUDAFStreamingEvaluator.SumAvgEnhancer.SumAvgStreamingState ss) throws HiveException { - SumHiveDecimalAgg myagg = (SumHiveDecimalAgg) ss.wrappedBuf; - return myagg.empty ? null : myagg.sum; + SumHiveDecimalWritableAgg myagg = (SumHiveDecimalWritableAgg) ss.wrappedBuf; + return myagg.empty ? null : myagg.sum.getHiveDecimal(); } }; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAbs.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAbs.java index a8e2786..bf1fed0 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAbs.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFAbs.java @@ -141,7 +141,8 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { HiveDecimalWritable val = decimalOI.getPrimitiveWritableObject(valObject); if (val != null) { - resultDecimal.set(val.getHiveDecimal().abs()); + resultDecimal.set(val); + resultDecimal.mutateAbs(); val = resultDecimal; } return val; diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBRound.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBRound.java index 4a59eb3..df3f822 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBRound.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBRound.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncBRoundDecimalToDecimal; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncBRoundDoubleToDouble; import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; @Description(name = "bround", value = "_FUNC_(x[, d]) - round x to d decimal places using HALF_EVEN rounding mode.", @@ -37,8 +38,10 @@ public class GenericUDFBRound extends GenericUDFRound { @Override - protected HiveDecimal round(HiveDecimal input, int scale) { - return RoundUtils.bround(input, scale); + protected HiveDecimalWritable round(HiveDecimalWritable inputDecWritable, int scale) { + HiveDecimalWritable result = new HiveDecimalWritable(inputDecWritable); + result.mutateRound(scale, HiveDecimal.ROUND_HALF_EVEN); + return result; } @Override diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCeil.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCeil.java index 95ec32e..fb040fc 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCeil.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCeil.java @@ -50,8 +50,8 @@ protected LongWritable evaluate(DoubleWritable input) { @Override protected HiveDecimalWritable evaluate(HiveDecimalWritable input) { - HiveDecimal bd = input.getHiveDecimal(); - decimalWritable.set(bd.setScale(0, HiveDecimal.ROUND_CEILING)); + decimalWritable.set(input); + decimalWritable.mutateRound(0, HiveDecimal.ROUND_CEILING); return decimalWritable; } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFloor.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFloor.java index 8ad15e9..1c6ba62 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFloor.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFloor.java @@ -50,8 +50,8 @@ protected LongWritable evaluate(DoubleWritable input) { @Override protected HiveDecimalWritable evaluate(HiveDecimalWritable input) { - HiveDecimal bd = input.getHiveDecimal(); - decimalWritable.set(bd.setScale(0, HiveDecimal.ROUND_FLOOR)); + decimalWritable.set(input); + decimalWritable.mutateRound(0, HiveDecimal.ROUND_FLOOR); return decimalWritable; } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNegative.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNegative.java index de964d6..45a3504 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNegative.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNegative.java @@ -82,8 +82,8 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { doubleWritable.set(-(((DoubleWritable)input).get())); return doubleWritable; case DECIMAL: - HiveDecimal dec = ((HiveDecimalWritable)input).getHiveDecimal(); - decimalWritable.set(dec.negate()); + decimalWritable.set((HiveDecimalWritable)input); + decimalWritable.mutateNegate(); return decimalWritable; case INTERVAL_YEAR_MONTH: HiveIntervalYearMonth intervalYearMonth = diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNumericMinus.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNumericMinus.java index a31cf78..9413a4e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNumericMinus.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNumericMinus.java @@ -78,11 +78,8 @@ protected DoubleWritable evaluate(DoubleWritable left, DoubleWritable right) { @Override protected HiveDecimalWritable evaluate(HiveDecimal left, HiveDecimal right) { - HiveDecimal dec = left.subtract(right); - if (dec == null) { - return null; - } - decimalWritable.set(dec); + decimalWritable.set(left); + decimalWritable.mutateSubtract(right); return decimalWritable; } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNumericPlus.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNumericPlus.java index b055776..269f6de 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNumericPlus.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNumericPlus.java @@ -87,13 +87,8 @@ protected DoubleWritable evaluate(DoubleWritable left, DoubleWritable right) { @Override protected HiveDecimalWritable evaluate(HiveDecimal left, HiveDecimal right) { - HiveDecimal dec = left.add(right); - - if (dec == null) { - return null; - } - - decimalWritable.set(dec); + decimalWritable.set(left); + decimalWritable.mutateAdd(right); return decimalWritable; } diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java index ae81fe3..f458932 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRound.java @@ -201,12 +201,12 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { case VOID: return null; case DECIMAL: - HiveDecimalWritable decimalWritable = (HiveDecimalWritable) inputOI.getPrimitiveWritableObject(input); - HiveDecimal dec = round(decimalWritable.getHiveDecimal(), scale); - if (dec == null) { - return null; + { + // The getPrimitiveWritableObject method returns a new writable. + HiveDecimalWritable decimalWritable = (HiveDecimalWritable) inputOI.getPrimitiveWritableObject(input); + // Call the different round flavor. + return round(decimalWritable, scale); } - return new HiveDecimalWritable(dec); case BYTE: ByteWritable byteWritable = (ByteWritable)inputOI.getPrimitiveWritableObject(input); if (scale >= 0) { @@ -255,8 +255,10 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { } } - protected HiveDecimal round(HiveDecimal input, int scale) { - return RoundUtils.round(input, scale); + protected HiveDecimalWritable round(HiveDecimalWritable inputDecWritable, int scale) { + HiveDecimalWritable result = new HiveDecimalWritable(inputDecWritable); + result.mutateRound(scale, HiveDecimal.ROUND_HALF_UP); + return result; } protected long round(long input, int scale) { diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/generic/RoundUtils.java ql/src/java/org/apache/hadoop/hive/ql/udf/generic/RoundUtils.java index 7fd1641..9743d6e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/generic/RoundUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/generic/RoundUtils.java @@ -65,10 +65,10 @@ public static long bround(long input, int scale) { } public static HiveDecimal round(HiveDecimal input, int scale) { - return input.setScale(scale, HiveDecimal.ROUND_HALF_UP); + return input.round(scale, HiveDecimal.ROUND_HALF_UP); } public static HiveDecimal bround(HiveDecimal input, int scale) { - return input.setScale(scale, HiveDecimal.ROUND_HALF_EVEN); + return input.round(scale, HiveDecimal.ROUND_HALF_EVEN); } } diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java index f5b5d9d..0ddebf8 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java @@ -732,12 +732,12 @@ public void testSumDecimalHive6508() throws HiveException { "sum", 4, Arrays.asList(new Object[]{ - HiveDecimal.create("1234.2401").setScale(scale), - HiveDecimal.create("1868.52").setScale(scale), - HiveDecimal.ZERO.setScale(scale), - HiveDecimal.create("456.84").setScale(scale), - HiveDecimal.create("121.89").setScale(scale)}), - HiveDecimal.create("3681.4901").setScale( scale)); + HiveDecimal.create("1234.2401"), + HiveDecimal.create("1868.52"), + HiveDecimal.ZERO, + HiveDecimal.create("456.84"), + HiveDecimal.create("121.89")}), + HiveDecimal.create("3681.4901")); } @Test diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java index 8ffff9d..b29bb8b 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java @@ -97,7 +97,9 @@ void deserializeAndVerify(Output output, DeserializeRead deserializeRead, PrimitiveCategory primitiveCategory = primitiveCategories[i]; PrimitiveTypeInfo primitiveTypeInfo = source.primitiveTypeInfos()[i]; if (!deserializeRead.readNextField()) { - throw new HiveException("Unexpected NULL"); + throw new HiveException("Unexpected NULL when reading primitiveCategory " + primitiveCategory + + " expected (" + expected.getClass().getName() + ", " + expected.toString() + ") " + + " deserializeRead " + deserializeRead.getClass().getName()); } switch (primitiveCategory) { case BOOLEAN: @@ -307,7 +309,7 @@ void serializeBatch(VectorizedRowBatch batch, VectorSerializeRow vectorSerialize } } - void testVectorSerializeRow(int caseNum, Random r, SerializationType serializationType) + void testVectorSerializeRow(Random r, SerializationType serializationType) throws HiveException, IOException, SerDeException { String[] emptyScratchTypeNames = new String[0]; @@ -383,7 +385,9 @@ void examineBatch(VectorizedRowBatch batch, VectorExtractRow vectorExtractRow, Object rowObj = row[c]; Object expectedObj = expectedRow[c]; if (rowObj == null) { - fail("Unexpected NULL from extractRow"); + fail("Unexpected NULL from extractRow. Expected class " + + expectedObj.getClass().getName() + " value " + expectedObj.toString() + + " batch index " + i + " firstRandomRowIndex " + firstRandomRowIndex); } if (!rowObj.equals(expectedObj)) { fail("Row " + (firstRandomRowIndex + i) + " and column " + c + " mismatch (" + primitiveTypeInfos[c].getPrimitiveCategory() + " actual value " + rowObj + " and expected value " + expectedObj + ")"); @@ -541,7 +545,7 @@ private LazySerDeParameters getSerDeParams(Configuration conf, Properties tbl, S return new LazySerDeParameters(conf, tbl, LazySimpleSerDe.class.getName()); } - void testVectorDeserializeRow(int caseNum, Random r, SerializationType serializationType, + void testVectorDeserializeRow(Random r, SerializationType serializationType, boolean alternate1, boolean alternate2, boolean useExternalBuffer) throws HiveException, IOException, SerDeException { @@ -689,113 +693,111 @@ void testVectorDeserializeRow(int caseNum, Random r, SerializationType serializa } } - public void testVectorSerDeRow() throws Throwable { - - Random r = new Random(5678); - - int c = 0; - - /* - * SERIALIZE tests. - */ - testVectorSerializeRow(c++, r, SerializationType.BINARY_SORTABLE); - - testVectorSerializeRow(c++, r, SerializationType.LAZY_BINARY); - - testVectorSerializeRow(c++, r, SerializationType.LAZY_SIMPLE); - - /* - * DESERIALIZE tests. - */ - - // BINARY_SORTABLE - - testVectorDeserializeRow(c++, r, - SerializationType.BINARY_SORTABLE, - /* alternate1 = useColumnSortOrderIsDesc */ false, - /* alternate2 = useBinarySortableCharsNeedingEscape */ false, - /* useExternalBuffer */ false); - - testVectorDeserializeRow(c++, r, - SerializationType.BINARY_SORTABLE, - /* alternate1 = useColumnSortOrderIsDesc */ true, - /* alternate2 = useBinarySortableCharsNeedingEscape */ false, - /* useExternalBuffer */ false); - - testVectorDeserializeRow(c++, r, - SerializationType.BINARY_SORTABLE, - /* alternate1 = useColumnSortOrderIsDesc */ false, - /* alternate2 = useBinarySortableCharsNeedingEscape */ false, - /* useExternalBuffer */ true); - - testVectorDeserializeRow(c++, r, - SerializationType.BINARY_SORTABLE, - /* alternate1 = useColumnSortOrderIsDesc */ true, - /* alternate2 = useBinarySortableCharsNeedingEscape */ false, - /* useExternalBuffer */ true); - - testVectorDeserializeRow(c++, r, - SerializationType.BINARY_SORTABLE, - /* alternate1 = useColumnSortOrderIsDesc */ false, - /* alternate2 = useBinarySortableCharsNeedingEscape */ true, - /* useExternalBuffer */ false); - - testVectorDeserializeRow(c++, r, - SerializationType.BINARY_SORTABLE, - /* alternate1 = useColumnSortOrderIsDesc */ true, - /* alternate2 = useBinarySortableCharsNeedingEscape */ true, - /* useExternalBuffer */ false); - - testVectorDeserializeRow(c++, r, - SerializationType.BINARY_SORTABLE, - /* alternate1 = useColumnSortOrderIsDesc */ false, - /* alternate2 = useBinarySortableCharsNeedingEscape */ true, - /* useExternalBuffer */ true); - - testVectorDeserializeRow(c++, r, - SerializationType.BINARY_SORTABLE, - /* alternate1 = useColumnSortOrderIsDesc */ true, - /* alternate2 = useBinarySortableCharsNeedingEscape */ true, - /* useExternalBuffer */ true); - - // LAZY_BINARY - - testVectorDeserializeRow(c++, r, - SerializationType.LAZY_BINARY, - /* alternate1 = unused */ false, - /* alternate2 = unused */ false, - /* useExternalBuffer */ false); - - testVectorDeserializeRow(c++, r, - SerializationType.LAZY_BINARY, - /* alternate1 = unused */ false, - /* alternate2 = unused */ false, - /* useExternalBuffer */ true); - - // LAZY_SIMPLE - - testVectorDeserializeRow(c++, r, - SerializationType.LAZY_SIMPLE, - /* alternate1 = useLazySimpleEscapes */ false, - /* alternate2 = unused */ false, - /* useExternalBuffer */ false); - - testVectorDeserializeRow(c++, r, - SerializationType.LAZY_SIMPLE, - /* alternate1 = useLazySimpleEscapes */ false, - /* alternate2 = unused */ false, - /* useExternalBuffer */ true); - - testVectorDeserializeRow(c++, r, - SerializationType.LAZY_SIMPLE, - /* alternate1 = useLazySimpleEscapes */ true, - /* alternate2 = unused */ false, - /* useExternalBuffer */ false); - - testVectorDeserializeRow(c++, r, - SerializationType.LAZY_SIMPLE, - /* alternate1 = useLazySimpleEscapes */ true, - /* alternate2 = unused */ false, - /* useExternalBuffer */ true); + public void testVectorBinarySortableSerializeRow() throws Throwable { + Random r = new Random(8732); + testVectorSerializeRow(r, SerializationType.BINARY_SORTABLE); + } + + public void testVectorLazyBinarySerializeRow() throws Throwable { + Random r = new Random(8732); + testVectorSerializeRow(r, SerializationType.LAZY_BINARY); + } + + public void testVectorLazySimpleSerializeRow() throws Throwable { + Random r = new Random(8732); + testVectorSerializeRow(r, SerializationType.LAZY_SIMPLE); + } + + public void testVectorBinarySortableDeserializeRow() throws Throwable { + Random r = new Random(8732); + testVectorDeserializeRow(r, + SerializationType.BINARY_SORTABLE, + /* alternate1 = useColumnSortOrderIsDesc */ false, + /* alternate2 = useBinarySortableCharsNeedingEscape */ false, + /* useExternalBuffer */ false); + + testVectorDeserializeRow(r, + SerializationType.BINARY_SORTABLE, + /* alternate1 = useColumnSortOrderIsDesc */ true, + /* alternate2 = useBinarySortableCharsNeedingEscape */ false, + /* useExternalBuffer */ false); + + testVectorDeserializeRow(r, + SerializationType.BINARY_SORTABLE, + /* alternate1 = useColumnSortOrderIsDesc */ false, + /* alternate2 = useBinarySortableCharsNeedingEscape */ false, + /* useExternalBuffer */ true); + + testVectorDeserializeRow(r, + SerializationType.BINARY_SORTABLE, + /* alternate1 = useColumnSortOrderIsDesc */ true, + /* alternate2 = useBinarySortableCharsNeedingEscape */ false, + /* useExternalBuffer */ true); + + testVectorDeserializeRow(r, + SerializationType.BINARY_SORTABLE, + /* alternate1 = useColumnSortOrderIsDesc */ false, + /* alternate2 = useBinarySortableCharsNeedingEscape */ true, + /* useExternalBuffer */ false); + + testVectorDeserializeRow(r, + SerializationType.BINARY_SORTABLE, + /* alternate1 = useColumnSortOrderIsDesc */ true, + /* alternate2 = useBinarySortableCharsNeedingEscape */ true, + /* useExternalBuffer */ false); + + testVectorDeserializeRow(r, + SerializationType.BINARY_SORTABLE, + /* alternate1 = useColumnSortOrderIsDesc */ false, + /* alternate2 = useBinarySortableCharsNeedingEscape */ true, + /* useExternalBuffer */ true); + + testVectorDeserializeRow(r, + SerializationType.BINARY_SORTABLE, + /* alternate1 = useColumnSortOrderIsDesc */ true, + /* alternate2 = useBinarySortableCharsNeedingEscape */ true, + /* useExternalBuffer */ true); + } + + public void testVectorLazyBinaryDeserializeRow() throws Throwable { + Random r = new Random(8732); + testVectorDeserializeRow(r, + SerializationType.LAZY_BINARY, + /* alternate1 = unused */ false, + /* alternate2 = unused */ false, + /* useExternalBuffer */ false); + + testVectorDeserializeRow(r, + SerializationType.LAZY_BINARY, + /* alternate1 = unused */ false, + /* alternate2 = unused */ false, + /* useExternalBuffer */ true); + } + + public void testVectorLazySimpleDeserializeRow() throws Throwable { + Random r = new Random(8732); + testVectorDeserializeRow(r, + SerializationType.LAZY_SIMPLE, + /* alternate1 = useLazySimpleEscapes */ false, + /* alternate2 = unused */ false, + /* useExternalBuffer */ false); + + testVectorDeserializeRow(r, + SerializationType.LAZY_SIMPLE, + /* alternate1 = useLazySimpleEscapes */ false, + /* alternate2 = unused */ false, + /* useExternalBuffer */ true); + + testVectorDeserializeRow(r, + SerializationType.LAZY_SIMPLE, + /* alternate1 = useLazySimpleEscapes */ true, + /* alternate2 = unused */ false, + /* useExternalBuffer */ false); + + testVectorDeserializeRow(r, + SerializationType.LAZY_SIMPLE, + /* alternate1 = useLazySimpleEscapes */ true, + /* alternate2 = unused */ false, + /* useExternalBuffer */ true); } } \ No newline at end of file diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java index 57bf60d..cbde615 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Random; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.common.type.HiveChar; @@ -336,27 +337,28 @@ public static Object getWritableObject(int column, Object object, { WritableHiveCharObjectInspector writableCharObjectInspector = new WritableHiveCharObjectInspector( (CharTypeInfo) primitiveTypeInfo); - return writableCharObjectInspector.create(new HiveChar(StringUtils.EMPTY, -1)); + return writableCharObjectInspector.create((HiveChar) object); } case VARCHAR: { WritableHiveVarcharObjectInspector writableVarcharObjectInspector = new WritableHiveVarcharObjectInspector( (VarcharTypeInfo) primitiveTypeInfo); - return writableVarcharObjectInspector.create(new HiveVarchar(StringUtils.EMPTY, -1)); + return writableVarcharObjectInspector.create((HiveVarchar) object); } case BINARY: - return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector.create(ArrayUtils.EMPTY_BYTE_ARRAY); + return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector.create((byte[]) object); case TIMESTAMP: - return ((WritableTimestampObjectInspector) objectInspector).create(new Timestamp(0)); + return ((WritableTimestampObjectInspector) objectInspector).create((Timestamp) object); case INTERVAL_YEAR_MONTH: - return ((WritableHiveIntervalYearMonthObjectInspector) objectInspector).create(new HiveIntervalYearMonth(0)); + return ((WritableHiveIntervalYearMonthObjectInspector) objectInspector).create((HiveIntervalYearMonth) object); case INTERVAL_DAY_TIME: - return ((WritableHiveIntervalDayTimeObjectInspector) objectInspector).create(new HiveIntervalDayTime(0, 0)); + return ((WritableHiveIntervalDayTimeObjectInspector) objectInspector).create((HiveIntervalDayTime) object); case DECIMAL: { WritableHiveDecimalObjectInspector writableDecimalObjectInspector = new WritableHiveDecimalObjectInspector((DecimalTypeInfo) primitiveTypeInfo); - return writableDecimalObjectInspector.create(HiveDecimal.ZERO); + HiveDecimalWritable result = (HiveDecimalWritable) writableDecimalObjectInspector.create((HiveDecimal) object); + return result; } default: throw new Error("Unknown primitive category " + primitiveCategory); @@ -420,7 +422,7 @@ public static Object randomObject(int column, Random r, PrimitiveCategory[] prim case CHAR: return new HiveChar(result, ((CharTypeInfo) primitiveTypeInfo).getLength()); case VARCHAR: - return new HiveChar(result, ((VarcharTypeInfo) primitiveTypeInfo).getLength()); + return new HiveVarchar(result, ((VarcharTypeInfo) primitiveTypeInfo).getLength()); default: throw new Error("Unknown primitive category " + primitiveCategory); } @@ -497,13 +499,9 @@ public static HiveDecimal getRandHiveDecimal(Random r, DecimalTypeInfo decimalTy sb.append(RandomTypeUtil.getRandString(r, DECIMAL_CHARS, scale)); } - HiveDecimal bd = HiveDecimal.create(sb.toString()); - if (bd.scale() > bd.precision()) { - // Sometimes weird decimals are produced? - continue; - } + HiveDecimal dec = HiveDecimal.create(sb.toString()); - return bd; + return dec; } } diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java index da9ebca..9f4bd9d 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestDecimalUtil.java @@ -229,7 +229,6 @@ public void testSign() { HiveDecimal d3 = HiveDecimal.create("0.00000"); Assert.assertEquals(0, d3.scale()); - d3.setScale(5); DecimalUtil.sign(0, d3, lcv); Assert.assertEquals(0, lcv.vector[0]); } diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java index e7a044e..c7d9fae 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java @@ -204,6 +204,10 @@ public void testCastDecimalToLong() { // test basic case VectorizedRowBatch b = getBatchDecimalLong(); VectorExpression expr = new CastDecimalToLong(0, 1); + + // With the integer type range checking, we need to know the Hive data type. + expr.setOutputType("bigint"); + expr.evaluate(b); LongColumnVector r = (LongColumnVector) b.cols[1]; assertEquals(1, r.vector[0]); @@ -265,9 +269,9 @@ private VectorizedRowBatch getBatchDecimalLong() { b.size = 3; - dv.vector[0].set(HiveDecimal.create("1.1").setScale(scale)); - dv.vector[1].set(HiveDecimal.create("-2.2").setScale(scale)); - dv.vector[2].set(HiveDecimal.create("9999999999999999.00").setScale(scale)); + dv.vector[0].set(HiveDecimal.create("1.1")); + dv.vector[1].set(HiveDecimal.create("-2.2")); + dv.vector[2].set(HiveDecimal.create("9999999999999999.00")); return b; } @@ -325,9 +329,9 @@ private VectorizedRowBatch getBatchDecimalDouble() { b.size = 3; - dv.vector[0].set(HiveDecimal.create("1.1").setScale(scale)); - dv.vector[1].set(HiveDecimal.create("-2.2").setScale(scale)); - dv.vector[2].set(HiveDecimal.create("9999999999999999.00").setScale(scale)); + dv.vector[0].set(HiveDecimal.create("1.1")); + dv.vector[1].set(HiveDecimal.create("-2.2")); + dv.vector[2].set(HiveDecimal.create("9999999999999999.00")); return b; } @@ -366,9 +370,9 @@ private VectorizedRowBatch getBatchDecimalString() { b.size = 3; - dv.vector[0].set(HiveDecimal.create("1.1").setScale(scale)); - dv.vector[1].set(HiveDecimal.create("-2.2").setScale(scale)); - dv.vector[2].set(HiveDecimal.create("9999999999999999.00").setScale(scale)); + dv.vector[0].set(HiveDecimal.create("1.1")); + dv.vector[1].set(HiveDecimal.create("-2.2")); + dv.vector[2].set(HiveDecimal.create("9999999999999999.00")); return b; } @@ -399,9 +403,9 @@ private VectorizedRowBatch getBatchDecimalLong2() { b.size = 3; - dv.vector[0].set(HiveDecimal.create("1.111111111").setScale(scale)); - dv.vector[1].set(HiveDecimal.create("-2.222222222").setScale(scale)); - dv.vector[2].set(HiveDecimal.create("31536000.999999999").setScale(scale)); + dv.vector[0].set(HiveDecimal.create("1.111111111")); + dv.vector[1].set(HiveDecimal.create("-2.222222222")); + dv.vector[2].set(HiveDecimal.create("31536000.999999999")); return b; } diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java index bc7a658..638ccc5 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/CheckFastRowHashMap.java @@ -166,7 +166,9 @@ public static void verifyHashMapRowsMore(List rows, int[] actualToValu } } else { if (thrown) { - TestCase.fail("Not expecting an exception to be thrown for the non-clipped case..."); + TestCase.fail("Not expecting an exception to be thrown for the non-clipped case... " + + " exception message " + debugExceptionMessage + + " stack trace " + getStackTraceAsSingleLine(debugStackTrace)); } TestCase.assertTrue(lazyBinaryDeserializeRead.isEndOfInputReached()); } @@ -382,4 +384,27 @@ public void verify(VectorMapJoinFastHashTable map, } } } + + static int STACK_LENGTH_LIMIT = 20; + public static String getStackTraceAsSingleLine(StackTraceElement[] stackTrace) { + StringBuilder sb = new StringBuilder(); + sb.append("Stack trace: "); + int length = stackTrace.length; + boolean isTruncated = false; + if (length > STACK_LENGTH_LIMIT) { + length = STACK_LENGTH_LIMIT; + isTruncated = true; + } + for (int i = 0; i < length; i++) { + if (i > 0) { + sb.append(", "); + } + sb.append(stackTrace[i]); + } + if (isTruncated) { + sb.append(", ..."); + } + + return sb.toString(); + } } \ No newline at end of file diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VerifyFastRow.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VerifyFastRow.java index 239db73..91b3ead 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VerifyFastRow.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VerifyFastRow.java @@ -67,7 +67,10 @@ public static void verifyDeserializeRead(DeserializeRead deserializeRead, isNull = !deserializeRead.readNextField(); if (isNull) { if (writable != null) { - TestCase.fail("Field reports null but object is not null"); + TestCase.fail( + deserializeRead.getClass().getName() + + " field reports null but object is not null " + + "(class " + writable.getClass().getName() + ", " + writable.toString() + ")"); } return; } else if (writable == null) { diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java index 84717b1..91936bc 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/VectorizedRowGroupGenUtil.java @@ -151,7 +151,7 @@ public static DecimalColumnVector generateDecimalColumnVector(DecimalTypeInfo ty HiveDecimalWritable repeatingValue = new HiveDecimalWritable(); do{ - repeatingValue.set(HiveDecimal.create(((Double) rand.nextDouble()).toString()).setScale((short)typeInfo.scale())); + repeatingValue.set(HiveDecimal.create(((Double) rand.nextDouble()).toString()).round((short)typeInfo.scale(), HiveDecimal.ROUND_HALF_UP)); }while(repeatingValue.getHiveDecimal().doubleValue() == 0); int nullFrequency = generateNullFrequency(rand); @@ -159,14 +159,14 @@ public static DecimalColumnVector generateDecimalColumnVector(DecimalTypeInfo ty for(int i = 0; i < size; i++) { if(nulls && (repeating || i % nullFrequency == 0)) { dcv.isNull[i] = true; - dcv.vector[i] = null;//Decimal128.ONE; + dcv.vector[i] = null; }else { dcv.isNull[i] = false; if (repeating) { dcv.vector[i].set(repeatingValue); } else { - dcv.vector[i].set(HiveDecimal.create(((Double) rand.nextDouble()).toString()).setScale((short) typeInfo.scale())); + dcv.vector[i].set(HiveDecimal.create(((Double) rand.nextDouble()).toString()).round((short) typeInfo.scale(), HiveDecimal.ROUND_HALF_UP)); } if(dcv.vector[i].getHiveDecimal().doubleValue() == 0) { diff --git ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReaderBase.java ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReaderBase.java index eecccce..fbb5e34 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReaderBase.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestVectorizedColumnReaderBase.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.MapWork; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; +import org.apache.hadoop.hive.serde2.io.OldHiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; @@ -234,7 +235,7 @@ protected static void writeData(ParquetWriter writer, boolean isDictionar int intVal = getIntValue(isDictionaryEncoding, i); long longVal = getLongValue(isDictionaryEncoding, i); Binary timeStamp = getTimestamp(isDictionaryEncoding, i); - HiveDecimal decimalVal = getDecimal(isDictionaryEncoding, i).setScale(2); + HiveDecimal decimalVal = getDecimal(isDictionaryEncoding, i); double doubleVal = getDoubleValue(isDictionaryEncoding, i); float floatVal = getFloatValue(isDictionaryEncoding, i); boolean booleanVal = getBooleanValue(i); @@ -258,8 +259,9 @@ protected static void writeData(ParquetWriter writer, boolean isDictionar group.append("binary_field_some_null", binary); } - HiveDecimalWritable w = new HiveDecimalWritable(decimalVal); - group.append("value", Binary.fromConstantByteArray(w.getInternalStorage())); + // NOTE: Previously, we did OldHiveDecimal.setScale(2), created an OldHiveDecimalWritable + // and passed internalStorage(). + group.append("value", Binary.fromConstantByteArray(decimalVal.bigIntegerBytesScaled(2))); group.addGroup("struct_field") .append("a", intVal) diff --git ql/src/test/results/clientpositive/avro_decimal.q.out ql/src/test/results/clientpositive/avro_decimal.q.out index 64e65ca..7f57ec7 100644 --- ql/src/test/results/clientpositive/avro_decimal.q.out +++ ql/src/test/results/clientpositive/avro_decimal.q.out @@ -34,7 +34,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@dec # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment -value decimal(8,4) -12.25 234.79 0 6 from deserializer +value decimal(8,4) -12.25 234.79 0 9 from deserializer PREHOOK: query: DROP TABLE IF EXISTS avro_dec PREHOOK: type: DROPTABLE POSTHOOK: query: DROP TABLE IF EXISTS avro_dec diff --git ql/src/test/results/clientpositive/avro_decimal_native.q.out ql/src/test/results/clientpositive/avro_decimal_native.q.out index cebc342..33afd8e 100644 --- ql/src/test/results/clientpositive/avro_decimal_native.q.out +++ ql/src/test/results/clientpositive/avro_decimal_native.q.out @@ -38,7 +38,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@dec # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment -value decimal(8,4) -12.25 234.79 0 6 from deserializer +value decimal(8,4) -12.25 234.79 0 9 from deserializer PREHOOK: query: DROP TABLE IF EXISTS avro_dec PREHOOK: type: DROPTABLE POSTHOOK: query: DROP TABLE IF EXISTS avro_dec diff --git ql/src/test/results/clientpositive/compute_stats_decimal.q.out ql/src/test/results/clientpositive/compute_stats_decimal.q.out index c1a2062..d03eef1 100644 --- ql/src/test/results/clientpositive/compute_stats_decimal.q.out +++ ql/src/test/results/clientpositive/compute_stats_decimal.q.out @@ -35,4 +35,4 @@ select compute_stats(a, 18) from tab_decimal POSTHOOK: type: QUERY POSTHOOK: Input: default@tab_decimal #### A masked pattern was here #### -{"columntype":"Decimal","min":-87.2,"max":123456789012345678901234567890.123,"countnulls":2,"numdistinctvalues":13,"ndvbitvector":"{0, 1, 2, 3, 4}{0, 1, 2, 3, 5}{0, 1, 2, 3}{0, 1, 2}{0, 1, 2}{0, 1, 2, 3, 5}{0, 1, 3}{0, 1, 2, 4}{0, 1, 2, 3, 5}{0, 1, 2, 3}{0, 1, 2}{0, 1}{0, 1, 2}{0, 1, 2, 3}{0, 1, 2, 6, 8}{0, 1, 2, 3}{0, 1, 2}{0, 1, 4, 5}"} +{"columntype":"Decimal","min":-87.2,"max":123456789012345678901234567890.123,"countnulls":2,"numdistinctvalues":12,"ndvbitvector":"{0, 1, 2}{0, 1, 4}{0, 1, 2, 3}{0, 1, 2, 3, 4}{0, 1, 3}{0, 1, 2, 3}{0, 1, 3}{0, 1, 2, 4}{0, 1, 2}{0, 1, 2, 4}{0, 1, 2, 3}{0, 1, 2, 3}{0, 1, 2, 4}{0, 1, 2}{0, 1}{0, 1, 2, 3, 4}{0, 1, 2}{0, 1, 2, 3, 5}"} diff --git ql/src/test/results/clientpositive/decimal_2.q.out ql/src/test/results/clientpositive/decimal_2.q.out index 934590c..f3168f6 100644 --- ql/src/test/results/clientpositive/decimal_2.q.out +++ ql/src/test/results/clientpositive/decimal_2.q.out @@ -129,7 +129,7 @@ POSTHOOK: query: select cast(t as tinyint) from decimal_2 POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_2 #### A masked pattern was here #### -13 +NULL PREHOOK: query: select cast(t as smallint) from decimal_2 PREHOOK: type: QUERY PREHOOK: Input: default@decimal_2 @@ -138,7 +138,7 @@ POSTHOOK: query: select cast(t as smallint) from decimal_2 POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_2 #### A masked pattern was here #### --3827 +NULL PREHOOK: query: select cast(t as int) from decimal_2 PREHOOK: type: QUERY PREHOOK: Input: default@decimal_2 diff --git ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out index 839c30e..a94871c 100644 --- ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out +++ ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out @@ -143,7 +143,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@loc_orc_1d # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment -cnt decimal(10,0) 10 2000 0 5 from deserializer +cnt decimal(10,0) 10 2000 0 4 from deserializer PREHOOK: query: describe formatted loc_orc_1d PARTITION(year='2002') cnt PREHOOK: type: DESCTABLE PREHOOK: Input: default@loc_orc_1d @@ -152,7 +152,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@loc_orc_1d # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment -cnt decimal(10,0) 10 910 0 4 from deserializer +cnt decimal(10,0) 10 910 0 5 from deserializer PREHOOK: query: describe formatted loc_orc_1d PARTITION(year='2001') zip PREHOOK: type: DESCTABLE PREHOOK: Input: default@loc_orc_1d @@ -428,7 +428,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@loc_orc_1d # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment -cnt decimal(10,0) 1000 1010 0 3 from deserializer +cnt decimal(10,0) 1000 1010 0 2 from deserializer PREHOOK: query: describe formatted loc_orc_1d PARTITION(year='2003') cnt PREHOOK: type: DESCTABLE PREHOOK: Input: default@loc_orc_1d @@ -437,7 +437,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@loc_orc_1d # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment -cnt decimal(10,0) 1000 2000 0 3 from deserializer +cnt decimal(10,0) 1000 2000 0 2 from deserializer PREHOOK: query: describe formatted loc_orc_1d PARTITION(year='2000') zip PREHOOK: type: DESCTABLE PREHOOK: Input: default@loc_orc_1d @@ -793,7 +793,7 @@ POSTHOOK: type: DESCTABLE POSTHOOK: Input: default@loc_orc_2d # col_name data_type min max num_nulls distinct_count avg_col_len max_col_len num_trues num_falses comment -cnt decimal(10,0) 10 100 0 2 from deserializer +cnt decimal(10,0) 10 100 0 3 from deserializer PREHOOK: query: explain extended select state,locid,cnt,zip from loc_orc_2d PREHOOK: type: QUERY POSTHOOK: query: explain extended select state,locid,cnt,zip from loc_orc_2d diff --git ql/src/test/results/clientpositive/llap/orc_ppd_date.q.out ql/src/test/results/clientpositive/llap/orc_ppd_date.q.out index e07739d..cb155ee 100644 --- ql/src/test/results/clientpositive/llap/orc_ppd_date.q.out +++ ql/src/test/results/clientpositive/llap/orc_ppd_date.q.out @@ -28,7 +28,7 @@ select sum(hash(*)) from newtypesorc where da='1970-02-20' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where da='1970-02-20' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -37,7 +37,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da='1970-02-20' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where da= date '1970-02-20' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -46,7 +46,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da= date '1970-02-20 POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where da=cast('1970-02-20' as date) PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -55,7 +55,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da=cast('1970-02-20' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where da=cast('1970-02-20' as date) PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -64,7 +64,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da=cast('1970-02-20' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where da=cast('1970-02-20' as varchar(20)) PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -73,7 +73,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da=cast('1970-02-20' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where da=cast('1970-02-20' as varchar(20)) PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -82,7 +82,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da=cast('1970-02-20' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where da!='1970-02-20' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -91,7 +91,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da!='1970-02-20' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -334427804500 +664732893500 PREHOOK: query: select sum(hash(*)) from newtypesorc where da!='1970-02-20' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -100,7 +100,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da!='1970-02-20' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -334427804500 +664732893500 PREHOOK: query: select sum(hash(*)) from newtypesorc where da<'1970-02-27' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -109,7 +109,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da<'1970-02-27' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where da<'1970-02-27' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -118,7 +118,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da<'1970-02-27' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where da<'1970-02-29' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -127,7 +127,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da<'1970-02-29' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -81475875500 +133573917500 PREHOOK: query: select sum(hash(*)) from newtypesorc where da<'1970-02-29' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -136,7 +136,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da<'1970-02-29' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -81475875500 +133573917500 PREHOOK: query: select sum(hash(*)) from newtypesorc where da<'1970-02-15' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -163,7 +163,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da<='1970-02-20' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where da<='1970-02-20' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -172,7 +172,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da<='1970-02-20' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where da<='1970-02-27' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -181,7 +181,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da<='1970-02-27' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -81475875500 +133573917500 PREHOOK: query: select sum(hash(*)) from newtypesorc where da<='1970-02-27' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -190,7 +190,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da<='1970-02-27' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -81475875500 +133573917500 PREHOOK: query: select sum(hash(*)) from newtypesorc where da in (cast('1970-02-21' as date), cast('1970-02-27' as date)) PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -199,7 +199,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da in (cast('1970-02 POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -334427804500 +664732893500 PREHOOK: query: select sum(hash(*)) from newtypesorc where da in (cast('1970-02-21' as date), cast('1970-02-27' as date)) PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -208,7 +208,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da in (cast('1970-02 POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -334427804500 +664732893500 PREHOOK: query: select sum(hash(*)) from newtypesorc where da in (cast('1970-02-20' as date), cast('1970-02-27' as date)) PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -217,7 +217,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da in (cast('1970-02 POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -81475875500 +133573917500 PREHOOK: query: select sum(hash(*)) from newtypesorc where da in (cast('1970-02-20' as date), cast('1970-02-27' as date)) PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -226,7 +226,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da in (cast('1970-02 POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -81475875500 +133573917500 PREHOOK: query: select sum(hash(*)) from newtypesorc where da in (cast('1970-02-21' as date), cast('1970-02-22' as date)) PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -253,7 +253,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da between '1970-02- POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where da between '1970-02-19' and '1970-02-22' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -262,7 +262,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da between '1970-02- POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where da between '1970-02-19' and '1970-02-28' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -271,7 +271,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da between '1970-02- POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -81475875500 +133573917500 PREHOOK: query: select sum(hash(*)) from newtypesorc where da between '1970-02-19' and '1970-02-28' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -280,7 +280,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where da between '1970-02- POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -81475875500 +133573917500 PREHOOK: query: select sum(hash(*)) from newtypesorc where da between '1970-02-18' and '1970-02-19' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc diff --git ql/src/test/results/clientpositive/llap/orc_ppd_decimal.q.out ql/src/test/results/clientpositive/llap/orc_ppd_decimal.q.out index 6ddff10..e0b2a0b 100644 --- ql/src/test/results/clientpositive/llap/orc_ppd_decimal.q.out +++ ql/src/test/results/clientpositive/llap/orc_ppd_decimal.q.out @@ -28,7 +28,7 @@ select sum(hash(*)) from newtypesorc where d=0.22 POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where d=0.22 PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -37,7 +37,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d=0.22 POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where d='0.22' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -46,7 +46,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d='0.22' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where d='0.22' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -55,7 +55,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d='0.22' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where d=cast('0.22' as float) PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -64,7 +64,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d=cast('0.22' as flo POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where d=cast('0.22' as float) PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -73,7 +73,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d=cast('0.22' as flo POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where d!=0.22 PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -82,7 +82,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d!=0.22 POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -334427804500 +664732893500 PREHOOK: query: select sum(hash(*)) from newtypesorc where d!=0.22 PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -91,7 +91,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d!=0.22 POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -334427804500 +664732893500 PREHOOK: query: select sum(hash(*)) from newtypesorc where d!='0.22' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -100,7 +100,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d!='0.22' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -334427804500 +664732893500 PREHOOK: query: select sum(hash(*)) from newtypesorc where d!='0.22' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -109,7 +109,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d!='0.22' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -334427804500 +664732893500 PREHOOK: query: select sum(hash(*)) from newtypesorc where d!=cast('0.22' as float) PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -118,7 +118,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d!=cast('0.22' as fl POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -334427804500 +664732893500 PREHOOK: query: select sum(hash(*)) from newtypesorc where d!=cast('0.22' as float) PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -127,7 +127,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d!=cast('0.22' as fl POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -334427804500 +664732893500 PREHOOK: query: select sum(hash(*)) from newtypesorc where d<11.22 PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -136,7 +136,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<11.22 POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where d<11.22 PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -145,7 +145,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<11.22 POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where d<'11.22' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -154,7 +154,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<'11.22' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where d<'11.22' PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -163,7 +163,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where d<'11.22' POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### --252951929000 +-531158976000 PREHOOK: query: select sum(hash(*)) from newtypesorc where d> 24) ^ 0x80), invert); + writeByte(buffer, (byte) ( factor >> 16), invert); + writeByte(buffer, (byte) ( factor >> 8), invert); + writeByte(buffer, (byte) factor, invert); + + // The toDigitsOnlyBytes stores digits at the end of the scratch buffer. + serializeBytes( + buffer, + scratchBuffer, index, scratchBuffer.length - index, + signum == -1 ? !invert : invert); + } + + // A HiveDecimalWritable version. + public static void serializeHiveDecimal( + ByteStream.Output buffer, HiveDecimalWritable decWritable, boolean invert, + byte[] scratchBuffer) { + + // Get the sign of the decimal. + int signum = decWritable.signum(); + + // Get the 10^N power to turn digits into the desired decimal with a possible + // fractional part. + // To be compatible with the OldHiveDecimal version, zero has factor 1. + int factor; + if (signum == 0) { + factor = 1; + } else { + factor = decWritable.rawPrecision() - decWritable.scale(); + } + + // To make comparisons work properly, the "factor" gets the decimal's sign, too. + factor = signum == 1 ? factor : -factor; + + // Convert just the decimal digits (no dot, sign, etc) into bytes. + // + // This is much faster than converting the BigInteger value from unscaledValue() which is no + // longer part of the HiveDecimal representation anymore to string, then bytes. + int index = decWritable.toDigitsOnlyBytes(scratchBuffer); + + /* + * Finally write out the pieces (sign, power, digits) + */ + writeByte(buffer, (byte) ( signum + 1), invert); + writeByte(buffer, (byte) ((factor >> 24) ^ 0x80), invert); + writeByte(buffer, (byte) ( factor >> 16), invert); + writeByte(buffer, (byte) ( factor >> 8), invert); + writeByte(buffer, (byte) factor, invert); + + // The toDigitsOnlyBytes stores digits at the end of the scratch buffer. + serializeBytes( + buffer, + scratchBuffer, index, scratchBuffer.length - index, + signum == -1 ? !invert : invert); + } + @Override public SerDeStats getSerDeStats() { // no support for statistics diff --git serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableDeserializeRead.java serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableDeserializeRead.java index a7785b2..41087dc 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableDeserializeRead.java +++ serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableDeserializeRead.java @@ -21,10 +21,11 @@ import java.io.IOException; import java.math.BigInteger; import java.util.Arrays; +import java.nio.charset.StandardCharsets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.FastHiveDecimal; import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe; import org.apache.hadoop.hive.serde2.binarysortable.InputByteBuffer; import org.apache.hadoop.hive.serde2.fast.DeserializeRead; @@ -391,6 +392,7 @@ public boolean readNextField() throws IOException { length++; } while (true); + // CONSIDER: Allocate a larger initial size. if(tempDecimalBuffer == null || tempDecimalBuffer.length < length) { tempDecimalBuffer = new byte[length]; } @@ -403,29 +405,30 @@ public boolean readNextField() throws IOException { // read the null byte again inputByteBuffer.read(positive ? invert : !invert); - String digits = new String(tempDecimalBuffer, 0, length, BinarySortableSerDe.decimalCharSet); - BigInteger bi = new BigInteger(digits); - HiveDecimal bd = HiveDecimal.create(bi).scaleByPowerOfTen(factor-length); + String digits = new String(tempDecimalBuffer, 0, length, StandardCharsets.UTF_8); - if (!positive) { - bd = bd.negate(); - } + // Set the value of the writable from the decimal digits that were written with no dot. + int scale = length - factor; + currentHiveDecimalWritable.setFromDigitsOnlyBytesWithScale( + !positive, tempDecimalBuffer, 0, length, scale); + boolean decimalIsNull = !currentHiveDecimalWritable.isSet(); + if (!decimalIsNull) { - // We have a decimal. After we enforce precision and scale, will it become a NULL? + // We have a decimal. After we enforce precision and scale, will it become a NULL? - currentHiveDecimalWritable.set(bd); + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfos[fieldIndex]; - DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfos[fieldIndex]; + int enforcePrecision = decimalTypeInfo.getPrecision(); + int enforceScale = decimalTypeInfo.getScale(); - int precision = decimalTypeInfo.getPrecision(); - int scale = decimalTypeInfo.getScale(); + decimalIsNull = + !currentHiveDecimalWritable.mutateEnforcePrecisionScale( + enforcePrecision, enforceScale); - HiveDecimal decimal = currentHiveDecimalWritable.getHiveDecimal(precision, scale); - if (decimal == null) { + } + if (decimalIsNull) { return false; } - // Put value back into writable. - currentHiveDecimalWritable.set(decimal); } return true; default: diff --git serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableSerializeWrite.java serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableSerializeWrite.java index 62bcaa5..a9ea7c0 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableSerializeWrite.java +++ serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableSerializeWrite.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe; import org.apache.hadoop.hive.serde2.fast.SerializeWrite; import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hive.common.util.DateUtils; import org.slf4j.Logger; @@ -61,6 +62,8 @@ private TimestampWritable tempTimestampWritable; + private byte[] decimalBytesScratch; + public BinarySortableSerializeWrite(boolean[] columnSortOrderIsDesc, byte[] columnNullMarker, byte[] columnNotNullMarker) { this(); @@ -397,6 +400,9 @@ public void writeHiveIntervalDayTime(HiveIntervalDayTime vidt) throws IOExceptio /* * DECIMAL. + * + * NOTE: The scale parameter is for text serialization (e.g. HiveDecimal.toFormatString) that + * creates trailing zeroes output decimals. */ @Override public void writeHiveDecimal(HiveDecimal dec, int scale) throws IOException { @@ -407,6 +413,24 @@ public void writeHiveDecimal(HiveDecimal dec, int scale) throws IOException { // This field is not a null. BinarySortableSerDe.writeByte(output, columnNotNullMarker[index], invert); - BinarySortableSerDe.serializeHiveDecimal(output, dec, invert); + if (decimalBytesScratch == null) { + decimalBytesScratch = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES]; + } + BinarySortableSerDe.serializeHiveDecimal(output, dec, invert, decimalBytesScratch); + } + + @Override + public void writeHiveDecimal(HiveDecimalWritable decWritable, int scale) throws IOException { + ++index; + + final boolean invert = columnSortOrderIsDesc[index]; + + // This field is not a null. + BinarySortableSerDe.writeByte(output, columnNotNullMarker[index], invert); + + if (decimalBytesScratch == null) { + decimalBytesScratch = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES]; + } + BinarySortableSerDe.serializeHiveDecimal(output, decWritable, invert, decimalBytesScratch); } } diff --git serde/src/java/org/apache/hadoop/hive/serde2/fast/SerializeWrite.java serde/src/java/org/apache/hadoop/hive/serde2/fast/SerializeWrite.java index fb41420..17d2385 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/fast/SerializeWrite.java +++ serde/src/java/org/apache/hadoop/hive/serde2/fast/SerializeWrite.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.common.type.HiveVarchar; @@ -147,6 +148,10 @@ /* * DECIMAL. + * + * NOTE: The scale parameter is for text serialization (e.g. HiveDecimal.toFormatString) that + * creates trailing zeroes output decimals. */ void writeHiveDecimal(HiveDecimal dec, int scale) throws IOException; + void writeHiveDecimal(HiveDecimalWritable decWritable, int scale) throws IOException; } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveDecimal.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveDecimal.java index 4e82e9b..4d2ff22 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveDecimal.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveDecimal.java @@ -21,6 +21,7 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; +import java.nio.charset.StandardCharsets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,31 +67,21 @@ public LazyHiveDecimal(LazyHiveDecimal copy) { */ @Override public void init(ByteArrayRef bytes, int start, int length) { - String byteData = null; - try { - byteData = Text.decode(bytes.getData(), start, length); - } catch (CharacterCodingException e) { - isNull = true; - LOG.debug("Data not in the HiveDecimal data type range so converted to null.", e); - return; - } - HiveDecimal dec = HiveDecimal.create(byteData); - dec = enforcePrecisionScale(dec); - if (dec != null) { - data.set(dec); - isNull = false; + // Set the HiveDecimalWritable from bytes without converting to String first for + // better performance. + data.setFromBytes(bytes.getData(), start, length); + if (!data.isSet()) { + isNull = true; } else { + isNull = !data.mutateEnforcePrecisionScale(precision, scale); + } + if (isNull) { LOG.debug("Data not in the HiveDecimal data type range so converted to null. Given data is :" - + byteData); - isNull = true; + + new String(bytes.getData(), start, length, StandardCharsets.UTF_8)); } } - private HiveDecimal enforcePrecisionScale(HiveDecimal dec) { - return HiveDecimal.enforcePrecisionScale(dec, precision, scale); - } - @Override public HiveDecimalWritable getWritableObject() { return data; @@ -107,8 +98,47 @@ public static void writeUTF8(OutputStream outputStream, HiveDecimal hiveDecimal, if (hiveDecimal == null) { outputStream.write(nullBytes); } else { - ByteBuffer b = Text.encode(hiveDecimal.toFormatString(scale)); - outputStream.write(b.array(), 0, b.limit()); + byte[] scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES]; + int index = hiveDecimal.toFormatBytes(scale, scratchBuffer); + outputStream.write(scratchBuffer, index, scratchBuffer.length - index); + } + } + + /** + * Writes HiveDecimal object to output stream as string + * @param outputStream + * @param hiveDecimal + * @throws IOException + */ + public static void writeUTF8( + OutputStream outputStream, + HiveDecimal hiveDecimal, int scale, + byte[] scratchBuffer) + throws IOException { + if (hiveDecimal == null) { + outputStream.write(nullBytes); + } else { + int index = hiveDecimal.toFormatBytes(scale, scratchBuffer); + outputStream.write(scratchBuffer, index, scratchBuffer.length - index); + } + } + + /** + * Writes HiveDecimalWritable object to output stream as string + * @param outputStream + * @param hiveDecimal + * @throws IOException + */ + public static void writeUTF8( + OutputStream outputStream, + HiveDecimalWritable hiveDecimalWritable, int scale, + byte[] scratchBuffer) + throws IOException { + if (hiveDecimalWritable == null || !hiveDecimalWritable.isSet()) { + outputStream.write(nullBytes); + } else { + int index = hiveDecimalWritable.toFormatBytes(scale, scratchBuffer); + outputStream.write(scratchBuffer, index, scratchBuffer.length - index); } } } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java index daf2cfb..a597fd7 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java @@ -27,7 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; import org.apache.hadoop.hive.serde2.fast.DeserializeRead; @@ -556,20 +555,24 @@ public boolean readField(int fieldIndex) throws IOException { if (!LazyUtils.isNumberMaybe(bytes, fieldStart, fieldLength)) { return false; } - String byteData = new String(bytes, fieldStart, fieldLength, StandardCharsets.UTF_8); - HiveDecimal decimal = HiveDecimal.create(byteData); - DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfos[fieldIndex]; - int precision = decimalTypeInfo.getPrecision(); - int scale = decimalTypeInfo.getScale(); - decimal = HiveDecimal.enforcePrecisionScale(decimal, precision, scale); - if (decimal == null) { + // Trim blanks because OldHiveDecimal did... + currentHiveDecimalWritable.setFromBytes(bytes, fieldStart, fieldLength, /* trimBlanks */ true); + boolean decimalIsNull = !currentHiveDecimalWritable.isSet(); + if (!decimalIsNull) { + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfos[fieldIndex]; + + int precision = decimalTypeInfo.getPrecision(); + int scale = decimalTypeInfo.getScale(); + + decimalIsNull = !currentHiveDecimalWritable.mutateEnforcePrecisionScale(precision, scale); + } + if (decimalIsNull) { if (LOG.isDebugEnabled()) { LOG.debug("Data not in the HiveDecimal data type range so converted to null. Given data is :" - + byteData); + + new String(bytes, fieldStart, fieldLength, StandardCharsets.UTF_8)); } return false; } - currentHiveDecimalWritable.set(decimal); } return true; diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleSerializeWrite.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleSerializeWrite.java index 280c2b0..1401ac3 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleSerializeWrite.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleSerializeWrite.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.common.type.HiveVarchar; import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable; import org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; @@ -76,6 +77,7 @@ private HiveIntervalYearMonthWritable hiveIntervalYearMonthWritable; private HiveIntervalDayTimeWritable hiveIntervalDayTimeWritable; private HiveIntervalDayTime hiveIntervalDayTime; + private byte[] decimalScratchBuffer; public LazySimpleSerializeWrite(int fieldCount, byte separator, LazySerDeParameters lazyParams) { @@ -475,14 +477,34 @@ public void writeHiveIntervalDayTime(HiveIntervalDayTime vidt) throws IOExceptio /* * DECIMAL. + * + * NOTE: The scale parameter is for text serialization (e.g. HiveDecimal.toFormatString) that + * creates trailing zeroes output decimals. */ @Override - public void writeHiveDecimal(HiveDecimal v, int scale) throws IOException { + public void writeHiveDecimal(HiveDecimal dec, int scale) throws IOException { + if (index > 0) { + output.write(separator); + } + + if (decimalScratchBuffer == null) { + decimalScratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES]; + } + LazyHiveDecimal.writeUTF8(output, dec, scale, decimalScratchBuffer); + + index++; + } + + @Override + public void writeHiveDecimal(HiveDecimalWritable decWritable, int scale) throws IOException { if (index > 0) { output.write(separator); } - LazyHiveDecimal.writeUTF8(output, v, scale); + if (decimalScratchBuffer == null) { + decimalScratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_TO_BYTES]; + } + LazyHiveDecimal.writeUTF8(output, decWritable, scale, decimalScratchBuffer); index++; } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveDecimalObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveDecimalObjectInspector.java index 55ab3e6..225c32e 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveDecimalObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveDecimalObjectInspector.java @@ -43,8 +43,13 @@ public HiveDecimal getPrimitiveJavaObject(Object o) { return null; } - HiveDecimal dec = ((LazyHiveDecimal)o).getWritableObject().getHiveDecimal(); - return HiveDecimalUtils.enforcePrecisionScale(dec, (DecimalTypeInfo) typeInfo); + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo; + // We do not want to modify the object o. + HiveDecimalWritable decWritable = ((LazyHiveDecimal)o).getWritableObject(); + HiveDecimalWritable result = HiveDecimalWritable.enforcePrecisionScale( + decWritable, decimalTypeInfo.getPrecision(), decimalTypeInfo.getScale()); + + return (result != null && result.isSet() ? result.getHiveDecimal() : null); } } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryHiveDecimal.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryHiveDecimal.java index f8469a7..b7bb67e 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryHiveDecimal.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryHiveDecimal.java @@ -44,9 +44,7 @@ @Override public void init(ByteArrayRef bytes, int start, int length) { - LazyBinarySerDe.setFromBytes(bytes.getData(), start, length, data); - HiveDecimal dec = data.getHiveDecimal(precision, scale); - data = dec == null ? null : new HiveDecimalWritable(dec); + LazyBinarySerDe.setFromBigIntegerBytesAndScale(bytes.getData(), start, length, data); + data.mutateEnforcePrecisionScale(precision, scale); } - } diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java index 54bfd2d..8be269f 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.io.DateWritable; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; +import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable; import org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable; import org.apache.hadoop.hive.serde2.io.TimestampWritable; @@ -316,7 +317,7 @@ private static void writeDateToByteStream(RandomAccessOutput byteStream, LazyBinaryUtils.writeVInt(byteStream, date.getDays()); } - public static void setFromBytes(byte[] bytes, int offset, int length, + public static void setFromBigIntegerBytesAndScale(byte[] bytes, int offset, int length, HiveDecimalWritable dec) { LazyBinaryUtils.VInt vInt = new LazyBinaryUtils.VInt(); LazyBinaryUtils.readVInt(bytes, offset, vInt); @@ -324,20 +325,66 @@ public static void setFromBytes(byte[] bytes, int offset, int length, offset += vInt.length; LazyBinaryUtils.readVInt(bytes, offset, vInt); offset += vInt.length; - byte[] internalStorage = dec.getInternalStorage(); - if (internalStorage.length != vInt.value) { - internalStorage = new byte[vInt.value]; - } - System.arraycopy(bytes, offset, internalStorage, 0, vInt.value); - dec.set(internalStorage, scale); + dec.setFromBigIntegerBytesAndScale(bytes, offset, vInt.value, scale); } public static void writeToByteStream(RandomAccessOutput byteStream, - HiveDecimalWritable dec) { - LazyBinaryUtils.writeVInt(byteStream, dec.getScale()); - byte[] internalStorage = dec.getInternalStorage(); - LazyBinaryUtils.writeVInt(byteStream, internalStorage.length); - byteStream.write(internalStorage, 0, internalStorage.length); + HiveDecimalWritable decWritable) { + LazyBinaryUtils.writeVInt(byteStream, decWritable.scale()); + + // NOTE: This writes into a scratch buffer within HiveDecimalWritable. + // + int byteLength = decWritable.bigIntegerBytesInternalScratch(); + + LazyBinaryUtils.writeVInt(byteStream, byteLength); + byteStream.write(decWritable.bigIntegerBytesInternalScratchBuffer(), 0, byteLength); + } + + /** + * + * Allocate scratchLongs with HiveDecimal.SCRATCH_LONGS_LEN longs. + * And, allocate scratch buffer with HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes. + * + * @param byteStream + * @param dec + * @param scratchLongs + * @param buffer + */ + public static void writeToByteStream( + RandomAccessOutput byteStream, + HiveDecimal dec, + long[] scratchLongs, byte[] scratchBytes) { + LazyBinaryUtils.writeVInt(byteStream, dec.scale()); + + // Convert decimal into the scratch buffer without allocating a byte[] each time + // for better performance. + int byteLength = + dec.bigIntegerBytes( + scratchLongs, scratchBytes); + LazyBinaryUtils.writeVInt(byteStream, byteLength); + byteStream.write(scratchBytes, 0, byteLength); + } + + /** + * + * Allocate scratchLongs with HiveDecimal.SCRATCH_LONGS_LEN longs. + * And, allocate scratch buffer with HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes. + * + * @param byteStream + * @param dec + * @param scratchLongs + * @param buffer + */ + public static void writeToByteStream( + RandomAccessOutput byteStream, + HiveDecimalWritable decWritable, + long[] scratchLongs, byte[] scratchBytes) { + LazyBinaryUtils.writeVInt(byteStream, decWritable.scale()); + int byteLength = + decWritable.bigIntegerBytes( + scratchLongs, scratchBytes); + LazyBinaryUtils.writeVInt(byteStream, byteLength); + byteStream.write(scratchBytes, 0, byteLength); } /** diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinaryDeserializeRead.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinaryDeserializeRead.java index ee945d4..e94ae99 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinaryDeserializeRead.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinaryDeserializeRead.java @@ -24,7 +24,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.serde2.fast.DeserializeRead; import org.apache.hadoop.hive.serde2.io.TimestampWritable; import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe; @@ -313,8 +312,8 @@ public boolean readNextField() throws IOException { throw new EOFException(); } LazyBinaryUtils.readVInt(bytes, offset, tempVInt); - int saveStart = offset; offset += tempVInt.length; + int readScale = tempVInt.value; // Parse the first byte of a vint/vlong to determine the number of bytes. if (offset + WritableUtils.decodeVIntSize(bytes[offset]) > end) { @@ -322,7 +321,7 @@ public boolean readNextField() throws IOException { } LazyBinaryUtils.readVInt(bytes, offset, tempVInt); offset += tempVInt.length; - + int saveStart = offset; offset += tempVInt.value; // Last item -- ok to be at end. if (offset > end) { @@ -330,16 +329,23 @@ public boolean readNextField() throws IOException { } int length = offset - saveStart; - LazyBinarySerDe.setFromBytes(bytes, saveStart, length, - currentHiveDecimalWritable); + // scale = 2, length = 6, value = -6065716379.11 + // \002\006\255\114\197\131\083\105 + // \255\114\197\131\083\105 - DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfos[fieldIndex]; + currentHiveDecimalWritable.setFromBigIntegerBytesAndScale( + bytes, saveStart, length, readScale); + boolean decimalIsNull = !currentHiveDecimalWritable.isSet(); + if (!decimalIsNull) { - int precision = decimalTypeInfo.getPrecision(); - int scale = decimalTypeInfo.getScale(); + DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfos[fieldIndex]; - HiveDecimal decimal = currentHiveDecimalWritable.getHiveDecimal(precision, scale); - if (decimal == null) { + int precision = decimalTypeInfo.getPrecision(); + int scale = decimalTypeInfo.getScale(); + + decimalIsNull = !currentHiveDecimalWritable.mutateEnforcePrecisionScale(precision, scale); + } + if (decimalIsNull) { // Logically move past this field. fieldIndex++; @@ -356,8 +362,6 @@ public boolean readNextField() throws IOException { } return false; } - // Put value back into writable. - currentHiveDecimalWritable.set(decimal); } break; diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinarySerializeWrite.java serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinarySerializeWrite.java index 91ef12d..6bc4622 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinarySerializeWrite.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/fast/LazyBinarySerializeWrite.java @@ -56,11 +56,12 @@ private long nullOffset; // For thread safety, we allocate private writable objects for our use only. - private HiveDecimalWritable hiveDecimalWritable; private TimestampWritable timestampWritable; private HiveIntervalYearMonthWritable hiveIntervalYearMonthWritable; private HiveIntervalDayTimeWritable hiveIntervalDayTimeWritable; private HiveIntervalDayTime hiveIntervalDayTime; + private long[] scratchLongs; + private byte[] scratchBuffer; public LazyBinarySerializeWrite(int fieldCount) { this(); @@ -675,9 +676,48 @@ public void writeHiveIntervalDayTime(HiveIntervalDayTime vidt) throws IOExceptio /* * DECIMAL. + * + * NOTE: The scale parameter is for text serialization (e.g. HiveDecimal.toFormatString) that + * creates trailing zeroes output decimals. */ @Override - public void writeHiveDecimal(HiveDecimal v, int scale) throws IOException { + public void writeHiveDecimal(HiveDecimal dec, int scale) throws IOException { + + // Every 8 fields we write a NULL byte. + if ((fieldIndex % 8) == 0) { + if (fieldIndex > 0) { + // Write back previous 8 field's NULL byte. + output.writeByte(nullOffset, nullByte); + nullByte = 0; + nullOffset = output.getLength(); + } + // Allocate next NULL byte. + output.reserve(1); + } + + // Set bit in NULL byte when a field is NOT NULL. + nullByte |= 1 << (fieldIndex % 8); + + if (scratchLongs == null) { + scratchLongs = new long[HiveDecimal.SCRATCH_LONGS_LEN]; + scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; + } + LazyBinarySerDe.writeToByteStream( + output, + dec, + scratchLongs, + scratchBuffer); + + fieldIndex++; + + if (fieldIndex == fieldCount) { + // Write back the final NULL byte before the last fields. + output.writeByte(nullOffset, nullByte); + } + } + + @Override + public void writeHiveDecimal(HiveDecimalWritable decWritable, int scale) throws IOException { // Every 8 fields we write a NULL byte. if ((fieldIndex % 8) == 0) { @@ -694,11 +734,15 @@ public void writeHiveDecimal(HiveDecimal v, int scale) throws IOException { // Set bit in NULL byte when a field is NOT NULL. nullByte |= 1 << (fieldIndex % 8); - if (hiveDecimalWritable == null) { - hiveDecimalWritable = new HiveDecimalWritable(); + if (scratchLongs == null) { + scratchLongs = new long[HiveDecimal.SCRATCH_LONGS_LEN]; + scratchBuffer = new byte[HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; } - hiveDecimalWritable.set(v); - LazyBinarySerDe.writeToByteStream(output, hiveDecimalWritable); + LazyBinarySerDe.writeToByteStream( + output, + decWritable, + scratchLongs, + scratchBuffer); fieldIndex++; diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java index 1ac72c6..7613247 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java @@ -698,7 +698,9 @@ public static int hashCode(Object o, ObjectInspector objIns) { .getPrimitiveWritableObject(o); return intervalDayTime.hashCode(); case DECIMAL: - return ((HiveDecimalObjectInspector) poi).getPrimitiveWritableObject(o).hashCode(); + // Since getBucketHashCode uses this, so we must return the old (much slower) hash code for + // compatibility. + return ((HiveDecimalObjectInspector) poi).getPrimitiveWritableObject(o).oldHiveDecimalHashCode(); default: { throw new RuntimeException("Unknown type: " diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java index 26b19f5..9642a7e 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java @@ -580,7 +580,24 @@ public static boolean getBoolean(Object o, PrimitiveObjectInspector oi) { * NumberFormatException will be thrown if o is not a valid number. */ public static byte getByte(Object o, PrimitiveObjectInspector oi) { - return (byte) getInt(o, oi); + byte result; + switch (oi.getPrimitiveCategory()) { + case DECIMAL: + { + HiveDecimal dec = ((HiveDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o); + if (!dec.isByte()) { + throw new NumberFormatException(); + } + result = dec.byteValue(); + } + break; + default: + // For all other data types, use int conversion. At some point, we should have all + // conversions make sure the value fits. + return (byte) getInt(o, oi); + } + return result; } /** @@ -589,7 +606,24 @@ public static byte getByte(Object o, PrimitiveObjectInspector oi) { * NumberFormatException will be thrown if o is not a valid number. */ public static short getShort(Object o, PrimitiveObjectInspector oi) { - return (short) getInt(o, oi); + short result; + switch (oi.getPrimitiveCategory()) { + case DECIMAL: + { + HiveDecimal dec = ((HiveDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o); + if (!dec.isShort()) { + throw new NumberFormatException(); + } + result = dec.shortValue(); + } + break; + default: + // For all other data types, use int conversion. At some point, we should have all + // conversions make sure the value fits. + return (short) getInt(o, oi); + } + return result; } /** @@ -653,8 +687,14 @@ public static int getInt(Object o, PrimitiveObjectInspector oi) { .getPrimitiveWritableObject(o).getSeconds()); break; case DECIMAL: - result = ((HiveDecimalObjectInspector) oi) - .getPrimitiveJavaObject(o).intValue(); // TODO: lossy conversion! + { + HiveDecimal dec = ((HiveDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o); + if (!dec.isInt()) { + throw new NumberFormatException(); + } + result = dec.intValue(); + } break; case DATE: // unsupported conversion default: { @@ -717,8 +757,14 @@ public static long getLong(Object o, PrimitiveObjectInspector oi) { .getSeconds(); break; case DECIMAL: - result = ((HiveDecimalObjectInspector) oi) - .getPrimitiveJavaObject(o).longValue(); // TODO: lossy conversion! + { + HiveDecimal dec = ((HiveDecimalObjectInspector) oi) + .getPrimitiveJavaObject(o); + if (!dec.isLong()) { + throw new NumberFormatException(); + } + result = dec.longValue(); + } break; case DATE: // unsupported conversion default: diff --git serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantHiveDecimalObjectInspector.java serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantHiveDecimalObjectInspector.java index b87d1f8..a07c969 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantHiveDecimalObjectInspector.java +++ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableConstantHiveDecimalObjectInspector.java @@ -43,17 +43,16 @@ protected WritableConstantHiveDecimalObjectInspector() { @Override public HiveDecimalWritable getWritableConstantValue() { + // We need to enforce precision/scale here. - // A little inefficiency here as we need to create a HiveDecimal instance from the writable and - // recreate a HiveDecimalWritable instance on the HiveDecimal instance. However, we don't know - // the precision/scale of the original writable until we get a HiveDecimal instance from it. - DecimalTypeInfo decTypeInfo = (DecimalTypeInfo)typeInfo; - HiveDecimal dec = value == null ? null : - value.getHiveDecimal(decTypeInfo.precision(), decTypeInfo.scale()); - if (dec == null) { + + DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) typeInfo; + HiveDecimalWritable result = new HiveDecimalWritable(value); + result.mutateEnforcePrecisionScale(decTypeInfo.precision(), decTypeInfo.scale()); + if (!result.isSet()) { return null; } - return new HiveDecimalWritable(dec); + return result; } @Override @@ -61,7 +60,7 @@ public int precision() { if (value == null) { return super.precision(); } - return value.getHiveDecimal().precision(); + return value.sqlPrecision(); } @Override diff --git serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java index 5caaf6b..cee9c45 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java +++ serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/HiveDecimalUtils.java @@ -34,8 +34,9 @@ public static HiveDecimalWritable enforcePrecisionScale(HiveDecimalWritable writ return null; } - HiveDecimal dec = enforcePrecisionScale(writable.getHiveDecimal(), typeInfo); - return dec == null ? null : new HiveDecimalWritable(dec); + HiveDecimalWritable result = new HiveDecimalWritable(writable); + result.mutateEnforcePrecisionScale(typeInfo.precision(), typeInfo.scale()); + return (result.isSet() ? result : null); } public static void validateParameter(int precision, int scale) { diff --git serde/src/test/org/apache/hadoop/hive/serde2/SerdeRandomRowSource.java serde/src/test/org/apache/hadoop/hive/serde2/SerdeRandomRowSource.java index f08a075..301ee8b 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/SerdeRandomRowSource.java +++ serde/src/test/org/apache/hadoop/hive/serde2/SerdeRandomRowSource.java @@ -269,27 +269,27 @@ public Object getWritableObject(int column, Object object) { { WritableHiveCharObjectInspector writableCharObjectInspector = new WritableHiveCharObjectInspector( (CharTypeInfo) primitiveTypeInfo); - return writableCharObjectInspector.create(new HiveChar(StringUtils.EMPTY, -1)); + return writableCharObjectInspector.create((HiveChar) object); } case VARCHAR: { WritableHiveVarcharObjectInspector writableVarcharObjectInspector = new WritableHiveVarcharObjectInspector( (VarcharTypeInfo) primitiveTypeInfo); - return writableVarcharObjectInspector.create(new HiveVarchar(StringUtils.EMPTY, -1)); + return writableVarcharObjectInspector.create((HiveVarchar) object); } case BINARY: - return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector.create(ArrayUtils.EMPTY_BYTE_ARRAY); + return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector.create((byte[]) object); case TIMESTAMP: - return ((WritableTimestampObjectInspector) objectInspector).create(new Timestamp(0)); + return ((WritableTimestampObjectInspector) objectInspector).create((Timestamp) object); case INTERVAL_YEAR_MONTH: - return ((WritableHiveIntervalYearMonthObjectInspector) objectInspector).create(new HiveIntervalYearMonth(0)); + return ((WritableHiveIntervalYearMonthObjectInspector) objectInspector).create((HiveIntervalYearMonth) object); case INTERVAL_DAY_TIME: - return ((WritableHiveIntervalDayTimeObjectInspector) objectInspector).create(new HiveIntervalDayTime(0, 0)); + return ((WritableHiveIntervalDayTimeObjectInspector) objectInspector).create((HiveIntervalDayTime) object); case DECIMAL: { WritableHiveDecimalObjectInspector writableDecimalObjectInspector = new WritableHiveDecimalObjectInspector((DecimalTypeInfo) primitiveTypeInfo); - return writableDecimalObjectInspector.create(HiveDecimal.ZERO); + return writableDecimalObjectInspector.create((HiveDecimal) object); } default: throw new Error("Unknown primitive category " + primitiveCategory); @@ -331,7 +331,10 @@ public Object randomObject(int column) { case INTERVAL_DAY_TIME: return getRandIntervalDayTime(r); case DECIMAL: - return getRandHiveDecimal(r, (DecimalTypeInfo) primitiveTypeInfo); + { + HiveDecimal dec = getRandHiveDecimal(r, (DecimalTypeInfo) primitiveTypeInfo); + return dec; + } default: throw new Error("Unknown primitive category " + primitiveCategory); } @@ -382,14 +385,9 @@ public static HiveDecimal getRandHiveDecimal(Random r, DecimalTypeInfo decimalTy sb.append("."); sb.append(RandomTypeUtil.getRandString(r, DECIMAL_CHARS, scale)); } + HiveDecimal dec = HiveDecimal.create(sb.toString()); - HiveDecimal bd = HiveDecimal.create(sb.toString()); - if (bd.scale() > bd.precision()) { - // Sometimes weird decimals are produced? - continue; - } - - return bd; + return dec; } } diff --git serde/src/test/org/apache/hadoop/hive/serde2/VerifyFast.java serde/src/test/org/apache/hadoop/hive/serde2/VerifyFast.java index 3ac339d..19b04bb 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/VerifyFast.java +++ serde/src/test/org/apache/hadoop/hive/serde2/VerifyFast.java @@ -68,7 +68,7 @@ public static void verifyDeserializeRead(DeserializeRead deserializeRead, isNull = !deserializeRead.readNextField(); if (isNull) { if (writable != null) { - TestCase.fail("Field reports null but object is not null"); + TestCase.fail("Field reports null but object is not null (class " + writable.getClass().getName() + ", " + writable.toString() + ")"); } return; } else if (writable == null) { diff --git serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java index 45be2dd..5469b36 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java +++ serde/src/test/org/apache/hadoop/hive/serde2/avro/TestAvroSerializer.java @@ -24,6 +24,7 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericEnumSymbol; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.OldHiveDecimal; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; @@ -96,7 +97,11 @@ private void singleFieldTest(String fieldName, Object fieldValue, String fieldTy throws SerDeException, IOException { GenericRecord r2 = serializeAndDeserialize("{ \"name\":\"" + fieldName + "\", \"type\":" + fieldType + " }", fieldName, fieldValue); - assertEquals(fieldValue, r2.get(fieldName)); + if (!fieldValue.equals(r2.get(fieldName))) { + System.out.println("SINGLE_FIELD_TEST " + fieldValue); + System.out.println("SINGLE_FIELD_TEST " + r2.get(fieldName)); + assertTrue(false); + } } @Test @@ -121,7 +126,7 @@ public void canSerializeDoubles() throws SerDeException, IOException { @Test public void canSerializeDecimals() throws SerDeException, IOException { - ByteBuffer bb = ByteBuffer.wrap(HiveDecimal.create("3.1416").unscaledValue().toByteArray()); + ByteBuffer bb = ByteBuffer.wrap(HiveDecimal.create("3.1416").bigIntegerBytes()); singleFieldTest("dec1", bb.rewind(), "{\"type\":\"bytes\", \"logicalType\":\"decimal\", \"precision\":5, \"scale\":4}"); } @@ -229,7 +234,10 @@ public void canSerializeUnions() throws SerDeException, IOException { HiveDecimal dec = HiveDecimal.create("3.1415926"); r = serializeAndDeserialize(field, "union1", AvroSerdeUtils.getBufferFromDecimal(dec, 4)); HiveDecimal dec1 = AvroSerdeUtils.getHiveDecimalFromByteBuffer((ByteBuffer) r.get("union1"), 4); - assertEquals(dec.setScale(4), dec1); + + // For now, old class. + OldHiveDecimal oldDec = OldHiveDecimal.create(dec.bigDecimalValue()); + assertEquals(oldDec.setScale(4).toString(), dec1.toString()); } private enum enum1 {BLUE, RED , GREEN}; diff --git serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestPrimitiveClass.java serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestPrimitiveClass.java index 321b574..5e9efd1 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestPrimitiveClass.java +++ serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestPrimitiveClass.java @@ -203,18 +203,10 @@ public static HiveDecimal getRandHiveDecimal(Random r, ExtraTypeInfo extraTypeIn sb.append(getRandString(r, DECIMAL_CHARS, scale)); } - HiveDecimal bd = HiveDecimal.create(sb.toString()); - extraTypeInfo.precision = bd.precision(); - extraTypeInfo.scale = bd.scale(); - if (extraTypeInfo.scale > extraTypeInfo.precision) { - // Sometimes weird decimals are produced? - continue; - } - - // For now, punt. - extraTypeInfo.precision = HiveDecimal.SYSTEM_DEFAULT_PRECISION; - extraTypeInfo.scale = HiveDecimal.SYSTEM_DEFAULT_SCALE; - return bd; + HiveDecimal dec = HiveDecimal.create(sb.toString()); + extraTypeInfo.precision = dec.sqlPrecision(); + extraTypeInfo.scale = dec.scale(); + return dec; } } @@ -447,7 +439,7 @@ public void nonRandomFill(int idx, ExtraTypeInfo extraTypeInfo) { myHiveVarchar = new HiveVarchar(myString, myString.length()); extraTypeInfo.hiveVarcharMaxLength = myString.length(); myDecimal = (HiveDecimal) MyTestClass.getNonRandValue(MyTestClass.nrDecimal, idx); - extraTypeInfo.precision = myDecimal.precision(); + extraTypeInfo.precision = myDecimal.sqlPrecision(); extraTypeInfo.scale = myDecimal.scale(); myDate = (Date) MyTestClass.getNonRandValue(MyTestClass.nrDate, idx); myIntervalYearMonth = (HiveIntervalYearMonth) MyTestClass.getNonRandValue(MyTestClass.nrIntervalYearMonth, idx); diff --git serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java index 1c84fe6..5f5b03a 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java +++ serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.serde2.binarysortable; +import java.util.ArrayList; import java.io.EOFException; import java.util.Arrays; import java.util.List; @@ -196,7 +197,11 @@ private void testBinarySortableFast( } } else { if (!object.equals(expected)) { - fail("SerDe deserialized value does not match"); + fail("SerDe deserialized value does not match (expected " + + expected.getClass().getName() + " " + + expected.toString() + ", actual " + + object.getClass().getName() + " " + + object.toString() + ")"); } } } @@ -233,11 +238,37 @@ private void testBinarySortableFast( fail("Different byte array lengths: serDeOutput.length " + serDeOutput.length + ", serializeWriteExpected.length " + serializeWriteExpected.length + " mismatchPos " + mismatchPos + " perFieldWriteLengths " + Arrays.toString(perFieldWriteLengthsArray[i])); } + List differentPositions = new ArrayList(); for (int b = 0; b < serDeOutput.length; b++) { if (serDeOutput[b] != serializeWriteExpected[b]) { - fail("SerializeWrite and SerDe serialization does not match at position " + b); + differentPositions.add(b); } } + if (differentPositions.size() > 0) { + List serializeWriteExpectedFields = new ArrayList(); + List serDeFields = new ArrayList(); + int f = 0; + int lastBegin = 0; + for (int b = 0; b < serDeOutput.length; b++) { + int writeLength = perFieldWriteLengthsArray[i][f]; + if (b + 1 == writeLength) { + serializeWriteExpectedFields.add( + displayBytes(serializeWriteExpected, lastBegin, writeLength - lastBegin)); + serDeFields.add( + displayBytes(serDeOutput, lastBegin, writeLength - lastBegin)); + f++; + lastBegin = b + 1; + } + } + fail("SerializeWrite and SerDe serialization does not match at positions " + differentPositions.toString() + + "\n(SerializeWrite: " + + serializeWriteExpectedFields.toString() + + "\nSerDe: " + + serDeFields.toString() + + "\nperFieldWriteLengths " + Arrays.toString(perFieldWriteLengthsArray[i]) + + "\nprimitiveTypeInfos " + Arrays.toString(primitiveTypeInfos) + + "\nrow " + Arrays.toString(row)); + } } serdeBytes[i] = bytesWritable; } @@ -426,4 +457,12 @@ public void testBinarySortableFast() throws Throwable { throw e; } } + + private static String displayBytes(byte[] bytes, int start, int length) { + StringBuilder sb = new StringBuilder(); + for (int i = start; i < start + length; i++) { + sb.append(String.format("\\%03d", (int) (bytes[i] & 0xff))); + } + return sb.toString(); + } } \ No newline at end of file diff --git serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java index 3b12514..51cef45 100644 --- serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java +++ serde/src/test/org/apache/hadoop/hive/serde2/io/TestHiveDecimalWritable.java @@ -39,212 +39,24 @@ */ public class TestHiveDecimalWritable { - @Rule public ConcurrentRule concurrentRule = new ConcurrentRule(); - @Rule public RepeatingRule repeatingRule = new RepeatingRule(); + @Rule public ConcurrentRule concurrentRule = new ConcurrentRule(); + @Rule public RepeatingRule repeatingRule = new RepeatingRule(); - @Before - public void setUp() throws Exception { - } - private void doTestFastStreamForHiveDecimal(String valueString) { - Decimal128FastBuffer scratch = new Decimal128FastBuffer(); - BigDecimal value = new BigDecimal(valueString); - Decimal128 dec = new Decimal128(); - dec.update(value); + @Test + @Concurrent(count=4) + @Repeating(repetition=100) + public void testHiveDecimalWritable() { - HiveDecimalWritable witness = new HiveDecimalWritable(); - witness.set(HiveDecimal.create(value)); + HiveDecimalWritable decWritable; - int bufferUsed = dec.fastSerializeForHiveDecimal(scratch); - HiveDecimalWritable hdw = new HiveDecimalWritable(); - hdw.set(scratch.getBytes(bufferUsed), dec.getScale()); + HiveDecimal nullDec = null; + decWritable = new HiveDecimalWritable(nullDec); + assertTrue(!decWritable.isSet()); + decWritable = new HiveDecimalWritable("1"); + assertTrue(decWritable.isSet()); - HiveDecimal hd = hdw.getHiveDecimal(); - - BigDecimal readValue = hd.bigDecimalValue(); - - assertEquals(value, readValue); - - // Now test fastUpdate from the same serialized HiveDecimal - Decimal128 decRead = new Decimal128().fastUpdateFromInternalStorage( - witness.getInternalStorage(), (short) witness.getScale()); - - assertEquals(dec, decRead); - - // Test fastUpdate from it's own (not fully compacted) serialized output - Decimal128 decReadSelf = new Decimal128().fastUpdateFromInternalStorage( - hdw.getInternalStorage(), (short) hdw.getScale()); - assertEquals(dec, decReadSelf); - } - - @Test - @Concurrent(count=4) - @Repeating(repetition=100) - public void testFastStreamForHiveDecimal() { - - doTestFastStreamForHiveDecimal("0"); - doTestFastStreamForHiveDecimal("-0"); - doTestFastStreamForHiveDecimal("1"); - doTestFastStreamForHiveDecimal("-1"); - doTestFastStreamForHiveDecimal("2"); - doTestFastStreamForHiveDecimal("-2"); - doTestFastStreamForHiveDecimal("127"); - doTestFastStreamForHiveDecimal("-127"); - doTestFastStreamForHiveDecimal("128"); - doTestFastStreamForHiveDecimal("-128"); - doTestFastStreamForHiveDecimal("255"); - doTestFastStreamForHiveDecimal("-255"); - doTestFastStreamForHiveDecimal("256"); - doTestFastStreamForHiveDecimal("-256"); - doTestFastStreamForHiveDecimal("65535"); - doTestFastStreamForHiveDecimal("-65535"); - doTestFastStreamForHiveDecimal("65536"); - doTestFastStreamForHiveDecimal("-65536"); - - doTestFastStreamForHiveDecimal("10"); - doTestFastStreamForHiveDecimal("1000"); - doTestFastStreamForHiveDecimal("1000000"); - doTestFastStreamForHiveDecimal("1000000000"); - doTestFastStreamForHiveDecimal("1000000000000"); - doTestFastStreamForHiveDecimal("1000000000000000"); - doTestFastStreamForHiveDecimal("1000000000000000000"); - doTestFastStreamForHiveDecimal("1000000000000000000000"); - doTestFastStreamForHiveDecimal("1000000000000000000000000"); - doTestFastStreamForHiveDecimal("1000000000000000000000000000"); - doTestFastStreamForHiveDecimal("1000000000000000000000000000000"); - - doTestFastStreamForHiveDecimal("-10"); - doTestFastStreamForHiveDecimal("-1000"); - doTestFastStreamForHiveDecimal("-1000000"); - doTestFastStreamForHiveDecimal("-1000000000"); - doTestFastStreamForHiveDecimal("-1000000000000"); - doTestFastStreamForHiveDecimal("-1000000000000000000"); - doTestFastStreamForHiveDecimal("-1000000000000000000000"); - doTestFastStreamForHiveDecimal("-1000000000000000000000000"); - doTestFastStreamForHiveDecimal("-1000000000000000000000000000"); - doTestFastStreamForHiveDecimal("-1000000000000000000000000000000"); - - - doTestFastStreamForHiveDecimal("0.01"); - doTestFastStreamForHiveDecimal("-0.01"); - doTestFastStreamForHiveDecimal("0.02"); - doTestFastStreamForHiveDecimal("-0.02"); - doTestFastStreamForHiveDecimal("0.0127"); - doTestFastStreamForHiveDecimal("-0.0127"); - doTestFastStreamForHiveDecimal("0.0128"); - doTestFastStreamForHiveDecimal("-0.0128"); - doTestFastStreamForHiveDecimal("0.0255"); - doTestFastStreamForHiveDecimal("-0.0255"); - doTestFastStreamForHiveDecimal("0.0256"); - doTestFastStreamForHiveDecimal("-0.0256"); - doTestFastStreamForHiveDecimal("0.065535"); - doTestFastStreamForHiveDecimal("-0.065535"); - doTestFastStreamForHiveDecimal("0.065536"); - doTestFastStreamForHiveDecimal("-0.065536"); - - doTestFastStreamForHiveDecimal("0.101"); - doTestFastStreamForHiveDecimal("0.10001"); - doTestFastStreamForHiveDecimal("0.10000001"); - doTestFastStreamForHiveDecimal("0.10000000001"); - doTestFastStreamForHiveDecimal("0.10000000000001"); - doTestFastStreamForHiveDecimal("0.10000000000000001"); - doTestFastStreamForHiveDecimal("0.10000000000000000001"); - doTestFastStreamForHiveDecimal("0.10000000000000000000001"); - doTestFastStreamForHiveDecimal("0.10000000000000000000000001"); - doTestFastStreamForHiveDecimal("0.10000000000000000000000000001"); - doTestFastStreamForHiveDecimal("0.10000000000000000000000000000001"); - - doTestFastStreamForHiveDecimal("-0.101"); - doTestFastStreamForHiveDecimal("-0.10001"); - doTestFastStreamForHiveDecimal("-0.10000001"); - doTestFastStreamForHiveDecimal("-0.10000000001"); - doTestFastStreamForHiveDecimal("-0.10000000000001"); - doTestFastStreamForHiveDecimal("-0.10000000000000000001"); - doTestFastStreamForHiveDecimal("-0.10000000000000000000001"); - doTestFastStreamForHiveDecimal("-0.10000000000000000000000001"); - doTestFastStreamForHiveDecimal("-0.10000000000000000000000000001"); - doTestFastStreamForHiveDecimal("-0.10000000000000000000000000000001"); - - doTestFastStreamForHiveDecimal(Integer.toString(Integer.MAX_VALUE)); - doTestFastStreamForHiveDecimal(Integer.toString(Integer.MIN_VALUE)); - doTestFastStreamForHiveDecimal(Long.toString(Long.MAX_VALUE)); - doTestFastStreamForHiveDecimal(Long.toString(Long.MIN_VALUE)); - doTestFastStreamForHiveDecimal(Decimal128.MAX_VALUE.toFormalString()); - doTestFastStreamForHiveDecimal(Decimal128.MIN_VALUE.toFormalString()); - - // Test known serialization tricky values - int[] values = new int[] { - 0x80, - 0x8000, - 0x800000, - 0x80000000, - 0x81, - 0x8001, - 0x800001, - 0x80000001, - 0x7f, - 0x7fff, - 0x7fffff, - 0x7fffffff, - 0xff, - 0xffff, - 0xffffff, - 0xffffffff}; - - - for(int value: values) { - for (int i = 0; i < 4; ++i) { - int[] pos = new int[] {1, 0, 0, 0, 0}; - int[] neg = new int[] {0xff, 0, 0, 0, 0}; - - pos[i+1] = neg[i+1] = value; - - doTestDecimalWithBoundsCheck(new Decimal128().update32(pos, 0)); - doTestDecimalWithBoundsCheck(new Decimal128().update32(neg, 0)); - doTestDecimalWithBoundsCheck(new Decimal128().update64(pos, 0)); - doTestDecimalWithBoundsCheck(new Decimal128().update64(neg, 0)); - doTestDecimalWithBoundsCheck(new Decimal128().update96(pos, 0)); - doTestDecimalWithBoundsCheck(new Decimal128().update96(neg, 0)); - doTestDecimalWithBoundsCheck(new Decimal128().update128(pos, 0)); - doTestDecimalWithBoundsCheck(new Decimal128().update128(neg, 0)); - } - } - } - - void doTestDecimalWithBoundsCheck(Decimal128 value) { - if ((value.compareTo(Decimal128.MAX_VALUE)) > 0 || - (value.compareTo(Decimal128.MIN_VALUE)) < 0) { - // Ignore this one, out of bounds and HiveDecimal will NPE - return; - } - doTestFastStreamForHiveDecimal(value.toFormalString()); - } - - @Test - @Concurrent(count=4) - @Repeating(repetition=100) - public void testHive6594() { - Decimal128FastBuffer scratch = new Decimal128FastBuffer(); - String[] vs = new String[] { - "-4033.445769230769", - "6984454.211097692"}; - - Decimal128 d = new Decimal128(0L, (short) 14); - for (String s:vs) { - Decimal128 p = new Decimal128(s, (short) 14); - d.addDestructive(p, (short) (short) 14); - } - - int bufferUsed = d.fastSerializeForHiveDecimal(scratch); - HiveDecimalWritable hdw = new HiveDecimalWritable(); - hdw.set(scratch.getBytes(bufferUsed), d.getScale()); - - HiveDecimal hd = hdw.getHiveDecimal(); - - BigDecimal readValue = hd.bigDecimalValue(); - - assertEquals(d.toBigDecimal().stripTrailingZeros(), - readValue.stripTrailingZeros()); - } + // UNDONE: more! + } } diff --git storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java new file mode 100644 index 0000000..a18f3a4 --- /dev/null +++ storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java @@ -0,0 +1,716 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.BigInteger; + +/** + * FastHiveDecimal is a mutable fast decimal object. It is the base class for both the + * HiveDecimal and HiveDecimalWritable classes. All fast* methods are protected so they + * cannot be accessed by clients of HiveDecimal and HiveDecimalWritable. HiveDecimal ensures + * it creates new objects when the value changes since it provides immutable semantics; + * HiveDecimalWritable does not create new objects since it provides mutable semantics. + * + * The methods in this class are shells that pickup the member variables from FastHiveDecimal + * parameters and pass them as individual parameters to static methods in the FastHiveDecimalImpl + * class that do the real work. + */ +public class FastHiveDecimal { + + + protected static final int FAST_ROUND_FLOOR = BigDecimal.ROUND_FLOOR; + protected static final int FAST_ROUND_CEILING = BigDecimal.ROUND_CEILING; + protected static final int FAST_ROUND_HALF_UP = BigDecimal.ROUND_HALF_UP; + protected static final int FAST_ROUND_HALF_EVEN = BigDecimal.ROUND_HALF_EVEN; + + // These two rounding modes are currently not exposed by HiveDecimal. + protected static final int FAST_ROUND_DOWN = BigDecimal.ROUND_DOWN; + protected static final int FAST_ROUND_UP = BigDecimal.ROUND_UP; + + /* + * We use protected for the fields so the FastHiveDecimalImpl class can access them. Other + * classes including HiveDecimal should not access these fields directly. + * + * See the FastHiveDecimalImpl class for the representation "theory" of fast decimals. + */ + protected int fastSignum; + + protected long fast2; + protected long fast1; + protected long fast0; + + protected int fastIntegerDigitCount; + protected int fastScale; + + protected FastHiveDecimal() { + fastReset(); + } + + protected FastHiveDecimal(FastHiveDecimal fastDec) { + this(); + fastSignum = fastDec.fastSignum; + fast0 = fastDec.fast0; + fast1 = fastDec.fast1; + fast2 = fastDec.fast2; + fastIntegerDigitCount = fastDec.fastIntegerDigitCount; + fastScale = fastDec.fastScale; + } + + protected FastHiveDecimal(int fastSignum, FastHiveDecimal fastDec) { + this(); + this.fastSignum = fastSignum; + fast0 = fastDec.fast0; + fast1 = fastDec.fast1; + fast2 = fastDec.fast2; + fastIntegerDigitCount = fastDec.fastIntegerDigitCount; + fastScale = fastDec.fastScale; + } + + protected FastHiveDecimal( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + this(); + this.fastSignum = fastSignum; + this.fast0 = fast0; + this.fast1 = fast1; + this.fast2 = fast2; + this.fastIntegerDigitCount = fastIntegerDigitCount; + this.fastScale = fastScale; + } + + protected FastHiveDecimal(long longValue) { + this(); + FastHiveDecimalImpl.fastSetFromLong(longValue, this); + } + + protected FastHiveDecimal(String string) { + this(); + FastHiveDecimalImpl.fastSetFromString(string, false, this); + } + + protected void fastReset() { + fastSignum = 0; + fast0 = 0; + fast1 = 0; + fast2 = 0; + fastIntegerDigitCount = 0; + fastScale = 0; + } + + protected void fastSet(FastHiveDecimal fastDec) { + fastSignum = fastDec.fastSignum; + fast0 = fastDec.fast0; + fast1 = fastDec.fast1; + fast2 = fastDec.fast2; + fastIntegerDigitCount = fastDec.fastIntegerDigitCount; + fastScale = fastDec.fastScale; + } + + protected void fastSet( + int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) { + this.fastSignum = fastSignum; + this.fast0 = fast0; + this.fast1 = fast1; + this.fast2 = fast2; + this.fastIntegerDigitCount = fastIntegerDigitCount; + this.fastScale = fastScale; + } + + protected static final int FAST_MAX_SCALE = 38; + protected static final int FAST_MAX_PRECISION = 38; + + protected static final String STRING_ENFORCE_PRECISION_OUT_OF_RANGE = + "Decimal precision out of allowed range [1," + FAST_MAX_PRECISION + "]"; + protected static final String STRING_ENFORCE_SCALE_OUT_OF_RANGE = + "Decimal scale out of allowed range [0," + FAST_MAX_SCALE + "]"; + protected static final String STRING_ENFORCE_SCALE_LESS_THAN_EQUAL_PRECISION = + "Decimal scale must be less than or equal to precision"; + + protected boolean fastSetFromBigDecimal( + BigDecimal bigDecimal, boolean allowRounding) { + return + FastHiveDecimalImpl.fastSetFromBigDecimal( + bigDecimal, allowRounding, this); + } + + protected boolean fastSetFromBigInteger( + BigInteger bigInteger) { + return + FastHiveDecimalImpl.fastSetFromBigInteger( + bigInteger, this); + } + + protected boolean fastSetFromBigIntegerAndScale( + BigInteger bigInteger, int scale) { + return + FastHiveDecimalImpl.fastSetFromBigInteger( + bigInteger, scale, this); + } + + protected boolean fastSetFromString(String string, boolean trimBlanks) { + byte[] bytes = string.getBytes(); + return + fastSetFromBytes( + bytes, 0, bytes.length, trimBlanks); + } + + protected boolean fastSetFromBytes(byte[] bytes, int offset, int length, boolean trimBlanks) { + return + FastHiveDecimalImpl.fastSetFromBytes( + bytes, offset, length, trimBlanks, this); + } + + protected boolean fastSetFromDigitsOnlyBytesAndScale( + boolean isNegative, byte[] bytes, int offset, int length, int scale) { + return + FastHiveDecimalImpl.fastSetFromDigitsOnlyBytesAndScale( + isNegative, bytes, offset, length, scale, this); + } + + protected void fastSetFromInt(int intValue) { + FastHiveDecimalImpl.fastSetFromInt(intValue, this); + } + + protected void fastSetFromLong(long longValue) { + FastHiveDecimalImpl.fastSetFromLong(longValue, this); + } + + protected boolean fastSetFromLongAndScale(long longValue, int scale) { + return + FastHiveDecimalImpl.fastSetFromLongAndScale( + longValue, scale, this); + } + + protected boolean fastSetFromFloat(float floatValue) { + return + FastHiveDecimalImpl.fastSetFromFloat( + floatValue, this); + } + + protected boolean fastSetFromDouble(double doubleValue) { + return + FastHiveDecimalImpl.fastSetFromDouble( + doubleValue, this); + } + + protected void fastFractionPortion() { + FastHiveDecimalImpl.fastFractionPortion( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, + this); + } + + protected void fastIntegerPortion() { + FastHiveDecimalImpl.fastIntegerPortion( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, + this); + } + + protected static final int FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ = 8 * 3; + + protected boolean fastSerializationUtilsRead( + InputStream inputStream, int scale, + byte[] scratchBytes) throws IOException, EOFException { + return + FastHiveDecimalImpl.fastSerializationUtilsRead( + inputStream, scale, scratchBytes, this); + } + + protected boolean fastSetFromBigIntegerBytesAndScale( + byte[] bytes, int offset, int length, int scale) { + return FastHiveDecimalImpl.fastSetFromBigIntegerBytesAndScale( + bytes, offset, length, scale, this); + } + + protected static final int SCRATCH_LONGS_LEN_FAST_SERIALIZATION_UTILS_WRITE = 6; + + protected boolean fastSerializationUtilsWrite(OutputStream outputStream, + long[] scratchLongs) + throws IOException { + return + FastHiveDecimalImpl.fastSerializationUtilsWrite( + outputStream, + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, + scratchLongs); + } + + // The fastBigIntegerBytes method returns 3 56 bit (7 byte) words and a possible sign byte. + // However, the fastBigIntegerBytes can take on trailing zeroes -- so make it larger. + protected static final int FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES = 1 + 48; + protected static final int FAST_SCRATCH_LONGS_LEN = 6; + + protected int fastBigIntegerBytes( + long[] scratchLongs, byte[] buffer) { + return + FastHiveDecimalImpl.fastBigIntegerBytes( + fastSignum, fast0, fast1, fast2, + scratchLongs, buffer); + } + + protected int fastBigIntegerBytesScaled( + int scale, + long[] scratchLongs, byte[] buffer) { + return + FastHiveDecimalImpl.fastBigIntegerBytesScaled( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, + scale, + scratchLongs, buffer); + } + + protected boolean fastIsByte() { + return + FastHiveDecimalImpl.fastIsByte( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected byte fastByteValueClip() { + return + FastHiveDecimalImpl.fastByteValueClip( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected boolean fastIsShort() { + return + FastHiveDecimalImpl.fastIsShort( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected short fastShortValueClip() { + return + FastHiveDecimalImpl.fastShortValueClip( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected boolean fastIsInt() { + return + FastHiveDecimalImpl.fastIsInt( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected int fastIntValueClip() { + return + FastHiveDecimalImpl.fastIntValueClip( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected boolean fastIsLong() { + return + FastHiveDecimalImpl.fastIsLong( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected long fastLongValueClip() { + return + FastHiveDecimalImpl.fastLongValueClip( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected float fastFloatValue() { + return + FastHiveDecimalImpl.fastFloatValue( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected double fastDoubleValue() { + return + FastHiveDecimalImpl.fastDoubleValue( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected BigInteger fastBigIntegerValue() { + return + FastHiveDecimalImpl.fastBigIntegerValue( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected BigDecimal fastBigDecimalValue() { + return + FastHiveDecimalImpl.fastBigDecimalValue( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale); + } + + protected int fastScale() { + return fastScale; + } + + protected int fastSignum() { + return fastSignum; + } + + protected int fastCompareTo(FastHiveDecimal right) { + return + FastHiveDecimalImpl.fastCompareTo( + fastSignum, fast0, fast1, fast2, + fastScale, + right.fastSignum, right.fast0, right.fast1, right.fast2, + right.fastScale); + } + + protected static int fastCompareTo(FastHiveDecimal left, FastHiveDecimal right) { + return + FastHiveDecimalImpl.fastCompareTo( + left.fastSignum, left.fast0, left.fast1, left.fast2, + left.fastScale, + right.fastSignum, right.fast0, right.fast1, right.fast2, + right.fastScale); + } + + protected boolean fastEquals(FastHiveDecimal that) { + return + FastHiveDecimalImpl.fastEquals( + fastSignum, fast0, fast1, fast2, + fastScale, + that.fastSignum, that.fast0, that.fast1, that.fast2, + that.fastScale); + } + + protected void fastAbs() { + fastSignum = 1; + } + + protected void fastNegate() { + if (fastSignum == 0) { + return; + } + fastSignum = (fastSignum == 1 ? -1 : 1); + } + + protected int fastHashCode() { + return + FastHiveDecimalImpl.fastHashCode( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected int fastOldHiveDecimalHashCode() { + return + FastHiveDecimalImpl.fastOldHiveDecimalHashCode( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + protected int fastIntegerDigitCount() { + return fastIntegerDigitCount; + } + + protected int fastSqlPrecision() { + return + FastHiveDecimalImpl.fastSqlPrecision( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale); + } + + protected int fastRawPrecision() { + return + FastHiveDecimalImpl.fastRawPrecision( + fastSignum, fast0, fast1, fast2); + } + + protected boolean fastScaleByPowerOfTen( + int n, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastScaleByPowerOfTen( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, + n, + fastResult); + } + + protected static String fastRoundingModeToString(int roundingMode) { + String roundingModeString; + switch (roundingMode) { + case FAST_ROUND_DOWN: + roundingModeString = "FAST_ROUND_DOWN"; + break; + case FAST_ROUND_UP: + roundingModeString = "FAST_ROUND_UP"; + break; + case FAST_ROUND_FLOOR: + roundingModeString = "FAST_ROUND_FLOOR"; + break; + case FAST_ROUND_CEILING: + roundingModeString = "FAST_ROUND_CEILING"; + break; + case FAST_ROUND_HALF_UP: + roundingModeString = "FAST_ROUND_HALF_UP"; + break; + case FAST_ROUND_HALF_EVEN: + roundingModeString = "FAST_ROUND_HALF_EVEN"; + break; + default: + roundingModeString = "Unknown"; + } + return roundingModeString + " (" + roundingMode + ")"; + } + + protected boolean fastRound( + int newScale, int roundingMode, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastRound( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + newScale, roundingMode, + fastResult); + } + + protected boolean isAllZeroesBelow( + int power) { + return + FastHiveDecimalImpl.isAllZeroesBelow( + fastSignum, fast0, fast1, fast2, power); + } + + protected boolean fastEnforcePrecisionScale( + int maxPrecision, int maxScale) { + if (maxPrecision <= 0 || maxPrecision > FAST_MAX_PRECISION) { + return false; + } + if (maxScale < 0 || maxScale > FAST_MAX_SCALE) { + return false; + } + /* + if (!fastIsValid()) { + fastRaiseInvalidException(); + } + */ + FastCheckPrecisionScaleStatus status = + FastHiveDecimalImpl.fastCheckPrecisionScale( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + maxPrecision, maxScale); + switch (status) { + case NO_CHANGE: + return true; + case OVERFLOW: + return false; + case UPDATE_SCALE_DOWN: + { + if (!FastHiveDecimalImpl.fastUpdatePrecisionScale( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + maxPrecision, maxScale, status, + this)) { + return false; + } + /* + if (!fastIsValid()) { + fastRaiseInvalidException(); + } + */ + return true; + } + default: + throw new RuntimeException("Unknown fast decimal check precision and scale status " + status); + } + } + + protected FastCheckPrecisionScaleStatus fastCheckPrecisionScale( + int maxPrecision, int maxScale) { + return + FastHiveDecimalImpl.fastCheckPrecisionScale( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + maxPrecision, maxScale); + } + + protected static enum FastCheckPrecisionScaleStatus { + NO_CHANGE, + OVERFLOW, + UPDATE_SCALE_DOWN; + } + + protected boolean fastUpdatePrecisionScale( + int maxPrecision, int maxScale, FastCheckPrecisionScaleStatus status, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastUpdatePrecisionScale( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + maxPrecision, maxScale, status, + fastResult); + } + + protected boolean fastAdd( + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastAdd( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + protected boolean fastSubtract( + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastSubtract( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + protected boolean fastMultiply( + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastMultiply( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + protected boolean fastRemainder( + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastRemainder( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + protected boolean fastDivide( + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastDivide( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + protected boolean fastPow( + int exponent, + FastHiveDecimal fastResult) { + return + FastHiveDecimalImpl.fastPow( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale, + exponent, + fastResult); + } + + protected String fastToString( + byte[] scratchBuffer) { + return + FastHiveDecimalImpl.fastToString( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, -1, + scratchBuffer); + } + + protected String fastToString() { + return + FastHiveDecimalImpl.fastToString( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, -1); + } + + protected String fastToFormatString(int formatScale) { + return + FastHiveDecimalImpl.fastToFormatString( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + formatScale); + } + + protected int fastToFormatString( + int formatScale, + byte[] scratchBuffer) { + return + FastHiveDecimalImpl.fastToFormatString( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + formatScale, + scratchBuffer); + } + + protected String fastToDigitsOnlyString() { + return + FastHiveDecimalImpl.fastToDigitsOnlyString( + fast0, fast1, fast2, + fastIntegerDigitCount); + } + + // Sign, zero, dot, 2 * digits (to support toFormatString which can add a lot of trailing zeroes). + public final static int FAST_SCRATCH_BUFFER_LEN_TO_BYTES = + 1 + 1 + 1 + 2 * FastHiveDecimalImpl.MAX_DECIMAL_DIGITS; + + protected int fastToBytes( + byte[] scratchBuffer) { + return + FastHiveDecimalImpl.fastToBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, -1, + scratchBuffer); + } + + protected int fastToFormatBytes( + int formatScale, + byte[] scratchBuffer) { + return + FastHiveDecimalImpl.fastToFormatBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + formatScale, + scratchBuffer); + } + + protected int fastToDigitsOnlyBytes( + byte[] scratchBuffer) { + return + FastHiveDecimalImpl.fastToDigitsOnlyBytes( + fast0, fast1, fast2, + fastIntegerDigitCount, + scratchBuffer); + } + + @Override + public String toString() { + return + FastHiveDecimalImpl.fastToString( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, -1); + } + + public boolean fastIsValid() { + return FastHiveDecimalImpl.fastIsValid(this); + } + + public void fastRaiseInvalidException() { + FastHiveDecimalImpl.fastRaiseInvalidException(this); + } + + public void fastRaiseInvalidException(String parameters) { + FastHiveDecimalImpl.fastRaiseInvalidException(this, parameters); + } +} \ No newline at end of file diff --git storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java new file mode 100644 index 0000000..f186dca --- /dev/null +++ storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java @@ -0,0 +1,9063 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import java.util.Arrays; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.nio.charset.StandardCharsets; + +import org.apache.commons.lang.StringUtils; + +/** + * This class is a companion to the FastHiveDecimal class that separates the essential of code + * out of FastHiveDecimal into static methods in this class so that they can be used directly + * by vectorization to implement decimals by storing the fast0, fast1, and fast2 longs and + * the fastSignum, fastScale, etc ints in the DecimalColumnVector class. + */ +public class FastHiveDecimalImpl extends FastHiveDecimal { + + /** + * Representation of fast decimals. + * + * We use 3 long words to store the 38 digits of fast decimals and and 3 integers for sign, + * integer digit count, and scale. + * + * The lower and middle long words store 16 decimal digits each; the high long word has + * 6 decimal digits; total 38 decimal digits. + * + * We do not try and represent fast decimal value as an unsigned 128 bit binary number in 2 longs. + * There are several important reasons for this. + * + * The effort to represent an unsigned 128 integer in 2 Java signed longs is very difficult, + * error prone, hard to debug, and not worth the effort. + * + * The focus here is on reusing memory (i.e. with HiveDecimalWritable) as often as possible. + * Reusing memory is good for grouping of fast decimal objects and related objects in CPU cache + * lines for fast memory access and eliminating the cost of allocating temporary objects and + * reducing the global cost of garbage collection. + * + * In other words, we are focused on avoiding the poor performance of Java general immutable + * objects. + * + * Reducing memory size or being concerned about the memory size of using 3 longs vs. 2 longs + * for 128 unsigned bits is not the focus here. + * + * Besides focusing on reusing memory, storing a limited number (16) decimal digits in the longs + * rather than compacting the value into all binary bits of 2 longs has a surprising benefit. + * + * One big part of implementing decimals turns out to be manipulating decimal digits. + * + * For example, rounding a decimal involves trimming off lower digits or clearing lower digits. + * Since radix 10 digits cannot be masked with binary masks, we use division and multiplication + * using powers of 10. We can easily manipulate the decimal digits in a long word using simple + * integer multiplication / division without doing emulated 128 binary bit multiplication / + * division. + * + * For example, say we want to scale (round) down the fraction digits of a decimal. + * + * final long divideFactor = powerOfTenTable[scaleDown]; + * final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; + * + * result0 = + * fast0 / divideFactor + * + ((fast1 % divideFactor) * multiplyFactor); + * result1 = + * fast1 / divideFactor + * + ((fast2 % divideFactor) * multiplyFactor); + * result2 = + * fast2 / divideFactor; + * + * It also turns out to do addition and subtraction of decimals with different scales can involve + * overlap using more than 3 long words. Manipulating extra words is a natural extension of + * the existing techniques. + * + * Why is the decimal digits representation easier to debug? You can see the decimal digits in + * the 3 long words and do not have to convert binary words to decimal to see the value. + * + * 16 decimal digits for a long was choose so that an int can have 1/2 or 8 decimal digits during + * multiplication of int half words so intermediate multiplication results do not overflow a long. + * And, so addition overflow is well below the sign bit of a long. + */ + + // Code Sections: + // Initialize (fastSetFrom*). + // Take Integer or Fractional Portion. + // Binary to Decimal Conversion. + // Decimal to Binary Conversion.r + // Emulate SerializationUtils Deserialization used by ORC. + // Emulate SerializationUtils Serialization used by ORC. + // Emulate BigInteger Deserialization used by LazyBinary and others. + // Emulate BigInteger Serialization used by LazyBinary and others. + // Decimal to Integer Conversion. + // Decimal to Non-Integer Conversion. + // Decimal Comparison. + // Decimal Rounding. + // Decimal Scale Up/Down. + // Decimal Precision / Trailing Zeroes. + // Decimal Addition / Subtraction. + // Decimal Multiply. + // Decimal Division / Remainder. + // Decimal String Formatting. + // Decimal Validation. + // Decimal Debugging. + + private static final long[] powerOfTenTable = { + 1L, // 0 + 10L, + 100L, + 1000L, + 10000L, + 100000L, + 1000000L, + 10000000L, + 100000000L, // 8 + 1000000000L, + 10000000000L, + 100000000000L, + 1000000000000L, + 10000000000000L, + 100000000000000L, + 1000000000000000L, + 10000000000000000L // 16 + }; + + public static final int MAX_DECIMAL_DIGITS = 38; + + /** + * Int: 8 decimal digits. An even number and 1/2 of MAX_LONGWORD_DECIMAL. + */ + private static final int INTWORD_DECIMAL_DIGITS = 8; + private static final int MAX_INTWORD_DECIMAL = (int) powerOfTenTable[INTWORD_DECIMAL_DIGITS] - 1; + private static final int MULTIPLER_INTWORD_DECIMAL = (int) powerOfTenTable[INTWORD_DECIMAL_DIGITS]; + + /** + * Long: 16 decimal digits. An even number and twice MAX_INTWORD_DECIMAL. + */ + private static final int LONGWORD_DECIMAL_DIGITS = 16; + private static final long MAX_LONGWORD_DECIMAL = powerOfTenTable[LONGWORD_DECIMAL_DIGITS] - 1; + private static final long MULTIPLER_LONGWORD_DECIMAL = powerOfTenTable[LONGWORD_DECIMAL_DIGITS]; + + private static final int TWO_X_LONGWORD_DECIMAL_DIGITS = 2 * LONGWORD_DECIMAL_DIGITS; + private static final int THREE_X_LONGWORD_DECIMAL_DIGITS = 3 * LONGWORD_DECIMAL_DIGITS; + private static final int FOUR_X_LONGWORD_DECIMAL_DIGITS = 4 * LONGWORD_DECIMAL_DIGITS; + + // 38 decimal maximum - 32 digits in 2 lower longs (6 digits here). + private static final int HIGHWORD_DECIMAL_DIGITS = MAX_DECIMAL_DIGITS - TWO_X_LONGWORD_DECIMAL_DIGITS; + private static final long MAX_HIGHWORD_DECIMAL = + powerOfTenTable[HIGHWORD_DECIMAL_DIGITS] - 1; + + private static long HIGHWORD_DIVIDE_FACTOR = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - HIGHWORD_DECIMAL_DIGITS]; + private static long HIGHWORD_MULTIPLY_FACTOR = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS]; + + // 38 * 2 or 76 full decimal maximum - (64 + 8) digits in 4 lower longs (4 digits here). + private static final long FULL_MAX_HIGHWORD_DECIMAL = + powerOfTenTable[MAX_DECIMAL_DIGITS * 2 - (FOUR_X_LONGWORD_DECIMAL_DIGITS + INTWORD_DECIMAL_DIGITS)] - 1; + + /** + * BigInteger constants. + */ + + private static final BigInteger BIG_INTEGER_TWO = BigInteger.valueOf(2); + private static final BigInteger BIG_INTEGER_FIVE = BigInteger.valueOf(5); + private static final BigInteger BIG_INTEGER_TEN = BigInteger.valueOf(10); + + public static final BigInteger BIG_INTEGER_MAX_DECIMAL = + BIG_INTEGER_TEN.pow(MAX_DECIMAL_DIGITS).subtract(BigInteger.ONE); + + private static final BigInteger BIG_INTEGER_MAX_LONGWORD_DECIMAL = + BigInteger.valueOf(MAX_LONGWORD_DECIMAL); + + private static final BigInteger BIG_INTEGER_LONGWORD_MULTIPLIER = + BigInteger.ONE.add(BIG_INTEGER_MAX_LONGWORD_DECIMAL); + private static final BigInteger BIG_INTEGER_LONGWORD_MULTIPLIER_2X = + BIG_INTEGER_LONGWORD_MULTIPLIER.multiply(BIG_INTEGER_LONGWORD_MULTIPLIER); + private static final BigInteger BIG_INTEGER_LONGWORD_MULTIPLIER_3X = + BIG_INTEGER_LONGWORD_MULTIPLIER_2X.multiply(BIG_INTEGER_LONGWORD_MULTIPLIER); + private static final BigInteger BIG_INTEGER_LONGWORD_MULTIPLIER_4X = + BIG_INTEGER_LONGWORD_MULTIPLIER_3X.multiply(BIG_INTEGER_LONGWORD_MULTIPLIER); + + private static final BigInteger BIG_INTEGER_MAX_HIGHWORD_DECIMAL = + BigInteger.valueOf(MAX_HIGHWORD_DECIMAL); + private static final BigInteger BIG_INTEGER_HIGHWORD_MULTIPLIER = + BigInteger.ONE.add(BIG_INTEGER_MAX_HIGHWORD_DECIMAL); + + //************************************************************************************************ + // Initialize (fastSetFrom*). + + private static void doRaiseSetFromBytesInvalid( + byte[] bytes, int offset, int length, + FastHiveDecimal fastResult) { + final int end = offset + length; + throw new RuntimeException( + "Invalid fast decimal \"" + + new String(bytes, offset, end, StandardCharsets.UTF_8) + "\"" + + " fastSignum " + fastResult.fastSignum + " fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2 + + " fastIntegerDigitCount " + fastResult.fastIntegerDigitCount +" fastScale " + fastResult.fastScale + + " stack trace: " + getStackTraceAsSingleLine(Thread.currentThread().getStackTrace())); + } + + /** + * Scan a byte array slice for a decimal number in UTF-8 characters. + * + * Syntax: + * [+|-][integerPortion][.[fractionalDigits]][{E|e}[+|-]exponent] + * // Where at least one integer or fractional + * // digit is required... + * + * @param bytes + * @param offset + * @param length + * @param trimBlanks + * @param fastResult + * @return + */ + public static boolean fastSetFromBytes(byte[] bytes, int offset, int length, boolean trimBlanks, + FastHiveDecimal fastResult) { + + final int bytesLength = bytes.length; + + if (offset < 0 || offset >= bytesLength) { + return false; + } + final int end = offset + length; + if (end <= offset || end > bytesLength) { + return false; + } + + // We start here with at least one byte. + int index = offset; + + if (trimBlanks) { + while (bytes[index] == BYTE_BLANK) { + if (++index >= end) { + return false; + } + } + } + + // Started with a few ideas from BigDecimal(char[] in, int offset, int len) constructor... + // But soon became very fast decimal specific. + + boolean isNegative = false; + if (bytes[index] == BYTE_MINUS) { + isNegative = true; + if (++index >= end) { + return false; + } + } else if (bytes[index] == BYTE_PLUS) { + if (++index >= end) { + return false; + } + } + + int precision = 0; + + // We fill starting with highest digit in highest longword and move down. At end will + // will shift everything down if necessary. + + int longWordIndex = 0; // Where 0 is the highest longword; 1 is middle longword, etc. + + int digitNum = HIGHWORD_DECIMAL_DIGITS; + long multiplier = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1]; + + int digitValue; + long longWord = 0; + + long fast0 = 0; + long fast1 = 0; + long fast2 = 0; + + byte work; + + // Parse integer portion. + + boolean haveInteger = false; + while (true) { + work = bytes[index]; + if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { + break; + } + haveInteger = true; + if (precision == 0 && work == BYTE_DIGIT_ZERO) { + // Ignore leading zeroes. + if (++index >= end) { + break; + } + continue; + } + digitValue = work - BYTE_DIGIT_ZERO; + if (digitNum == 0) { + + // Integer parsing move to next lower longword. + + // Save previous longword. + if (longWordIndex == 0) { + fast2 = longWord; + } else if (longWordIndex == 1) { + fast1 = longWord; + } else if (longWordIndex == 2) { + + // We have filled FAST_MAX_PRECISION digits and have no more room in our limit precision + // fast decimal. + return false; + } + longWordIndex++; + digitNum = LONGWORD_DECIMAL_DIGITS; + multiplier = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - 1]; + longWord = 0; + } + longWord += digitValue * multiplier; + multiplier /= 10; + digitNum--; + precision++; + if (++index >= end) { + break; + } + } + + // Try to eat a dot now since it could be the end. + boolean sawDot = false; + if (index < end && bytes[index] == BYTE_DOT) { + sawDot = true; + index++; + } + + if (sawDot && index < end) { + work = bytes[index]; + if (work >= BYTE_DIGIT_ZERO && work <= BYTE_DIGIT_NINE) { + digitValue = work - BYTE_DIGIT_ZERO; + } + } + + // Try to eat trailing blank padding. + if (trimBlanks && index < end && bytes[index] == BYTE_BLANK) { + index++; + while (index < end && bytes[index] == BYTE_BLANK) { + index++; + } + if (index < end) { + // Junk after trailing blank padding. + return false; + } + } + + // Just an integer with optional dot and optional blank padding? + if (index >= end) { + + // We hit the end after getting integer and optional dot and optional blank padding. + + if (precision == 0) { + // Zero(es). + /* + if (!fastResult.fastIsValid()) { + doRaiseSetFromBytesInvalid(bytes, offset, length, fastResult); + } + */ + return true; + } + // Save last longword. + if (longWordIndex == 0) { + fast2 = longWord; + } else if (longWordIndex == 1) { + fast1 = longWord; + } else { + fast0 = longWord; + } + fastResult.fastSignum = (isNegative ? -1 : 1); + fastResult.fastIntegerDigitCount = precision; + fastResult.fastScale = 0; + final int scaleDown = FAST_MAX_PRECISION - precision; + if (scaleDown > 0) { + doFastScaleDown(fast0, fast1, fast2, scaleDown, fastResult); + } else { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } + /* + if (!fastResult.fastIsValid()) { + doRaiseSetFromBytesInvalid(bytes, offset, length, fastResult); + } + */ + return true; + } + + int integerDigitCount = precision; + + int nonTrailingZeroScale = 0; + boolean roundingNecessary = false; + if (sawDot && (bytes[index] != BYTE_EXPONENT_UPPER && bytes[index] != BYTE_EXPONENT_LOWER)) { + + // Parse fraction portion. + + while (true) { + work = bytes[index]; + if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { + break; + } + digitValue = work - BYTE_DIGIT_ZERO; + if (digitNum == 0) { + + // Fraction digit parsing move to next lower longword. + + // Save previous longword. + if (longWordIndex == 0) { + fast2 = longWord; + } else if (longWordIndex == 1) { + fast1 = longWord; + } else if (longWordIndex == 2) { + + // We have filled FAST_MAX_PRECISION digits and have no more room in our limit precision + // fast decimal. However, since we are processing fractional digits, we do rounding. + // away. + if (digitValue >= 5) { + roundingNecessary = true; + } + + // Scan through any remaining digits... + while (++index < end) { + work = bytes[index]; + if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { + break; + } + } + break; + } + longWordIndex++; + digitNum = LONGWORD_DECIMAL_DIGITS; + multiplier = powerOfTenTable[digitNum - 1]; + longWord = 0; + } + longWord += digitValue * multiplier; + multiplier /= 10; + digitNum--; + precision++; + if (digitValue != 0) { + nonTrailingZeroScale = precision - integerDigitCount; + } + if (++index >= end) { + break; + } + } + } + + boolean haveExponent = false; + if (index < end && + (bytes[index] == BYTE_EXPONENT_UPPER || bytes[index] == BYTE_EXPONENT_LOWER)) { + haveExponent = true; + index++; + if (index >= end) { + // More required. + return false; + } + } + + // Did we see any integer digits or see any fraction digits? I.e. just a sign and/or dot? + if (!haveInteger && integerDigitCount == precision) { + return false; + } + + // Save last longword. + if (longWordIndex == 0) { + fast2 = longWord; + } else if (longWordIndex == 1) { + fast1 = longWord; + } else { + fast0 = longWord; + } + + int trailingZeroesScale = precision - integerDigitCount; + if (integerDigitCount == 0 && nonTrailingZeroScale == 0) { + // Zero(es). + } else { + fastResult.fastSignum = (isNegative ? -1 : 1); + fastResult.fastIntegerDigitCount = integerDigitCount; + fastResult.fastScale = nonTrailingZeroScale; + final int trailingZeroCount = trailingZeroesScale - fastResult.fastScale; + final int scaleDown = FAST_MAX_PRECISION - precision + trailingZeroCount; + if (scaleDown > 0) { + doFastScaleDown(fast0, fast1, fast2, scaleDown, fastResult); + } else { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } + /* + if (!fastResult.fastIsValid()) { + doRaiseSetFromBytesInvalid(bytes, offset, length, fastResult); + } + */ + } + + if (roundingNecessary) { + + if (fastResult.fastSignum == 0) { + fastResult.fastSignum = (isNegative ? -1 : 1); + fastResult.fast0 = 1; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = FAST_MAX_SCALE; + } else { + if (!fastAdd( + fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale, + fastResult.fastSignum, 1, 0, 0, 0, trailingZeroesScale, + fastResult)) { + return false; + } + /* + if (!fastResult.fastIsValid()) { + doRaiseSetFromBytesInvalid(bytes, offset, length, fastResult); + } + */ + } + } + + if (!haveExponent) { + + // Try to eat trailing blank padding. + if (trimBlanks && index < end && bytes[index] == BYTE_BLANK) { + index++; + while (index < end && bytes[index] == BYTE_BLANK) { + index++; + } + } + if (index < end) { + // Junk after trailing blank padding. + return false; + } + /* + if (!fastResult.fastIsValid()) { + doRaiseSetFromBytesInvalid(bytes, offset, length, fastResult); + } + */ + return true; + } + + // At this point, we have seen the exponent letter E or e and have decimal information as: + // isNegative, precision, integerDigitCount, nonTrailingZeroScale, and + // fast0, fast1, fast2. + // + // After we determine the exponent, we will do appropriate scaling and fill in fastResult. + + boolean isExponentNegative = false; + if (bytes[index] == BYTE_MINUS) { + isExponentNegative = true; + if (++index >= end) { + return false; + } + } else if (bytes[index] == BYTE_PLUS) { + if (++index >= end) { + return false; + } + } + + long exponent = 0; + multiplier = 1; + while (true) { + work = bytes[index]; + if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { + break; + } + if (multiplier > 10) { + // Power of ten way beyond our precision/scale... + return false; + } + digitValue = work - BYTE_DIGIT_ZERO; + if (digitValue != 0 || exponent != 0) { + exponent = exponent * 10 + digitValue; + multiplier *= 10; + } + if (++index >= end) { + break; + } + } + if (isExponentNegative) { + exponent = -exponent; + } + + // Try to eat trailing blank padding. + if (trimBlanks && index < end && bytes[index] == BYTE_BLANK) { + index++; + while (index < end && bytes[index] == BYTE_BLANK) { + index++; + } + } + if (index < end) { + // Junk after exponent. + return false; + } + + + if (integerDigitCount == 0 && nonTrailingZeroScale == 0) { + // Zero(es). + /* + if (!fastResult.fastIsValid()) { + doRaiseSetFromBytesInvalid(bytes, offset, length, fastResult); + } + */ + return true; + } + + if (exponent == 0) { + + // No effect since 10^0 = 1. + + } else { + + // We for these input with exponents, we have at this point an intermediate decimal, + // an exponent power, and a result: + // + // intermediate + // input decimal exponent result + // 701E+1 701 scale 0 +1 7010 scale 0 + // 3E+4 3 scale 0 +4 3 scale 0 + // 3.223E+9 3.223 scale 3 +9 3223000000 scale 0 + // 0.009E+10 0.009 scale 4 +10 90000000 scale 0 + // 0.3221E-2 0.3221 scale 4 -2 0.003221 scale 6 + // 0.00223E-20 0.00223 scale 5 -20 0.0000000000000000000000223 scale 25 + // + + if (!fastScaleByPowerOfTen( + fastResult, + (int) exponent, + fastResult)) { + return false; + } + /* + if (!fastResult.fastIsValid()) { + doRaiseSetFromBytesInvalid(bytes, offset, length, fastResult); + } + */ + } + + final int trailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (trailingZeroCount > 0) { + doFastScaleDown( + fastResult, + trailingZeroCount, + fastResult); + fastResult.fastScale -= trailingZeroCount; + } + + /* + if (!fastResult.fastIsValid()) { + doRaiseSetFromBytesInvalid(bytes, offset, length, fastResult); + } + */ + + return true; + } + + /** + * Scans a byte array slice for unscaled integer (i.e. digits only) in UTF-8 characters and + * forms a decimal from that and a specified scale. Designed for BinarySortable serialization + * format. + * + * @param isNegative + * @param bytes + * @param offset + * @param length + * @param scale + * @param fastResult + * @return + */ + public static boolean fastSetFromDigitsOnlyBytesAndScale( + boolean isNegative, byte[] bytes, int offset, int length, int scale, + FastHiveDecimal fastResult) { + + final int bytesLength = bytes.length; + + if (offset < 0 || offset >= bytesLength) { + return false; + } + final int end = offset + length; + if (end <= offset || end > bytesLength) { + return false; + } + + // We start here with at least one byte. + int index = offset; + + // A stripped down version of fastSetFromBytes. + + int precision = 0; + + // We fill starting with highest digit in highest longword and move down. At end will + // will shift everything down if necessary. + + int longWordIndex = 0; // Where 0 is the highest longword; 1 is middle longword, etc. + + int digitNum = HIGHWORD_DECIMAL_DIGITS; + long multiplier = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1]; + + int digitValue; + long longWord = 0; + + long fast0 = 0; + long fast1 = 0; + long fast2 = 0; + + byte work; + + // Parse digits. + + boolean haveInteger = false; + while (true) { + work = bytes[index]; + if (work < BYTE_DIGIT_ZERO || work > BYTE_DIGIT_NINE) { + break; + } + haveInteger = true; + if (precision == 0 && work == BYTE_DIGIT_ZERO) { + // Ignore leading zeroes. + if (++index >= end) { + break; + } + continue; + } + digitValue = work - BYTE_DIGIT_ZERO; + if (digitNum == 0) { + + // Integer parsing move to next lower longword. + + // Save previous longword. + if (longWordIndex == 0) { + fast2 = longWord; + } else if (longWordIndex == 1) { + fast1 = longWord; + } else if (longWordIndex == 2) { + + // We have filled FAST_MAX_PRECISION digits and have no more room in our limit precision + // fast decimal. + return false; + } + longWordIndex++; + digitNum = LONGWORD_DECIMAL_DIGITS; + multiplier = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - 1]; + longWord = 0; + } + longWord += digitValue * multiplier; + multiplier /= 10; + digitNum--; + precision++; + if (++index >= end) { + break; + } + } + + // Just an digits? + if (index < end) { + return false; + } + + if (precision == 0) { + // Zero(es). + /* + if (!fastResult.fastIsValid()) { + doRaiseSetFromBytesInvalid(bytes, offset, length, fastResult); + } + */ + return true; + } + // Save last longword. + if (longWordIndex == 0) { + fast2 = longWord; + } else if (longWordIndex == 1) { + fast1 = longWord; + } else { + fast0 = longWord; + } + fastResult.fastSignum = (isNegative ? -1 : 1); + fastResult.fastIntegerDigitCount = Math.max(0, precision - scale); + fastResult.fastScale = scale; + final int scaleDown = FAST_MAX_PRECISION - precision; + if (scaleDown > 0) { + doFastScaleDown(fast0, fast1, fast2, scaleDown, fastResult); + } else { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } + /* + if (!fastResult.fastIsValid()) { + doRaiseSetFromBytesInvalid(bytes, offset, length, fastResult); + } + */ + return true; + + } + + private static BigInteger doBigIntegerScaleDown(BigInteger unscaledValue, int scaleDown) { + BigInteger[] quotientAndRemainder = unscaledValue.divideAndRemainder(BigInteger.TEN.pow(scaleDown)); + BigInteger quotient = quotientAndRemainder[0]; + BigInteger round = quotientAndRemainder[1].divide(BigInteger.TEN.pow(scaleDown - 1)); + if (round.compareTo(BIG_INTEGER_FIVE) >= 0) { + quotient = quotient.add(BigInteger.ONE); + } + return quotient; + } + + /** + * Create a fast decimal from a BigDecimal. + * + * @param bigDecimal + * @param allowRounding + * @param result + * @return + */ + public static boolean fastSetFromBigDecimal( + BigDecimal bigDecimal, boolean allowRounding, FastHiveDecimal fastResult) { + + // We trim the trailing zero fraction digits so we don't cause unnecessary precision + // overflow later. + if (bigDecimal.signum() == 0) { + if (bigDecimal.scale() != 0) { + + // For some strange reason BigDecimal 0 can have a scale. We do not support that. + bigDecimal = BigDecimal.ZERO; + } + } else { + BigDecimal bigDecimalStripped = bigDecimal.stripTrailingZeros(); + int stripTrailingZerosScale = bigDecimalStripped.scale(); + // System.out.println("FAST_SET_FROM_BIG_DECIMAL bigDecimal " + bigDecimal); + // System.out.println("FAST_SET_FROM_BIG_DECIMAL bigDecimalStripped " + bigDecimalStripped); + // System.out.println("FAST_SET_FROM_BIG_DECIMAL stripTrailingZerosScale " + stripTrailingZerosScale); + if (stripTrailingZerosScale < 0) { + + // The trailing zeroes extend into the integer part -- we only want to eliminate the + // fractional zero digits. + + bigDecimal = bigDecimal.setScale(0); + } else { + + // Ok, use result with some or all fractional digits stripped. + + bigDecimal = bigDecimalStripped; + } + } + // System.out.println("FAST_SET_FROM_BIG_DECIMAL adjusted for zeroes/scale " + bigDecimal + " scale " + bigDecimal.scale()); + + BigInteger unscaledValue = bigDecimal.unscaledValue(); + // System.out.println("FAST_SET_FROM_BIG_DECIMAL unscaledValue " + unscaledValue + " length " + unscaledValue.toString().length()); + + final int scale = bigDecimal.scale(); + if (!allowRounding) { + if (scale < 0 || scale > FAST_MAX_SCALE) { + return false; + } + // The digits must fit without rounding. + if (!fastSetFromBigInteger(unscaledValue, fastResult)) { + return false; + } + if (fastResult.fastSignum != 0) { + fastResult.fastIntegerDigitCount = Math.max(0, fastResult.fastIntegerDigitCount - scale); + fastResult.fastScale = scale; + } + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + return true; + } + // This method will scale down and round to fit, if necessary. + if (!fastSetFromBigInteger(unscaledValue, scale, fastResult)) { + return false; + } + return true; + } + + /** + * Scan a String for a decimal number in UTF-8 characters. + * + * @param string + * @param trimBlanks + * @param result + * @return + */ + public static boolean fastSetFromString( + String string, boolean trimBlanks, FastHiveDecimal result) { + byte[] bytes = string.getBytes(); + return fastSetFromBytes(bytes, 0, bytes.length, trimBlanks, result); + } + + /** + * Creates a scale 0 fast decimal from an int. + * @param intValue + * @param fastResult + */ + public static void fastSetFromInt(int intValue, FastHiveDecimal fastResult) { + if (intValue == 0) { + // Zero special case. + return; + } + if (intValue > 0) { + fastResult.fastSignum = 1; + } else { + fastResult.fastSignum = -1; + intValue = Math.abs(intValue); + } + // 10 digit int is all in lowest 16 decimal digit longword. + // Since we are creating with scale 0, no fraction digits to zero trim. + fastResult.fast0 = intValue & 0xFFFFFFFFL; + fastResult.fastIntegerDigitCount = + fastLongWordPrecision(fastResult.fast0); + } + + /** + * Creates a scale 0 fast decimal from a long. + * + * @param longValue + * @param fastResult + */ + public static void fastSetFromLong( + long longValue, FastHiveDecimal fastResult) { + if (longValue == 0) { + // Zero special case. + return; + } + // Handle minimum integer case that doesn't have abs(). + if (longValue == Long.MIN_VALUE) { + // Split -9,223,372,036,854,775,808 into 16 digit middle and lowest longwords by hand. + fastResult.fastSignum = -1; + fastResult.fast1 = 922L; + fastResult.fast0 = 3372036854775808L; + fastResult.fastIntegerDigitCount = 19; + } else { + if (longValue > 0) { + fastResult.fastSignum = 1; + } else { + fastResult.fastSignum = -1; + longValue = Math.abs(longValue); + } + // Split into 16 digit middle and lowest longwords remainder / division. + fastResult.fast1 = longValue / MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast0 = longValue % MULTIPLER_LONGWORD_DECIMAL; + if (fastResult.fast1 != 0) { + fastResult.fastIntegerDigitCount = + LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast1); + } else { + fastResult.fastIntegerDigitCount = + fastLongWordPrecision(fastResult.fast0); + } + } + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + return; + } + + /** + * Creates a fast decimal from a long with a specified scale. + * + * @param longValue + * @param scale + * @param fastResult + */ + public static boolean fastSetFromLongAndScale( + long longValue, int scale, FastHiveDecimal fastResult) { + + if (scale < 0 || scale > FAST_MAX_SCALE) { + return false; + } + + fastSetFromLong(longValue, fastResult); + if (scale == 0) { + return true; + } + + if (!fastScaleByPowerOfTen( + fastResult, + -scale, + fastResult)) { + return false; + } + return true; + } + + /** + * Creates fast decimal from a float. + * + * @param floatValue + * @param result + */ + public static boolean fastSetFromFloat( + float floatValue, FastHiveDecimal fastResult) { + + String floatString = Float.toString(floatValue); + return fastSetFromString(floatString, false, fastResult); + + } + + /** + * Creates fast decimal from a double. + * + * @param doubleValue + * @param fastResult + */ + public static boolean fastSetFromDouble( + double doubleValue, FastHiveDecimal fastResult) { + + String doubleString = Double.toString(doubleValue); + return fastSetFromString(doubleString, false, fastResult); + + } + + /** + * Creates a fast decimal from a BigInteger with scale 0. + * + * For efficiency, we assume that fastResult is fastReset. This method does not set the + * fastScale field. + * + * @param unscaledValue + * @param fastResult Receives the decimal value. We assume fastResult was reset beforehand. + * @return Return true if the BigInteger value fit within FAST_MAX_PRECISION. Otherwise, + * false for overflow. + */ + public static boolean fastSetFromBigInteger( + BigInteger bigInteger, FastHiveDecimal fastResult) { + + final int signum = bigInteger.signum(); + if (signum == 0) { + // Zero special case. + return true; + } + fastResult.fastSignum = signum; + if (signum == -1) { + bigInteger = bigInteger.negate(); + } + if (bigInteger.compareTo(BIG_INTEGER_LONGWORD_MULTIPLIER) < 0) { + + // Fits in one longword. + fastResult.fast0 = bigInteger.longValue(); + if (fastResult.fast0 == 0) { + fastResult.fastSignum = 0; + } else { + fastResult.fastIntegerDigitCount = fastLongWordPrecision(fastResult.fast0); + } + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException("Fits in one longword"); + } + */ + return true; + } + BigInteger[] quotientAndRemainder = + bigInteger.divideAndRemainder(BIG_INTEGER_LONGWORD_MULTIPLIER); + fastResult.fast0 = quotientAndRemainder[1].longValue(); + BigInteger quotient = quotientAndRemainder[0]; + if (quotient.compareTo(BIG_INTEGER_LONGWORD_MULTIPLIER) < 0) { + + // Fits in two longwords. + fastResult.fast1 = quotient.longValue(); + if (fastResult.fast0 == 0 && fastResult.fast1 == 0) { + fastResult.fastSignum = 0; + } else { + fastResult.fastIntegerDigitCount = + LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast1); + } + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException("Fits in two longwords"); + } + */ + return true; + } + + // Uses all 3 decimal longs. + quotientAndRemainder = + quotient.divideAndRemainder(BIG_INTEGER_LONGWORD_MULTIPLIER); + fastResult.fast1 = quotientAndRemainder[1].longValue(); + quotient = quotientAndRemainder[0]; + if (quotient.compareTo(BIG_INTEGER_HIGHWORD_MULTIPLIER) >= 0) { + // Overflow. + return false; + } + fastResult.fast2 = quotient.longValue(); + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + } else { + fastResult.fastIntegerDigitCount = + TWO_X_LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(fastResult.fast2); + } + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException("Uses all 3 decimal longs"); + } + */ + return true; + } + + /** + * + * @param bigInteger + * @param scale + * @param fastResult + * @return + */ + public static boolean fastSetFromBigInteger( + BigInteger bigInteger, int scale, FastHiveDecimal fastResult) { + + if (scale < 0) { + + // Multiply by 10^(-scale) to normalize. We do not use negative scale in our representation. + // + // Example: + // 4.172529E+20 has a negative scale -20 since scale is number of digits below the dot. + // 417252900000000000000 normalized as scale 0. + // + bigInteger = bigInteger.multiply(BIG_INTEGER_TEN.pow(-scale)); + scale = 0; + } + + int signum = bigInteger.signum(); + if (signum == 0) { + // Zero. + return true; + } else if (signum == -1) { + // Normalize to positive. + bigInteger = bigInteger.negate(); + } + + // A slow way to get the number of decimal digits. + int precision = bigInteger.toString().length(); + + // System.out.println("FAST_SET_FROM_BIG_INTEGER adjusted bigInteger " + bigInteger + " precision " + precision); + + int integerDigitCount = precision - scale; + // System.out.println("FAST_SET_FROM_BIG_INTEGER integerDigitCount " + integerDigitCount + " scale " + scale); + int maxScale; + if (integerDigitCount >= 0) { + if (integerDigitCount > FAST_MAX_PRECISION) { + return false; + } + maxScale = FAST_MAX_SCALE - integerDigitCount; + } else { + maxScale = FAST_MAX_SCALE; + } + // System.out.println("FAST_SET_FROM_BIG_INTEGER maxScale " + maxScale); + + if (scale > maxScale) { + + // A larger scale is ok -- we will knock off lower digits and round. + + final int trimAwayCount = scale - maxScale; + // System.out.println("FAST_SET_FROM_BIG_INTEGER trimAwayCount " + trimAwayCount); + if (trimAwayCount > 1) { + // First, throw away digits below round digit. + BigInteger bigIntegerThrowAwayBelowRoundDigitDivisor = BIG_INTEGER_TEN.pow(trimAwayCount - 1); + bigInteger = bigInteger.divide(bigIntegerThrowAwayBelowRoundDigitDivisor); + } + // System.out.println("FAST_SET_FROM_BIG_INTEGER with round digit bigInteger " + bigInteger + " length " + bigInteger.toString().length()); + + BigInteger[] quotientAndRemainder = bigInteger.divideAndRemainder(BIG_INTEGER_TEN); + // System.out.println("FAST_SET_FROM_BIG_INTEGER quotientAndRemainder " + Arrays.toString(quotientAndRemainder)); + + BigInteger quotient = quotientAndRemainder[0]; + if (quotientAndRemainder[1].intValue() >= 5) { + if (quotient.equals(BIG_INTEGER_MAX_DECIMAL)) { + + // 38 9's digits. + // System.out.println("FAST_SET_FROM_BIG_INTEGER quotient is BIG_INTEGER_MAX_DECIMAL"); + + if (maxScale == 0) { + // No room above for rounding. + return false; + } + + // System.out.println("FAST_SET_FROM_BIG_INTEGER reached here... scale " + scale + " maxScale " + maxScale); + // Rounding results in 10^N. + bigInteger = BIG_INTEGER_TEN.pow(integerDigitCount); + maxScale = 0; + } else { + + // Round up. + bigInteger = quotient.add(BigInteger.ONE); + } + } else { + + // No rounding. + bigInteger = quotient; + } + scale = maxScale; + } + if (!fastSetFromBigInteger(bigInteger, fastResult)) { + return false; + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + } else { + fastResult.fastSignum = signum; + fastResult.fastIntegerDigitCount = Math.max(0, fastResult.fastIntegerDigitCount - scale); + fastResult.fastScale = scale; + + final int trailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, scale); + if (trailingZeroCount > 0) { + doFastScaleDown( + fastResult, + trailingZeroCount, + fastResult); + fastResult.fastScale -= trailingZeroCount; + } + } + + return true; + } + + //************************************************************************************************ + // Take Integer or Fractional Portion. + + /** + * Creates fast decimal from the fraction portion of a fast decimal. + * + * @param result + */ + public static void fastFractionPortion( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + FastHiveDecimal fastResult) { + + if (fastSignum == 0 || fastScale == 0) { + fastResult.fastReset(); + return; + } + + // Clear integer portion; keep fraction. + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + if (fastScale < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + final long clearFactor = powerOfTenTable[fastScale]; + + result0 = fast0 % clearFactor; + result1 = 0; + result2 = 0; + + } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; + + final long clearFactor = powerOfTenTable[adjustedScaleDown]; + + result0 = fast0; + result1 = fast1 % clearFactor; + result2 = 0; + + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; + + final long clearFactor = powerOfTenTable[adjustedScaleDown]; + + result0 = fast0; + result1 = fast1; + result2 = fast2 % clearFactor; + + } + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastReset(); + } else { + fastResult.fastSet(fastSignum, result0, result1, result2, /* fastIntegerDigitCount */ 0, fastScale); + } + } + + /** + * Creates fast decimal from the integer portion. + * + * @param result + */ + public static void fastIntegerPortion( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + FastHiveDecimal fastResult) { + + if (fastSignum == 0) { + fastResult.fastReset(); + return; + } + if (fastScale == 0) { + fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + // Scale down no rounding to clear fraction. + fastResult.fastSignum = fastSignum; + doFastScaleDown( + fast0, fast1, fast2, + fastScale, + fastResult); + fastResult.fastIntegerDigitCount = fastIntegerDigitCount; + fastResult.fastScale = 0; + } + + //************************************************************************************************ + // Binary to Decimal Conversion. + + /** + * Convert binary words of N bits each to a fast decimal (scale 0). + * + * @param lowerWord + * @param middleWord + * @param highWord + * @param middleWordMultiplier + * @param highWordMultiplier + * @param fastResult + * @return + */ + public static boolean doBinaryToDecimalConversion( + long lowerWord, long middleWord, long highWord, + FastHiveDecimal middleWordMultiplier, + FastHiveDecimal highWordMultiplier, + FastHiveDecimal fastResult) { + + /* + * Challenge: How to do the math to get this raw binary back to our decimal form. + */ + long result0 = + lowerWord % MULTIPLER_LONGWORD_DECIMAL; + long result1 = + lowerWord / MULTIPLER_LONGWORD_DECIMAL; + long result2 = 0; + + if (middleWord != 0 || highWord != 0) { + + if (highWord == 0) { + + // Form result from lower and middle words. + + if (!fastMultiply5x5HalfWords( + middleWord % MULTIPLER_LONGWORD_DECIMAL, + middleWord / MULTIPLER_LONGWORD_DECIMAL, + 0, + middleWordMultiplier.fast0, middleWordMultiplier.fast1, middleWordMultiplier.fast2, + fastResult)) { + return false; + } + + final long calc0 = + result0 + + fastResult.fast0; + result0 = + calc0 % MULTIPLER_LONGWORD_DECIMAL; + final long calc1 = + calc0 / MULTIPLER_LONGWORD_DECIMAL + + result1 + + fastResult.fast1; + result1 = + calc1 % MULTIPLER_LONGWORD_DECIMAL; + result2 = + calc1 / MULTIPLER_LONGWORD_DECIMAL + + fastResult.fast2; + + } else if (middleWord == 0) { + + // Form result from lower and high words. + + if (!fastMultiply5x5HalfWords( + highWord % MULTIPLER_LONGWORD_DECIMAL, + highWord / MULTIPLER_LONGWORD_DECIMAL, + 0, + highWordMultiplier.fast0, highWordMultiplier.fast1, highWordMultiplier.fast2, + fastResult)) { + return false; + } + + final long calc0 = + result0 + + fastResult.fast0; + result0 = + calc0 % MULTIPLER_LONGWORD_DECIMAL; + final long calc1 = + calc0 / MULTIPLER_LONGWORD_DECIMAL + + result1 + + fastResult.fast1; + result1 = + calc1 % MULTIPLER_LONGWORD_DECIMAL; + result2 = + calc1 / MULTIPLER_LONGWORD_DECIMAL + + fastResult.fast2; + + } else { + + // Form result from lower, middle, and middle words. + + if (!fastMultiply5x5HalfWords( + middleWord % MULTIPLER_LONGWORD_DECIMAL, + middleWord / MULTIPLER_LONGWORD_DECIMAL, + 0, + middleWordMultiplier.fast0, middleWordMultiplier.fast1, middleWordMultiplier.fast2, + fastResult)) { + return false; + } + + long middleResult0 = fastResult.fast0; + long middleResult1 = fastResult.fast1; + long middleResult2 = fastResult.fast2; + + if (!fastMultiply5x5HalfWords( + highWord % MULTIPLER_LONGWORD_DECIMAL, + highWord / MULTIPLER_LONGWORD_DECIMAL, + 0, + highWordMultiplier.fast0, highWordMultiplier.fast1, highWordMultiplier.fast2, + fastResult)) { + return false; + } + + long calc0 = + result0 + + middleResult0 + + fastResult.fast0; + result0 = + calc0 % MULTIPLER_LONGWORD_DECIMAL; + long calc1 = + calc0 / MULTIPLER_LONGWORD_DECIMAL + + result1 + + middleResult1 + + fastResult.fast1; + result1 = + calc1 % MULTIPLER_LONGWORD_DECIMAL; + result2 = + calc1 / MULTIPLER_LONGWORD_DECIMAL + + middleResult2 + + fastResult.fast2; + } + } + + // Let caller set negative sign if necessary. + if (result2 != 0) { + fastResult.fastIntegerDigitCount = TWO_X_LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(result2); + fastResult.fastSignum = 1; + } else if (result1 != 0) { + fastResult.fastIntegerDigitCount = LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(result1); + fastResult.fastSignum = 1; + } else if (result0 != 0) { + fastResult.fastIntegerDigitCount = fastHighWordPrecision(result0); + fastResult.fastSignum = 1; + } else { + fastResult.fastIntegerDigitCount = 0; + fastResult.fastSignum = 0; + } + + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + + return true; + } + + //************************************************************************************************ + // Decimal to Binary Conversion. + + /** + * Do binary division/remainder on a fast decimal to produce 1 binary word (remainder) and a + * decimal quotient for the higher part. + * + * @param dividendFast0 + * @param dividendFast1 + * @param dividendFast2 + * @param fastInverseConst + * @param quotientIntegerWordNum + * @param quotientIntegerDigitNum + * @param fastMultiplierConst + * @param scratchLongs + * @return + */ + public static boolean doDecimalToBinaryDivisionRemainder( + long dividendFast0, long dividendFast1, long dividendFast2, + FastHiveDecimal fastInverseConst, + int quotientIntegerWordNum, + int quotientIntegerDigitNum, + FastHiveDecimal fastMultiplierConst, + long[] scratchLongs) { + + // Multiply by inverse (2^-N) to do the 2^N division. + if (!fastMultiply5x6HalfWords( + dividendFast0, dividendFast1, dividendFast2, + fastInverseConst.fast0, fastInverseConst.fast1, fastInverseConst.fast2, + scratchLongs)) { + // Overflow. + return false; + } + + final long divideFactor = powerOfTenTable[quotientIntegerDigitNum]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - quotientIntegerDigitNum]; + + // Extract the integer portion to get the quotient. + long quotientFast0 = + scratchLongs[quotientIntegerWordNum] / divideFactor + + ((scratchLongs[quotientIntegerWordNum + 1] % divideFactor) * multiplyFactor); + long quotientFast1 = + scratchLongs[quotientIntegerWordNum + 1] / divideFactor + + ((scratchLongs[quotientIntegerWordNum + 2] % divideFactor) * multiplyFactor); + long quotientFast2 = + scratchLongs[quotientIntegerWordNum + 2] / divideFactor; + + // Multiply the integer quotient back out so we can subtract it from the original to get + // the remainder. + if (!fastMultiply5x6HalfWords( + quotientFast0, quotientFast1, quotientFast2, + fastMultiplierConst.fast0, fastMultiplierConst.fast1, fastMultiplierConst.fast2, + scratchLongs)) { + return false; + } + + long quotientMultiplied0 = scratchLongs[0]; + long quotientMultiplied1 = scratchLongs[1]; + long quotientMultiplied2 = scratchLongs[2]; + + if (!doSubtractSameScaleNoUnderflow( + dividendFast0, dividendFast1, dividendFast2, + quotientMultiplied0, quotientMultiplied1, quotientMultiplied2, + scratchLongs)) { + // Underflow. + return false; + } + + long remainderBinaryWord = + scratchLongs[1] * MULTIPLER_LONGWORD_DECIMAL + + scratchLongs[0]; + + // Pack the output into the scratch longs. + scratchLongs[0] = quotientFast0; + scratchLongs[1] = quotientFast1; + scratchLongs[2] = quotientFast2; + + scratchLongs[3] = remainderBinaryWord; + + return true; + } + + /** + * Convert a fast decimal into binary words of N bits each. + * + * @param fast0 + * @param fast1 + * @param fast2 + * @param fastIntegerDigitCount + * @param fastScale + * @param fastInverseConst + * @param quotientIntegerWordNum + * @param quotientIntegerDigitNum + * @param fastMultiplierConst + * @param scratchLongs + * @return + */ + private static boolean doDecimalToBinaryConversion( + long fast0, long fast1, long fast2, + FastHiveDecimal fastInverseConst, + int quotientIntegerWordNum, + int quotientIntegerDigitNum, + FastHiveDecimal fastMultiplierConst, + long[] scratchLongs) { + + long lowerBinaryWord; + long middleBinaryWord = 0; + long highBinaryWord = 0; + + if (fastCompareTo( + 1, + fast0, fast1, fast2, 0, + 1, + fastMultiplierConst.fast0, fastMultiplierConst.fast1, fastMultiplierConst.fast2, 0) < 0) { + + // Whole decimal fits in one binary word. + + lowerBinaryWord = + fast1 * MULTIPLER_LONGWORD_DECIMAL + + fast0; + + } else { + + // Do division/remainder to get lower binary word; quotient will either be middle decimal + // or be both high and middle decimal that requires another division/remainder. + + if (!doDecimalToBinaryDivisionRemainder( + fast0, fast1, fast2, + fastInverseConst, + quotientIntegerWordNum, + quotientIntegerDigitNum, + fastMultiplierConst, + scratchLongs)) { + // Overflow. + return false; + } + + // Unpack the output. + long quotientFast0 = scratchLongs[0]; + long quotientFast1 = scratchLongs[1]; + long quotientFast2 = scratchLongs[2]; + + lowerBinaryWord = scratchLongs[3]; + + if (fastCompareTo( + 1, + quotientFast0, quotientFast1, quotientFast2, 0, + 1, + fastMultiplierConst.fast0, fastMultiplierConst.fast1, fastMultiplierConst.fast2, 0) < 0) { + + // Whole decimal fits in two binary words. + + middleBinaryWord = + quotientFast1 * MULTIPLER_LONGWORD_DECIMAL + + quotientFast0; + + } else { + if (!doDecimalToBinaryDivisionRemainder( + quotientFast0, quotientFast1, quotientFast2, + fastInverseConst, + quotientIntegerWordNum, + quotientIntegerDigitNum, + fastMultiplierConst, + scratchLongs)) { + // Overflow. + return false; + } + + highBinaryWord = + scratchLongs[1] * MULTIPLER_LONGWORD_DECIMAL + + scratchLongs[0]; + + middleBinaryWord = scratchLongs[3]; + + } + } + + scratchLongs[0] = lowerBinaryWord; + scratchLongs[1] = middleBinaryWord; + scratchLongs[2] = highBinaryWord; + + return true; + } + + //************************************************************************************************ + // Emulate SerializationUtils Deserialization used by ORC. + + /* + * fastSerializationUtilsRead middle word 2^62 + * + * Convert middleWord into decimal number -- At most 19 digits or 2 decimal words. + * Multiply by 2^62 -- 19 digits or 2 decimal words. + * + * 2^62 = + * 4611686018427387904 or + * 4,611,686,018,427,387,904 or + * 461,1686018427387904 (16 digit comma'd) + */ + private static FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_62 = + new FastHiveDecimal(1, 1686018427387904L, 461L, 0, 19, 0); + + /* + * fastSerializationUtilsRead middle word 2^62 + * + * Convert middleWord into decimal number -- At most 19 digits or 2 decimal words. + * Multiply by 2^63 -- 19 digits or 2 decimal words. + * + * 2^63 = + * 9223372036854775808 (Long.MAX_VALUE) or + * 9,223,372,036,854,775,808 or + * 922,3372036854775808 (16 digit comma'd) + */ + private static FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_63 = + new FastHiveDecimal(1, 3372036854775808L, 922L, 0, 19, 0); + + /* + * fastSerializationUtilsRead high word multiplier: + * + * Convert highWord63 into decimal number -- At most 19 digits or 2 decimal words. + * Multiply by 2^(62 + 63) -- 38 digits or 3 decimal words. + * + * (2^62)*(2^63) = + * 42535295865117307932921825928971026432 or + * (12345678901234567890123456789012345678) + * ( 1 2 3 ) + * 42,535,295,865,117,307,932,921,825,928,971,026,432 or + * 425352,9586511730793292,1825928971026432 (16 digit comma'd) + */ + private static FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_125 = + new FastHiveDecimal(1, 1825928971026432L, 9586511730793292L, 425352L, 38, 0); + + /* + * Multiply by 1/2^63 = 1.08420217248550443400745280086994171142578125e-19 to divide by 2^63. + * As 16 digit comma'd 1084202172485,5044340074528008,6994171142578125 + * + * Scale down: 63 = 44 fraction digits + 19 (negative exponent or number of zeros after dot). + * + * 3*16 (48) + 15 --> 63 down shift. + */ + private static FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_63_INVERSE = + new FastHiveDecimal(1, 6994171142578125L, 5044340074528008L, 1084202172485L, 45, 0); + + private static final int SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_WORD_NUM = 3; + private static final int SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_DIGIT_NUM = 15; + + /** + * See HiveDecimal's comments for this. + * + * @param inputStream + * @param scale + * @param scratchLongs + * @param fastResult + * @return + */ + public static boolean fastSerializationUtilsRead(InputStream inputStream, int scale, + byte[] scratchBytes, + FastHiveDecimal fastResult) throws IOException, EOFException { + + // Following a suggestion from Gopal, quickly read in the bytes from the stream. + // CONSIDER: Have ORC read the whole input stream into a big byte array with one call to + // the read(byte[] b, int off, int len) method and then let this method read from the big + // byte array. + int readCount = 0; + int input; + do { + input = inputStream.read(); + if (input == -1) { + throw new EOFException("Reading BigInteger past EOF from " + inputStream); + } + scratchBytes[readCount++] = (byte) input; + } while (input >= 0x80); + + long lowerWord63 = 0; + long middleWord63 = 0; + long highWord63 = 0; + + long work = 0; + int offset = 0; + int readIndex = 0; + long b; + do { + b = scratchBytes[readIndex++]; + work |= (0x7f & b) << (offset % 63); + offset += 7; + // if we've read 63 bits, roll them into the result + if (offset == 63) { + lowerWord63 = work; + work = 0; + } else if (offset % 63 == 0) { + if (offset == 126) { + middleWord63 = work; + } else if (offset == 189) { + highWord63 = work; + } else { + throw new EOFException("Reading more than 3 words of BigInteger"); + } + work = 0; + } + } while (readIndex < readCount); + + if (work != 0) { + if (offset < 63) { + lowerWord63 = work; + } else if (offset < 126) { + middleWord63 = work; + } else if (offset < 189) { + highWord63 =work; + } else { + throw new EOFException("Reading more than 3 words of BigInteger"); + } + } + + // Grab sign bit and shift away. + boolean isNegative = ((lowerWord63 & 0x1) != 0); + lowerWord63 >>= 1; + + if (!doBinaryToDecimalConversion( + lowerWord63, middleWord63, highWord63, + FAST_HIVE_DECIMAL_TWO_POWER_62, + FAST_HIVE_DECIMAL_TWO_POWER_125, // 2^(62 + 63) + fastResult)) { + return false; + } + + if (isNegative) { + if (!doAddSameScaleSameSign( + /* resultSignum */ 1, + fastResult.fast0, fastResult.fast1, fastResult.fast2, + 1, 0, 0, + fastResult)) { + return false; + } + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + } else { + fastResult.fastSignum = (isNegative ? -1 : 1); + final int rawPrecision = fastRawPrecision(fastResult); + fastResult.fastIntegerDigitCount = Math.max(0, rawPrecision - scale); + fastResult.fastScale = scale; + + /* + * Just in case we deserialize a decimal with trailing zeroes... + */ + final int resultTrailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (resultTrailingZeroCount > 0) { + doFastScaleDown( + fastResult, + resultTrailingZeroCount, + fastResult); + + fastResult.fastScale -= resultTrailingZeroCount; + } + } + + return true; + } + + //************************************************************************************************ + // Emulate SerializationUtils Serialization used by ORC. + + /** + * Write the value of this decimal just like SerializationUtils.writeBigInteger. It header + * comments are: + * + * Write the arbitrarily sized signed BigInteger in vint format. + * + * Signed integers are encoded using the low bit as the sign bit using zigzag + * encoding. + * + * Each byte uses the low 7 bits for data and the high bit for stop/continue. + * + * Bytes are stored LSB first. + * + * NOTE: + * SerializationUtils.writeBigInteger sometimes pads the result with extra zeroes due to + * BigInteger.bitLength -- we do not emulate that. SerializationUtils.readBigInteger will + * produce the same result for both. + * + * @param outputStream + * @param fastSignum + * @param fast0 + * @param fast1 + * @param fast2 + * @param scratchLongs + */ + public static boolean fastSerializationUtilsWrite(OutputStream outputStream, + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + long[] scratchLongs) + throws IOException { + + boolean isNegative = (fastSignum == -1); + + /* + * The sign is encoded as the least significant bit. + * + * We need to adjust our decimal before conversion to binary. + * + * Positive: + * Multiply by 2. + * + * Negative: + * Logic in SerializationUtils.writeBigInteger does a negate on the BigInteger. We + * do not have to since FastHiveDecimal stores the numbers unsigned in fast0, fast1, + * and fast2. We do need to subtract one though. + * + * And then multiply by 2 and add in the 1 sign bit. + * + * CONSIDER: This could be combined. + */ + long adjust0; + long adjust1; + long adjust2; + + if (isNegative) { + + // Subtract 1. + long r0 = fast0 - 1; + long r1; + if (r0 < 0) { + adjust0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = fast1 - 1; + } else { + adjust0 = r0; + r1 = fast1; + } + if (r1 < 0) { + adjust1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + adjust2 = fast2 - 1; + } else { + adjust1 = r1; + adjust2 = fast2; + } + if (adjust2 < 0) { + return false; + } + + // Now multiply by 2 and add 1 sign bit. + r0 = adjust0 * 2 + 1; + adjust0 = + r0 % MULTIPLER_LONGWORD_DECIMAL; + r1 = + adjust1 * 2 + + r0 / MULTIPLER_LONGWORD_DECIMAL; + adjust1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + adjust2 = + adjust2 * 2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + + } else { + + // Multiply by 2 to make room for 0 sign bit. + long r0 = fast0 * 2; + adjust0 = + r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fast1 * 2 + + r0 / MULTIPLER_LONGWORD_DECIMAL; + adjust1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + adjust2 = + fast2 * 2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + + } + + + /* + * Use common conversion method we share with fastBigIntegerBytes. + */ + if (!doDecimalToBinaryConversion( + adjust0, adjust1, adjust2, + FAST_HIVE_DECIMAL_TWO_POWER_63_INVERSE, + SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_WORD_NUM, + SERIALIZATION_UTILS_WRITE_QUOTIENT_INTEGER_DIGIT_NUM, + FAST_HIVE_DECIMAL_TWO_POWER_63, + scratchLongs)) { + // Overflow. + return false; + } + + long lowerWord63 = scratchLongs[0]; + long middleWord63 = scratchLongs[1]; + long highWord63 = scratchLongs[2]; + + int wordCount; + if (highWord63 != 0) { + wordCount = 3; + } else if (middleWord63 != 0) { + wordCount = 2; + } else { + wordCount = 1; + } + + // Write out the first 63 bits worth of data. + long lowBits = lowerWord63; + for(int i=0; i < 9; ++i) { + // If this is the last byte, leave the high bit off + if (wordCount == 1 && (lowBits & ~0x7f) == 0) { + outputStream.write((byte) lowBits); + return true; + } else { + outputStream.write((byte) (0x80 | (lowBits & 0x7f))); + lowBits >>>= 7; + } + } + if (wordCount <= 1) { + throw new RuntimeException("Expecting write word count > 1"); + } + + lowBits = middleWord63; + for(int i=0; i < 9; ++i) { + // If this is the last byte, leave the high bit off + if (wordCount == 2 && (lowBits & ~0x7f) == 0) { + outputStream.write((byte) lowBits); + return true; + } else { + outputStream.write((byte) (0x80 | (lowBits & 0x7f))); + lowBits >>>= 7; + } + } + + lowBits = highWord63; + for(int i=0; i < 9; ++i) { + // If this is the last byte, leave the high bit off + if ((lowBits & ~0x7f) == 0) { + outputStream.write((byte) lowBits); + return true; + } else { + outputStream.write((byte) (0x80 | (lowBits & 0x7f))); + lowBits >>>= 7; + } + } + + // Should not get here. + throw new RuntimeException("Unexpected"); + } + + //************************************************************************************************ + // Emulate BigInteger deserialization used by LazyBinary and others. + + /* + * fastSetFromBigIntegerBytes middle word multiplier: + * + * Convert middleWord63 into decimal number -- At most 19 digits or 2 decimal words. + * Multiply by 2^56 -- 17 digits or 2 decimal words. + * + * 2^56 = + * 72057594037927936 or + * 72,057,594,037,927,936 or + * 7,2057594037927936 (16 digit comma'd) + */ + private static FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_56 = + new FastHiveDecimal(1, 2057594037927936L, 7L, 0, 17, 0); + + /* + * fastSetFromBigIntegerBytes high word multiplier: + * + * Convert highWord63 into decimal number -- At most 19 digits or 2 decimal words. + * Multiply by 2^(54 + 54) -- 34 digits or 3 decimal words. + * + * (2^56)*(2^56) = + * 5192296858534827628530496329220096 or + * (1234567890123456789012345678901234) + * ( 1 2 3 ) + * 5,192,296,858,534,827,628,530,496,329,220,096 or + * 51,9229685853482762,8530496329220096 (16 digit comma'd) + */ + private static FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_112 = + new FastHiveDecimal(1, 8530496329220096L, 9229685853482762L, 51L, 34, 0); + + // Multiply by 1/2^56 or 1.387778780781445675529539585113525390625e-17 to divide by 2^56. + // As 16 digit comma'd 13877787,8078144567552953,9585113525390625 + // + // Scale down: 56 = 39 fraction digits + 17 (negative exponent or number of zeros after dot). + // + // 3*16 (48) + 8 --> 56 down shift. + // + private static FastHiveDecimal FAST_HIVE_DECIMAL_TWO_POWER_56_INVERSE = + new FastHiveDecimal(1, 9585113525390625L, 8078144567552953L, 13877787L, 40, 0); + + private static final int BIG_INTEGER_BYTES_QUOTIENT_INTEGER_WORD_NUM = 3; + private static final int BIG_INTEGER_BYTES_QUOTIENT_INTEGER_DIGIT_NUM = 8; + + private static int INITIAL_SHIFT = 48; // 56 bits minus 1 byte. + private static long LONG_56_BIT_MASK = 0xFFFFFFFFFFFFFFL; + private static long LONG_TWO_TO_56_POWER = LONG_56_BIT_MASK + 1L; + private static long LONG_BYTE_MASK = 0xFFL; + private static long LONG_BYTE_HIGH_BIT_MASK = 0x80L; + private static byte BYTE_ALL_BITS = (byte) 0xFF; + + public static boolean fastSetFromBigIntegerBytesAndScale( + byte[] bytes, int offset, int length, int scale, + FastHiveDecimal fastResult) { + + System.out.println("fastSetFromBigIntegerBytesAndScale scale " + scale); + + final int bytesLength = bytes.length; + + if (offset < 0 || offset >= bytesLength) { + return false; + } + final int end = offset + length; + if (end <= offset || end > bytesLength) { + return false; + } + + final int startOffset = offset; + + // Roughly based on BigInteger code. + + boolean isNegative = (bytes[offset] < 0); + if (isNegative) { + + // Find first non-sign (0xff) byte of input. + while (offset < end) { + if (bytes[offset] != -1) { + break; + } + offset++; + } + if (offset > end) { + return false; + } + } else { + + // Strip leading zeroes -- although there shouldn't be any for a decimal. + + while (offset < end && bytes[offset] == 0) { + offset++; + } + if (offset >= end) { + // Zero. + return true; + } + } + + long lowerWord56 = 0; + long middleWord56 = 0; + long highWord56 = 0; + + int reverseIndex = end; + + long work; + int shift; + + final int lowestCount = Math.min(reverseIndex - offset, 7); + shift = 0; + for (int i = 0; i < lowestCount; i++) { + work = bytes[--reverseIndex] & 0xFF; + lowerWord56 |= work << shift; + shift += 8; + } + + if (reverseIndex <= offset) { + if (isNegative) { + lowerWord56 = ~lowerWord56 & ((1L << shift) - 1); + } + } else { + + // Go on to middle word. + + final int middleCount = Math.min(reverseIndex - offset, 7); + shift = 0; + for (int i = 0; i < middleCount; i++) { + work = bytes[--reverseIndex] & 0xFF; + middleWord56 |= work << shift; + shift += 8; + } + if (reverseIndex <= offset) { + if (isNegative) { + lowerWord56 = ~lowerWord56 & LONG_56_BIT_MASK; + middleWord56 = ~middleWord56 & ((1L << shift) - 1); + } + } else { + + // Go on to high word. + + final int highCount = Math.min(reverseIndex - offset, 7); + shift = 0; + for (int i = 0; i < highCount; i++) { + work = bytes[--reverseIndex] & 0xFF; + highWord56 |= work << shift; + shift += 8; + } + if (isNegative) { + lowerWord56 = ~lowerWord56 & LONG_56_BIT_MASK; + middleWord56 = ~middleWord56 & LONG_56_BIT_MASK; + highWord56 = ~highWord56 & ((1L << shift) - 1); + } + } + } + + if (!doBinaryToDecimalConversion( + lowerWord56, middleWord56, highWord56, + FAST_HIVE_DECIMAL_TWO_POWER_56, + FAST_HIVE_DECIMAL_TWO_POWER_112, // 2^(56 + 56) + fastResult)) { + // Overflow. Use slower alternate. + return doAlternateSetFromBigIntegerBytesAndScale( + bytes, startOffset, length, scale, + fastResult); + } + + // System.out.println("fastSetFromBigIntegerBytesAndScale fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2); + if (isNegative) { + if (!doAddSameScaleSameSign( + /* resultSignum */ 1, + fastResult.fast0, fastResult.fast1, fastResult.fast2, + 1, 0, 0, + fastResult)) { + // Overflow. Use slower alternate. + return doAlternateSetFromBigIntegerBytesAndScale( + bytes, startOffset, length, scale, + fastResult); + } + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + } else { + fastResult.fastSignum = (isNegative ? -1 : 1); + fastResult.fastScale = scale; + final int rawPrecision = fastRawPrecision(fastResult); + fastResult.fastIntegerDigitCount = Math.max(0, rawPrecision - scale); + + /* + * Just in case we deserialize a decimal with trailing zeroes... + */ + final int resultTrailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (resultTrailingZeroCount > 0) { + doFastScaleDown( + fastResult, + resultTrailingZeroCount, + fastResult); + + fastResult.fastScale -= resultTrailingZeroCount; + } + } + + return true; + } + + private static boolean doAlternateSetFromBigIntegerBytesAndScale( + byte[] bytes, int offset, int length, int scale, + FastHiveDecimal fastResult) { + + byte[] byteArray = Arrays.copyOfRange(bytes, offset, offset + length); + // System.out.println("FAST_BIG_INTEGER_BYTES_SCALED byteArray " + displayBytes(byteArray, 0, byteArray.length)); + + BigInteger bigInteger = new BigInteger(byteArray); + // System.out.println("doAlternateSetFromBigIntegerBytesAndScale bigInteger " + bigInteger); + BigDecimal bigDecimal = new BigDecimal(bigInteger, scale); + // System.out.println("doAlternateSetFromBigIntegerBytesAndScale bigDecimal " + bigDecimal); + fastResult.fastReset(); + return fastSetFromBigDecimal(bigDecimal, true, fastResult); + } + + //************************************************************************************************ + // Emulate BigInteger serialization used by LazyBinary and others. + + public static int fastBigIntegerBytes( + final int fastSignum, long fast0, long fast1, long fast2, + long[] scratchLongs, byte[] buffer) { + + if (fastSignum == 0) { + buffer[0] = 0; + return 1; + } + + boolean isNegative = (fastSignum == -1); + + /* + * Use common conversion method we share with fastSerializationUtilsWrite. + */ + if (!doDecimalToBinaryConversion( + fast0, fast1, fast2, + FAST_HIVE_DECIMAL_TWO_POWER_56_INVERSE, + BIG_INTEGER_BYTES_QUOTIENT_INTEGER_WORD_NUM, + BIG_INTEGER_BYTES_QUOTIENT_INTEGER_DIGIT_NUM, + FAST_HIVE_DECIMAL_TWO_POWER_56, + scratchLongs)) { + // Overflow. + return 0; + } + + int byteIndex = 0; + + long word0 = scratchLongs[0]; + long word1 = scratchLongs[1]; + long word2 = scratchLongs[2]; + + if (!isNegative) { + + // Positive number. + + long longWork = 0; + + int shift = INITIAL_SHIFT; + + if (word2 != 0L) { + + // Skip leading zeroes in word2. + + while (true) { + longWork = ((word2 & (LONG_BYTE_MASK << shift)) >> shift); + if (longWork != 0) { + break; + } + if (shift == 0) { + throw new RuntimeException("Unexpected #1"); + } + shift -= Byte.SIZE; + } + + // Now that we have found real data, emit sign byte if necessary. + if ((longWork & LONG_BYTE_HIGH_BIT_MASK) != 0) { + // Add sign byte since high bit is on. + buffer[byteIndex++] = (byte) 0; + } + + // Emit the rest of word2 + while (true) { + buffer[byteIndex++] = (byte) longWork; + if (shift == 0) { + break; + } + shift -= Byte.SIZE; + longWork = ((word2 & (LONG_BYTE_MASK << shift)) >> shift); + } + + shift = INITIAL_SHIFT; + } + + if (byteIndex == 0 && word1 == 0L) { + + // Skip word1, also. + + } else { + + if (byteIndex == 0) { + + // Skip leading zeroes in word1. + + while (true) { + longWork = ((word1 & (LONG_BYTE_MASK << shift)) >> shift); + if (longWork != 0) { + break; + } + if (shift == 0) { + throw new RuntimeException("Unexpected #2"); + } + shift -= Byte.SIZE; + } + + // Now that we have found real data, emit sign byte if necessary. + if ((longWork & LONG_BYTE_HIGH_BIT_MASK) != 0) { + // Add sign byte since high bit is on. + buffer[byteIndex++] = (byte) 0; + } + + } else { + longWork = ((word1 & (LONG_BYTE_MASK << shift)) >> shift); + } + + // Emit the rest of word1 + + while (true) { + buffer[byteIndex++] = (byte) longWork; + if (shift == 0) { + break; + } + shift -= Byte.SIZE; + longWork = ((word1 & (LONG_BYTE_MASK << shift)) >> shift); + } + + shift = INITIAL_SHIFT; + } + + if (byteIndex == 0) { + + // Skip leading zeroes in word0. + + while (true) { + longWork = ((word0 & (LONG_BYTE_MASK << shift)) >> shift); + if (longWork != 0) { + break; + } + if (shift == 0) { + + // All zeroes -- we should have handled this earlier. + throw new RuntimeException("Unexpected #3"); + } + shift -= Byte.SIZE; + } + + // Now that we have found real data, emit sign byte if necessary. + if ((longWork & LONG_BYTE_HIGH_BIT_MASK) != 0) { + // Add sign byte since high bit is on. + buffer[byteIndex++] = (byte) 0; + } + + } else { + longWork = ((word0 & (LONG_BYTE_MASK << shift)) >> shift); + } + + // Emit the rest of word0. + while (true) { + buffer[byteIndex++] = (byte) longWork; + if (shift == 0) { + break; + } + shift -= Byte.SIZE; + longWork = (byte) ((word0 & (LONG_BYTE_MASK << shift)) >> shift); + } + + } else { + + // Negative number. + + // Subtract 1 for two's compliment adjustment. + word0--; + if (word0 < 0) { + word0 += LONG_TWO_TO_56_POWER; + word1--; + if (word1 < 0) { + word1 += LONG_TWO_TO_56_POWER; + word2--; + if (word2 < 0) { + // Underflow. + return 0; + } + } + } + + long longWork = 0; + + int shift = INITIAL_SHIFT; + + if (word2 != 0L) { + + // Skip leading zeroes in word2. + + while (true) { + longWork = ((word2 & (LONG_BYTE_MASK << shift)) >> shift); + if (longWork != 0) { + break; + } + if (shift == 0) { + throw new RuntimeException("Unexpected #1"); + } + shift -= Byte.SIZE; + } + + // Now that we have found real data, emit sign byte if necessary and do negative fixup. + + longWork = (~longWork & LONG_BYTE_MASK); + if (((longWork) & LONG_BYTE_HIGH_BIT_MASK) == 0) { + // Add sign byte since high bit is off. + buffer[byteIndex++] = BYTE_ALL_BITS; + } + + // Invert words. + word2 = ~word2; + word1 = ~word1; + word0 = ~word0; + + // Emit the rest of word2 + while (true) { + buffer[byteIndex++] = (byte) longWork; + if (shift == 0) { + break; + } + shift -= Byte.SIZE; + longWork = ((word2 & (LONG_BYTE_MASK << shift)) >> shift); + } + + shift = INITIAL_SHIFT; + } + + if (byteIndex == 0 && word1 == 0L) { + + // Skip word1, also. + + } else { + + if (byteIndex == 0) { + + // Skip leading zeroes in word1. + + while (true) { + longWork = ((word1 & (LONG_BYTE_MASK << shift)) >> shift); + if (longWork != 0) { + break; + } + if (shift == 0) { + throw new RuntimeException("Unexpected #2"); + } + shift -= Byte.SIZE; + } + + // Now that we have found real data, emit sign byte if necessary and do negative fixup. + + longWork = (~longWork & LONG_BYTE_MASK); + if ((longWork & LONG_BYTE_HIGH_BIT_MASK) == 0) { + // Add sign byte since high bit is off. + buffer[byteIndex++] = BYTE_ALL_BITS; + } + + // Invert words. + word1 = ~word1; + word0 = ~word0; + + } else { + longWork = ((word1 & (LONG_BYTE_MASK << shift)) >> shift); + } + + // Emit the rest of word1 + + while (true) { + buffer[byteIndex++] = (byte) longWork; + if (shift == 0) { + break; + } + shift -= Byte.SIZE; + longWork = ((word1 & (LONG_BYTE_MASK << shift)) >> shift); + } + + shift = INITIAL_SHIFT; + } + + if (byteIndex == 0) { + + // Skip leading zeroes in word0. + + while (true) { + longWork = ((word0 & (LONG_BYTE_MASK << shift)) >> shift); + if (longWork != 0) { + break; + } + if (shift == 0) { + + // All zeroes. + + // -1 special case. Unsigned magnitude 1 - two's compliment adjustment 1 = 0. + buffer[0] = BYTE_ALL_BITS; + return 1; + } + shift -= Byte.SIZE; + } + + // Now that we have found real data, emit sign byte if necessary and do negative fixup. + + longWork = (~longWork & LONG_BYTE_MASK); + if ((longWork & LONG_BYTE_HIGH_BIT_MASK) == 0) { + // Add sign byte since high bit is off. + buffer[byteIndex++] = BYTE_ALL_BITS; + } + + // Invert words. + word0 = ~word0; + + } else { + longWork = ((word0 & (LONG_BYTE_MASK << shift)) >> shift); + } + + // Emit the rest of word0. + while (true) { + buffer[byteIndex++] = (byte) longWork; + if (shift == 0) { + break; + } + shift -= Byte.SIZE; + longWork = (byte) ((word0 & (LONG_BYTE_MASK << shift)) >> shift); + } + } + + return byteIndex; + } + + public static int fastBigIntegerBytesScaled( + final int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int serializeScale, + long[] scratchLongs, byte[] buffer) { + + // System.out.println("FAST_BIG_INTEGER_BYTES_SCALED serializeScale " + serializeScale); + + // Normally, trailing fractional digits are removed. But to emulate the + // OldHiveDecimal setScale and OldHiveDecimalWritable internalStorage, we need to trailing zeroes + // here. + + + if (fastSignum == 0 || serializeScale == fastScale) { + return + fastBigIntegerBytes( + fastSignum, fast0, fast1, fast2, + scratchLongs, buffer); + } else if (serializeScale > fastScale) { + + final int scaleUp = serializeScale - fastScale; + final int maxScale = FAST_MAX_SCALE - fastIntegerDigitCount; + // System.out.println("FAST_BIG_INTEGER_BYTES_SCALED scaleUp " + scaleUp); + // System.out.println("FAST_BIG_INTEGER_BYTES_SCALED maxScale " + maxScale); + if (serializeScale > maxScale) { + + // We cannot to scaled up decimals that cannot be represented. + // Instead, we use a BigInteger instead. + + BigInteger bigInteger = + fastBigIntegerValue( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale); + + BigInteger bigIntegerScaled = bigInteger.multiply(BIG_INTEGER_TEN.pow(scaleUp)); + byte[] bigIntegerBytesScaled = bigIntegerScaled.toByteArray(); + final int length = bigIntegerBytesScaled.length; + // System.out.println("FAST_BIG_INTEGER_BYTES_SCALED bigInteger " + bigInteger); + // System.out.println("FAST_BIG_INTEGER_BYTES_SCALED bigIntegerScaled " + bigIntegerScaled); + // System.out.println("FAST_BIG_INTEGER_BYTES_SCALED bigIntegerBytesScaled " + displayBytes(bigIntegerBytesScaled, 0, bigIntegerBytesScaled.length)); + // System.out.println("FAST_BIG_INTEGER_BYTES_SCALED length " + length); + System.arraycopy(bigIntegerBytesScaled, 0, buffer, 0, length); + return length; + } + + FastHiveDecimal fastTemp = new FastHiveDecimal(); + if (!fastScaleUp( + fast0, fast1, fast2, + scaleUp, + fastTemp)) { + throw new RuntimeException("Unexpected"); + } + // System.out.println("FAST_BIG_INTEGER_BYTES_SCALED scaleUp " + scaleUp + " fast0 " + fastTemp.fast0 + " fast1 " + fastTemp.fast1 + " fast2 " + fastTemp.fast2); + return + fastBigIntegerBytes( + fastSignum, fastTemp.fast0, fastTemp.fast1, fastTemp.fast2, + scratchLongs, buffer); + } else { + + // serializeScale < fastScale. + + FastHiveDecimal fastTemp = new FastHiveDecimal(); + if (!fastRound( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + serializeScale, FAST_ROUND_HALF_UP, + fastTemp)) { + return 0; + } + return + fastBigIntegerBytes( + fastSignum, fastTemp.fast0, fastTemp.fast1, fastTemp.fast2, + scratchLongs, buffer); + } + } + + //************************************************************************************************ + // Decimal to Integer conversion. + + private static final int MAX_BYTE_DIGITS = 3; + private static FastHiveDecimal FASTHIVEDECIMAL_MIN_BYTE_VALUE_MINUS_ONE = + new FastHiveDecimal((long) Byte.MIN_VALUE - 1L); + private static FastHiveDecimal FASTHIVEDECIMAL_MAX_BYTE_VALUE_PLUS_ONE = + new FastHiveDecimal((long) Byte.MAX_VALUE + 1L); + + private static final int MAX_SHORT_DIGITS = 5; + private static FastHiveDecimal FASTHIVEDECIMAL_MIN_SHORT_VALUE_MINUS_ONE = + new FastHiveDecimal((long) Short.MIN_VALUE - 1L); + private static FastHiveDecimal FASTHIVEDECIMAL_MAX_SHORT_VALUE_PLUS_ONE = + new FastHiveDecimal((long) Short.MAX_VALUE + 1L); + + private static final int MAX_INT_DIGITS = 10; + private static FastHiveDecimal FASTHIVEDECIMAL_MIN_INT_VALUE_MINUS_ONE = + new FastHiveDecimal((long) Integer.MIN_VALUE - 1L); + private static FastHiveDecimal FASTHIVEDECIMAL_MAX_INT_VALUE_PLUS_ONE = + new FastHiveDecimal((long) Integer.MAX_VALUE + 1L); + + private static FastHiveDecimal FASTHIVEDECIMAL_MIN_LONG_VALUE = + new FastHiveDecimal(Long.MIN_VALUE); + private static FastHiveDecimal FASTHIVEDECIMAL_MAX_LONG_VALUE = + new FastHiveDecimal(Long.MAX_VALUE); + private static final int MAX_LONG_DIGITS = + FASTHIVEDECIMAL_MAX_LONG_VALUE.fastIntegerDigitCount; + private static FastHiveDecimal FASTHIVEDECIMAL_MIN_LONG_VALUE_MINUS_ONE = + new FastHiveDecimal("-9223372036854775809"); + private static FastHiveDecimal FASTHIVEDECIMAL_MAX_LONG_VALUE_PLUS_ONE = + new FastHiveDecimal("9223372036854775808"); + + private static final BigInteger BIG_INTEGER_UNSIGNED_BYTE_MAX_VALUE = BIG_INTEGER_TWO.pow(Byte.SIZE).subtract(BigInteger.ONE); + private static final BigInteger BIG_INTEGER_UNSIGNED_SHORT_MAX_VALUE = BIG_INTEGER_TWO.pow(Short.SIZE).subtract(BigInteger.ONE); + private static final BigInteger BIG_INTEGER_UNSIGNED_INT_MAX_VALUE = BIG_INTEGER_TWO.pow(Integer.SIZE).subtract(BigInteger.ONE); + private static final BigInteger BIG_INTEGER_UNSIGNED_LONG_MAX_VALUE = BIG_INTEGER_TWO.pow(Long.SIZE).subtract(BigInteger.ONE); + + public static boolean fastIsByte( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastIntegerDigitCount < MAX_BYTE_DIGITS) { + + // Definitely a byte; most bytes fall here + return true; + + } else if (fastIntegerDigitCount > MAX_BYTE_DIGITS) { + + // Definitely not a byte. + return false; + + } else if (fastScale == 0) { + if (fast1 != 0 || fast2 != 0) { + return false; + } + if (fastSignum == 1) { + return (fast0 <= Byte.MAX_VALUE); + } else { + return (-fast0 >= Byte.MIN_VALUE); + } + } else { + + // We need to work a little harder for our comparison. Note we round down for + // integer conversion so anything below the next min/max will work. + + if (fastSignum == 1) { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MAX_BYTE_VALUE_PLUS_ONE) < 0); + } else { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MIN_BYTE_VALUE_MINUS_ONE) > 0); + } + } + } + + public static byte fastByteValueClip( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastScale == 0) { + if (fast1 == 0 && fast2 == 0) { + if (fastSignum == 1) { + if (fast0 <= Byte.MAX_VALUE) { + return (byte) fast0; + } + } else { + if (-fast0 >= Byte.MIN_VALUE) { + return (byte) -fast0; + }; + } + } + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValue( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_BYTE_MAX_VALUE).byteValue(); + } else { + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + if (fastScale < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[fastScale]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale]; + + result0 = + fast0 / divideFactor + + ((fast1 % divideFactor) * multiplyFactor); + result1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result2 = + fast2 / divideFactor; + + } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + result0 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result1 = + fast2 / divideFactor; + result2 = 0; + + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; + + result0 = + fast2 / powerOfTenTable[adjustedScaleDown]; + result1 = 0; + result2 = 0; + + } + + if (result1 == 0 && result2 == 0) { + if (fastSignum == 1) { + if (result0 <= Byte.MAX_VALUE) { + return (byte) result0; + } + } else { + if (-result0 >= Byte.MIN_VALUE) { + return (byte) -result0; + }; + } + } + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValue( + fastSignum, result0, result1, result2, fastIntegerDigitCount, /* fastScale */ 0); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_BYTE_MAX_VALUE).byteValue(); + } + } + + public static boolean fastIsShort( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastIntegerDigitCount < MAX_SHORT_DIGITS) { + + // Definitely a short; most shorts fall here + return true; + + } else if (fastIntegerDigitCount > MAX_SHORT_DIGITS) { + + // Definitely not a short. + return false; + + } else if (fastScale == 0) { + if (fast1 != 0 || fast2 != 0) { + return false; + } + if (fastSignum == 1) { + return (fast0 <= Short.MAX_VALUE); + } else { + return (-fast0 >= Short.MIN_VALUE); + } + } else { + + + // We need to work a little harder for our comparison. Note we round down for + // integer conversion so anything below the next min/max will work. + + if (fastSignum == 1) { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MAX_SHORT_VALUE_PLUS_ONE) < 0); + } else { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MIN_SHORT_VALUE_MINUS_ONE) > 0); + } + } + } + + public static short fastShortValueClip( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastScale == 0) { + if (fast1 == 0 && fast2 == 0) { + if (fastSignum == 1) { + if (fast0 <= Short.MAX_VALUE) { + return (short) fast0; + } + } else { + if (-fast0 >= Short.MIN_VALUE) { + return (short) -fast0; + }; + } + } + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValue( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_SHORT_MAX_VALUE).shortValue(); + } else { + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + if (fastScale < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[fastScale]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale]; + + result0 = + fast0 / divideFactor + + ((fast1 % divideFactor) * multiplyFactor); + result1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result2 = + fast2 / divideFactor; + + } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + result0 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result1 = + fast2 / divideFactor; + result2 = 0; + + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; + + result0 = + fast2 / powerOfTenTable[adjustedScaleDown]; + result1 = 0; + result2 = 0; + + } + + if (result1 == 0 && result2 == 0) { + if (fastSignum == 1) { + if (result0 <= Short.MAX_VALUE) { + return (short) result0; + } + } else { + if (-result0 >= Short.MIN_VALUE) { + return (short) -result0; + }; + } + } + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValue( + fastSignum, result0, result1, result2, fastIntegerDigitCount, /* fastScale */ 0); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_SHORT_MAX_VALUE).shortValue(); + } + } + + public static boolean fastIsInt( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastIntegerDigitCount < MAX_INT_DIGITS) { + + // Definitely a int; most ints fall here + return true; + + } else if (fastIntegerDigitCount > MAX_INT_DIGITS) { + + // Definitely not an int. + return false; + + } else if (fastScale == 0) { + if (fast1 != 0 || fast2 != 0) { + return false; + } + if (fastSignum == 1) { + return (fast0 <= Integer.MAX_VALUE); + } else { + return (-fast0 >= Integer.MIN_VALUE); + } + } else { + + // We need to work a little harder for our comparison. Note we round down for + // integer conversion so anything below the next min/max will work. + + if (fastSignum == 1) { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MAX_INT_VALUE_PLUS_ONE) < 0); + } else { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MIN_INT_VALUE_MINUS_ONE) > 0); + } + } + } + + public static int fastIntValueClip( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastScale == 0) { + if (fast1 == 0 && fast2 == 0) { + if (fastSignum == 1) { + if (fast0 <= Integer.MAX_VALUE) { + return (int) fast0; + } + } else { + if (-fast0 >= Integer.MIN_VALUE) { + return (int) -fast0; + }; + } + } + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValue( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_INT_MAX_VALUE).intValue(); + } else { + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + if (fastScale < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[fastScale]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale]; + + result0 = + fast0 / divideFactor + + ((fast1 % divideFactor) * multiplyFactor); + result1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result2 = + fast2 / divideFactor; + + } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + result0 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result1 = + fast2 / divideFactor; + result2 = 0; + + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; + + result0 = + fast2 / powerOfTenTable[adjustedScaleDown]; + result1 = 0; + result2 = 0; + + } + + if (result1 == 0 && result2 == 0) { + if (fastSignum == 1) { + if (result0 <= Integer.MAX_VALUE) { + return (int) result0; + } + } else { + if (-result0 >= Integer.MIN_VALUE) { + return (int) -result0; + }; + } + } + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValue( + fastSignum, result0, result1, result2, fastIntegerDigitCount, /* fastScale */ 0); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_INT_MAX_VALUE).intValue(); + } + } + + public static boolean fastIsLong( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastIntegerDigitCount < MAX_LONG_DIGITS) { + + // Definitely a long; most longs fall here + return true; + + } else if (fastIntegerDigitCount > MAX_LONG_DIGITS) { + + // Definitely not a long. + return false; + + } else if (fastScale == 0) { + + // From the above checks, we know fast2 is zero. + + if (fastSignum == 1) { + FastHiveDecimal max = FASTHIVEDECIMAL_MAX_LONG_VALUE; + if (fast1 > max.fast1 || (fast1 == max.fast1 && fast0 > max.fast0)) { + return false; + } + return true; + } else { + FastHiveDecimal min = FASTHIVEDECIMAL_MIN_LONG_VALUE; + if (fast1 > min.fast1 || (fast1 == min.fast1 && fast0 > min.fast0)) { + return false; + } + return true; + } + + } else { + + // We need to work a little harder for our comparison. Note we round down for + // integer conversion so anything below the next min/max will work. + + if (fastSignum == 1) { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MAX_LONG_VALUE_PLUS_ONE) < 0); + } else { + return + (fastCompareTo( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MIN_LONG_VALUE_MINUS_ONE) > 0); + } + } + } + + public static long fastLongValueClip( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + if (fastSignum == 0) { + return 0; + } + + if (fastScale == 0) { + // Do first comparison as unsigned. + if (fastCompareTo( + 1, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MAX_LONG_VALUE) <= 0) { + if (fastSignum == 1) { + return + fast1 * MULTIPLER_LONGWORD_DECIMAL + + fast0; + } else { + return + -(fast1 * MULTIPLER_LONGWORD_DECIMAL + + fast0); + } + } if (fastEquals( + fastSignum, fast0, fast1, fast2, fastScale, + FASTHIVEDECIMAL_MIN_LONG_VALUE)) { + return Long.MIN_VALUE; + } else { + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValue( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_LONG_MAX_VALUE).longValue(); + } + } else { + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + if (fastScale < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[fastScale]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - fastScale]; + + result0 = + fast0 / divideFactor + + ((fast1 % divideFactor) * multiplyFactor); + result1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result2 = + fast2 / divideFactor; + + } else if (fastScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = fastScale - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + result0 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result1 = + fast2 / divideFactor; + result2 = 0; + + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = fastScale - 2*LONGWORD_DECIMAL_DIGITS; + + result0 = + fast2 / powerOfTenTable[adjustedScaleDown]; + result1 = 0; + result2 = 0; + + } + + // Do first comparison as unsigned. + if (fastCompareTo( + 1, result0, result1, result2, /* fastScale */ 0, + FASTHIVEDECIMAL_MAX_LONG_VALUE) <= 0) { + if (fastSignum == 1) { + return + result1 * MULTIPLER_LONGWORD_DECIMAL + + result0; + } else { + return + -(result1 * MULTIPLER_LONGWORD_DECIMAL + + result0); + } + } if (fastEquals( + fastSignum, result0, result1, result2, /* fastScale */ 0, + FASTHIVEDECIMAL_MIN_LONG_VALUE)) { + return Long.MIN_VALUE; + } else { + // SLOW: Do remainder with BigInteger. + BigInteger bigInteger = + fastBigIntegerValue( + fastSignum, result0, result1, result2, fastIntegerDigitCount, /* fastScale */ 0); + return bigInteger.remainder(BIG_INTEGER_UNSIGNED_LONG_MAX_VALUE).longValue(); + } + } + } + + //************************************************************************************************ + // Decimal to Non-Integer conversion. + + public static float fastFloatValue( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + if (fastSignum == 0) { + return 0; + } + BigDecimal bigDecimal = fastBigDecimalValue( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale); + return bigDecimal.floatValue(); + } + + public static double fastDoubleValue( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + if (fastSignum == 0) { + return 0; + } + BigDecimal bigDecimal = fastBigDecimalValue( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale); + return bigDecimal.doubleValue(); + } + + public static BigInteger fastBigIntegerValue( + int fastSignum, long fast0, long fast1, long fast2) { + if (fastSignum == 0) { + return BigInteger.ZERO; + } + BigInteger result; + if (fast2 == 0) { + if (fast1 == 0) { + result = + BigInteger.valueOf(fast0); + } else { + result = + BigInteger.valueOf(fast0).add( + BigInteger.valueOf(fast1).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER)); + } + } else { + result = + BigInteger.valueOf(fast0).add( + BigInteger.valueOf(fast1).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER)).add( + BigInteger.valueOf(fast2).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER_2X)); + } + + return (fastSignum == 1 ? result : result.negate()); + } + + public static BigInteger fastBigIntegerValue( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + if (fastSignum == 0) { + return BigInteger.ZERO; + } + BigInteger result; + if (fast2 == 0) { + if (fast1 == 0) { + result = + BigInteger.valueOf(fast0); + } else { + result = + BigInteger.valueOf(fast0).add( + BigInteger.valueOf(fast1).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER)); + } + } else { + result = + BigInteger.valueOf(fast0).add( + BigInteger.valueOf(fast1).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER)).add( + BigInteger.valueOf(fast2).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER_2X)); + } + + /* + if (fastTrailingZeroesScale != -1) { + final int trailingZeroesCount = fastTrailingZeroesScale - fastScale; + if (trailingZeroesCount > 0) { + if (trailingZeroesCount == 1) { + result = BIG_INTEGER_TEN.multiply(result); + } else { + result = BIG_INTEGER_TEN.pow(trailingZeroesCount).multiply(result); + } + } + } + */ + + return (fastSignum == 1 ? result : result.negate()); + } + + public static BigInteger fastBigIntegerValueFull( + int signum, long fast0, long fast1, long fast2, long fast3, long fast4) { + if (signum == 0) { + return BigInteger.ZERO; + } + BigInteger result; + if (fast4 == 0) { + if (fast3 == 0) { + if (fast2 == 0) { + if (fast1 == 0) { + result = + BigInteger.valueOf(fast0); + } else { + result = + BigInteger.valueOf(fast0).add( + BigInteger.valueOf(fast1).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER)); + } + } else { + result = + BigInteger.valueOf(fast0).add( + BigInteger.valueOf(fast1).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER)).add( + BigInteger.valueOf(fast2).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER_2X)); + } + } else { + result = + BigInteger.valueOf(fast0).add( + BigInteger.valueOf(fast1).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER)).add( + BigInteger.valueOf(fast2).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER_2X).add( + BigInteger.valueOf(fast3).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER_3X))); + } + } else { + result = + BigInteger.valueOf(fast0).add( + BigInteger.valueOf(fast1).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER)).add( + BigInteger.valueOf(fast2).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER_2X).add( + BigInteger.valueOf(fast3).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER_3X).add( + BigInteger.valueOf(fast4).multiply(BIG_INTEGER_LONGWORD_MULTIPLIER_4X)))); + } + return (signum == 1 ? result : result.negate()); + } + + public static BigDecimal fastBigDecimalValue( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + BigInteger unscaledValue = + fastBigIntegerValue( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale); + return new BigDecimal(unscaledValue, fastScale); + } + + //************************************************************************************************ + // Decimal Comparison. + + public static int fastCompareTo( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftScale, + FastHiveDecimal fastRight) { + + return + fastCompareTo( + leftSignum, leftFast0, leftFast1, leftFast2, + leftScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastScale); + } + + private static int doCompareToSameScale( + int signum, + long leftFast0, long leftFast1, long leftFast2, + long rightFast0, long rightFast1, long rightFast2) { + + if (leftFast0 == rightFast0 && leftFast1 == rightFast1 && leftFast2 == rightFast2) { + return 0; + } + if (signum == 1) { + if (leftFast2 < rightFast2) { + return -1; + } else if (leftFast2 > rightFast2) { + return 1; + } + if (leftFast1 < rightFast1) { + return -1; + } else if (leftFast1 > rightFast1){ + return 1; + } + return (leftFast0 < rightFast0 ? -1 : 1); + } else { + // Reverse return values. + if (leftFast2 < rightFast2) { + return 1; + } else if (leftFast2 > rightFast2) { + return -1; + } + if (leftFast1 < rightFast1) { + return 1; + } else if (leftFast1 > rightFast1){ + return -1; + } + return (leftFast0 < rightFast0 ? 1 : -1); + } + } + + public static int fastCompareTo( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightScale) { + + if (leftSignum == 0 && rightSignum == 0) { + return 0; + } + + // Optimization copied from BigDecimal. + int signDiff = leftSignum - rightSignum; + if (signDiff != 0) { + return (signDiff > 0 ? 1 : -1); + } + + // We are here when the left and right are non-zero and have the same sign. + + if (leftScale == rightScale) { + + return doCompareToSameScale( + leftSignum, + leftFast0, leftFast1, leftFast2, + rightFast0, rightFast1, rightFast2); + + } else { + + // How do we handle different scales? + + // We at least know they are not equal. The one with the larger scale has non-zero digits + // below the other's scale (since the scale does not include trailing zeroes). + + // For comparison purposes, we can scale away those digits. And, we can not scale up since + // that could overflow. + + // Use modified portions of doFastScaleDown code here since we do not want to allocate a + // temporary FastHiveDecimal object. + + long compare0; + long compare1; + long compare2; + int scaleDown; + if (leftScale < rightScale) { + + // Scale down right and compare. + scaleDown = rightScale - leftScale; + + // Adjust all longs using power 10 division/remainder. + + if (scaleDown < LONGWORD_DECIMAL_DIGITS) { + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[scaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; + + compare0 = + rightFast0 / divideFactor + + ((rightFast1 % divideFactor) * multiplyFactor); + compare1 = + rightFast1 / divideFactor + + ((rightFast2 % divideFactor) * multiplyFactor); + compare2 = + rightFast2 / divideFactor; + } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { + // Throw away lowest word. + + final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + compare0 = + rightFast1 / divideFactor + + ((rightFast2 % divideFactor) * multiplyFactor); + compare1 = + rightFast2 / divideFactor; + compare2 = 0; + } else { + // Throw away middle and lowest words. + + final int adjustedScaleDown = scaleDown - TWO_X_LONGWORD_DECIMAL_DIGITS; + + compare0 = + rightFast2 / powerOfTenTable[adjustedScaleDown]; + compare1 = 0; + compare2 = 0; + } + + if (leftSignum == 1) { + if (leftFast0 == compare0 && leftFast1 == compare1 && leftFast2 == compare2) { + // Return less than because of right's digits below left's scale. + return -1; + } + if (leftFast2 < compare2) { + return -1; + } else if (leftFast2 > compare2) { + return 1; + } + if (leftFast1 < compare1) { + return -1; + } else if (leftFast1 > compare1){ + return 1; + } + return (leftFast0 < compare0 ? -1 : 1); + } else { + // Reverse return values. + if (leftFast0 == compare0 && leftFast1 == compare1 && leftFast2 == compare2) { + return 1; + } + if (leftFast2 < compare2) { + return 1; + } else if (leftFast2 > compare2) { + return -1; + } + if (leftFast1 < compare1) { + return 1; + } else if (leftFast1 > compare1){ + return -1; + } + return (leftFast0 < compare0 ? 1 : -1); + + } + } else { + + // Scale down left and compare. + scaleDown = leftScale - rightScale; + + // Adjust all longs using power 10 division/remainder. + + if (scaleDown < LONGWORD_DECIMAL_DIGITS) { + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[scaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; + + compare1 = + leftFast1 / divideFactor + + ((leftFast2 % divideFactor) * multiplyFactor); + compare0 = + leftFast0 / divideFactor + + ((leftFast1 % divideFactor) * multiplyFactor); + compare2 = + leftFast2 / divideFactor; + } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { + // Throw away lowest word. + + final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + compare0 = + leftFast1 / divideFactor + + ((leftFast2 % divideFactor) * multiplyFactor); + compare1 = + leftFast2 / divideFactor; + compare2 = 0; + } else { + // Throw away middle and lowest words. + + final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS; + + compare0 = + leftFast2 / powerOfTenTable[adjustedScaleDown]; + compare1 = 0; + compare2 = 0; + } + + if (leftSignum == 1) { + if (compare0 == rightFast0 && compare1 == rightFast1 && compare2 == rightFast2) { + // Return greater than because of left's digits below right's scale. + return 1; + } + if (compare2 < rightFast2) { + return -1; + } else if (compare2 > rightFast2) { + return 1; + } + if (compare1 < rightFast1) { + return -1; + } else if (compare1 > rightFast1){ + return 1; + } + return (compare0 < rightFast0 ? -1 : 1); + } else { + if (compare0 == rightFast0 && compare1 == rightFast1 && compare2 == rightFast2) { + // Return greater than because of left's digits below right's scale. + return -1; + } + if (compare2 < rightFast2) { + return 1; + } else if (compare2 > rightFast2) { + return -1; + } + if (compare1 < rightFast1) { + return 1; + } else if (compare1 > rightFast1){ + return -1; + } + return (compare0 < rightFast0 ? 1 : -1); + } + } + } + } + + public static boolean fastEquals( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftScale, + FastHiveDecimal fastRight) { + + if (leftSignum == 0) { + return (fastRight.fastSignum == 0); + } + if (leftSignum != fastRight.fastSignum) { + return false; + } + if (leftScale != fastRight.fastScale) { + // We know they are not equal because the one with the larger scale has non-zero digits + // below the other's scale (since the scale does not include trailing zeroes). + return false; + } + return ( + leftFast0 == fastRight.fast0 && leftFast1 == fastRight.fast1 && leftFast2 == fastRight.fast2); + } + + public static boolean fastEquals( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightScale) { + + if (leftSignum == 0) { + return (rightSignum == 0); + } + if (leftSignum != rightSignum) { + return false; + } + if (leftScale != rightScale) { + // We know they are not equal because the one with the larger scale has non-zero digits + // below the other's scale (since the scale does not include trailing zeroes). + return false; + } + return ( + leftFast0 == rightFast0 && leftFast1 == rightFast1 && leftFast2 == rightFast2); + } + + private static int doCalculateHashCode( + int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) { + + long longHashCode; + + long key = fast0; + + // Hash code logic from calculateLongHashCode. + + key = (~key) + (key << 21); // key = (key << 21) - key - 1; + key = key ^ (key >>> 24); + key = (key + (key << 3)) + (key << 8); // key * 265 + key = key ^ (key >>> 14); + key = (key + (key << 2)) + (key << 4); // key * 21 + key = key ^ (key >>> 28); + key = key + (key << 31); + + longHashCode = key; + + key = fast1; + + key = (~key) + (key << 21); // key = (key << 21) - key - 1; + key = key ^ (key >>> 24); + key = (key + (key << 3)) + (key << 8); // key * 265 + key = key ^ (key >>> 14); + key = (key + (key << 2)) + (key << 4); // key * 21 + key = key ^ (key >>> 28); + key = key + (key << 31); + + longHashCode ^= key; + + key = fast2; + + key = (~key) + (key << 21); // key = (key << 21) - key - 1; + key = key ^ (key >>> 24); + key = (key + (key << 3)) + (key << 8); // key * 265 + key = key ^ (key >>> 14); + key = (key + (key << 2)) + (key << 4); // key * 21 + key = key ^ (key >>> 28); + key = key + (key << 31); + + longHashCode ^= key; + + key = fastSignum; + + key = (~key) + (key << 21); // key = (key << 21) - key - 1; + key = key ^ (key >>> 24); + key = (key + (key << 3)) + (key << 8); // key * 265 + key = key ^ (key >>> 14); + key = (key + (key << 2)) + (key << 4); // key * 21 + key = key ^ (key >>> 28); + key = key + (key << 31); + + longHashCode ^= key; + + key = fastIntegerDigitCount; + + key = (~key) + (key << 21); // key = (key << 21) - key - 1; + key = key ^ (key >>> 24); + key = (key + (key << 3)) + (key << 8); // key * 265 + key = key ^ (key >>> 14); + key = (key + (key << 2)) + (key << 4); // key * 21 + key = key ^ (key >>> 28); + key = key + (key << 31); + + longHashCode ^= key; + + key = fastScale; + + key = (~key) + (key << 21); // key = (key << 21) - key - 1; + key = key ^ (key >>> 24); + key = (key + (key << 3)) + (key << 8); // key * 265 + key = key ^ (key >>> 14); + key = (key + (key << 2)) + (key << 4); // key * 21 + key = key ^ (key >>> 28); + key = key + (key << 31); + + longHashCode ^= key; + + return (int) longHashCode; + } + + private static final int ZERO_HASH_CODE = doCalculateHashCode(0, 0, 0, 0, 0, 0); + public static int fastHashCode( + int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) { + if (fastSignum == 0) { + return ZERO_HASH_CODE; + } + int hashCode = doCalculateHashCode(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + return hashCode; + } + + public static int fastOldHiveDecimalHashCode( + int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) { + + // OldHiveDecimal returns the hash code of its internal BigDecimal. Our TestHiveDecimal + // verifies the OldHiveDecimal.bigDecimalValue() matches (new) HiveDecimal.bigDecimalValue(). + + BigDecimal bigDecimal = + fastBigDecimalValue( + fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + + return bigDecimal.hashCode(); + } + + //************************************************************************************************ + // Decimal Math. + + public static boolean fastScaleByPowerOfTen( + FastHiveDecimal fastDec, + int power, + FastHiveDecimal fastResult) { + return fastScaleByPowerOfTen( + fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2, + fastDec.fastIntegerDigitCount, fastDec.fastScale, + power, + fastResult); + } + + // NOTE: power can be positive or negative. + // NOTE: e.g. power = 2 is effectively multiply by 10^2 + // NOTE: and power = -3 is multiply by 10^-3 + public static boolean fastScaleByPowerOfTen( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int power, + FastHiveDecimal fastResult) { + + if (fastSignum == 0) { + fastResult.fastReset(); + return true; + } + if (power == 0) { + fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + return true; + } + + final int absPower = Math.abs(power); + + if (power > 0) { + + int integerRoom; + int fractionalRoom; + if (fastIntegerDigitCount > 0) { + + // Is there integer room above? + + integerRoom = FAST_MAX_PRECISION - fastIntegerDigitCount; + if (integerRoom < power) { + return false; + } + fastResult.fastSignum = fastSignum; + if (fastScale <= power) { + + // All fractional digits become integer digits. + final int scaleUp = power - fastScale; + if (scaleUp > 0) { + if (!fastScaleUp( + fast0, fast1, fast2, + scaleUp, + fastResult)) { + throw new RuntimeException("Unexpected"); + } + } else { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } + fastResult.fastIntegerDigitCount = fastIntegerDigitCount + fastScale + scaleUp; + fastResult.fastScale = 0; + + } else { + + // Only a scale adjustment is needed. + fastResult.fastSet( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount + power, fastScale - power); + } + } else { + + // How much can the fraction be moved up? + + final int rawPrecision = fastRawPrecision(fastSignum, fast0, fast1, fast2); + final int zeroesBelowDot = fastScale - rawPrecision; + + // Our limit is max precision integer digits + "leading" zeros below the dot. + // E.g. 0.00021 has 3 zeroes below the dot. + // + if (power > FAST_MAX_PRECISION + zeroesBelowDot) { + + // Fractional part powered up too high. + return false; + } + + final int newIntegerDigitCount = Math.max(0, power - zeroesBelowDot); + if (newIntegerDigitCount > rawPrecision) { + + fastResult.fastSignum = fastSignum; + final int scaleUp = newIntegerDigitCount - rawPrecision; + if (!fastScaleUp( + fast0, fast1, fast2, + scaleUp, + fastResult)) { + throw new RuntimeException("Unexpected"); + } + fastResult.fastIntegerDigitCount = newIntegerDigitCount; + fastResult.fastScale = 0; + } else { + final int newScale = Math.max(0, fastScale - power); + fastResult.fastSet(fastSignum, fast0, fast1, fast2, newIntegerDigitCount, newScale); + } + } + + } else if (fastScale + absPower <= FAST_MAX_SCALE) { + + // Negative power with range -- adjust the scale. + + final int newScale = fastScale + absPower; + final int newIntegerDigitCount = Math.max(0, fastIntegerDigitCount - absPower); + + final int trailingZeroCount = + fastTrailingDecimalZeroCount( + fast0, fast1, fast2, + newIntegerDigitCount, newScale); + if (trailingZeroCount > 0) { + fastResult.fastSignum = fastSignum; + doFastScaleDown( + fast0, fast1, fast2, + trailingZeroCount, fastResult); + fastResult.fastScale = newScale - trailingZeroCount; + fastResult.fastIntegerDigitCount = newIntegerDigitCount; + } else { + fastResult.fastSet(fastSignum, fast0, fast1, fast2, + newIntegerDigitCount, newScale); + } + } else { + + // fastScale + absPower > FAST_MAX_SCALE + + // Look at getting rid of fractional digits that will now be below FAST_MAX_SCALE. + + final int scaleDown = fastScale + absPower - FAST_MAX_SCALE; + + if (scaleDown < FAST_MAX_SCALE) { + if (!fastRoundFractionalHalfUp( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult)) { + // Overflow. + return false; + } + if (fastResult.fastSignum != 0) { + + fastResult.fastScale = FAST_MAX_SCALE; + fastResult.fastIntegerDigitCount = + Math.max(0, fastRawPrecision(fastResult) - fastResult.fastScale); + + final int trailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (trailingZeroCount > 0) { + doFastScaleDown( + fastResult, + trailingZeroCount, + fastResult); + fastResult.fastScale -= trailingZeroCount; + } + } + } else { + // All precision has been lost -- result is 0. + fastResult.fastReset(); + } + } + + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + + return true; + } + + //************************************************************************************************ + // Decimal Rounding. + + public static boolean doFastRound( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int roundPower, + int roundingMode, + FastHiveDecimal fastResult) { + + if (fastSignum == 0) { + + // Zero result. + fastResult.fastReset(); + return true; + } else if (fastScale == roundPower) { + + // The roundPower same as scale means all zeroes below round point. + + fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + return true; + } + + if (roundPower > fastScale) { + + // We pretend to add trailing zeroes, EVEN WHEN it would exceed the FAST_MAX_PRECISION. + + // Copy current value; do not change current scale. + fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + } else if (roundPower < 0) { + + // roundPower < 0 + // + // Negative scale means we start rounding integer digits. + // + // The result will integer result will have at least abs(roundPower) trailing digits. + // + // Examples where the 'r's show the rounding digits: + // + // round(12500, -3) = 13000 // FAST_ROUND_HALF_UP + // rrr + // + // Or, ceiling(12400.8302, -2) = 12500 // FAST_ROUND_CEILING + // rr rrrr + // + // Notice that any fractional digits will be gone in the result. + // + switch (roundingMode) { + case FAST_ROUND_DOWN: + if (!fastRoundIntegerDown( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + break; + case FAST_ROUND_UP: + if (!fastRoundIntegerUp( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + break; + case FAST_ROUND_FLOOR: + // Round towards negative infinity. + if (fastSignum == 1) { + if (!fastRoundIntegerDown( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + } else { + if (!fastRoundIntegerUp( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) { + return false; + } + } + break; + case FAST_ROUND_CEILING: + // Round towards positive infinity. + if (fastSignum == 1) { + if (!fastRoundIntegerUp( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) { + return false; + } + } else { + if (!fastRoundIntegerDown( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + } + break; + case FAST_ROUND_HALF_UP: + if (!fastRoundIntegerHalfUp( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) { + return false; + } + break; + case FAST_ROUND_HALF_EVEN: + if (!fastRoundIntegerHalfEven( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + roundPower, + fastResult)) { + return false; + } + if (fastResult.fast2 > MAX_HIGHWORD_DECIMAL) { + return false; + } + break; + default: + throw new RuntimeException("Unsupported rounding mode " + roundingMode); + } + + // The fastRoundInteger* methods remove all fractional digits, set fastIntegerDigitCount, and + // set fastScale to 0. + return true; + + } else { + + // roundPower < fastScale + + // Do rounding of fractional digits. + final int scaleDown = fastScale - roundPower; + switch (roundingMode) { + case FAST_ROUND_DOWN: + fastRoundFractionalDown( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult); + break; + case FAST_ROUND_UP: + if (!fastRoundFractionalUp( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult)) { + return false; + } + break; + case FAST_ROUND_FLOOR: + // Round towards negative infinity. + if (fastSignum == 1) { + fastRoundFractionalDown( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult); + } else { + if (!fastRoundFractionalUp( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult)) { + return false; + } + } + break; + case FAST_ROUND_CEILING: + // Round towards positive infinity. + if (fastSignum == 1) { + if (!fastRoundFractionalUp( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult)) { + return false; + } + } else { + fastRoundFractionalDown( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult); + } + break; + case FAST_ROUND_HALF_UP: + if (!fastRoundFractionalHalfUp( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult)) { + return false; + } + break; + case FAST_ROUND_HALF_EVEN: + if (!fastRoundFractionalHalfEven( + fastSignum, fast0, fast1, fast2, + scaleDown, + fastResult)) { + return false; + } + break; + default: + throw new RuntimeException("Unsupported rounding mode " + roundingMode); + } + if (fastResult.fastSignum == 0) { + fastResult.fastScale = 0; + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + } else { + final int rawPrecision = fastRawPrecision(fastResult); + fastResult.fastIntegerDigitCount = Math.max(0, rawPrecision - roundPower); + fastResult.fastScale = roundPower; + + // Trim trailing zeroes and re-adjust scale. + final int trailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (trailingZeroCount > 0) { + doFastScaleDown( + fastResult, + trailingZeroCount, + fastResult); + fastResult.fastScale -= trailingZeroCount; + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + } + } + } + + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + return true; + } + + public static boolean fastRound( + FastHiveDecimal fastDec, + int newScale, int roundingMode, + FastHiveDecimal fastResult) { + return fastRound( + fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2, + fastDec.fastIntegerDigitCount, fastDec.fastScale, + newScale, roundingMode, + fastResult); + } + + public static boolean fastRound( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int newScale, int roundingMode, + FastHiveDecimal fastResult) { + if (!doFastRound( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + newScale, roundingMode, + fastResult)) { + return false; + } + return true; + } + + private static boolean isRoundPortionAllZeroes( + long fast0, long fast1, long fast2, + int roundingPoint) { + + boolean isRoundPortionAllZeroes; + if (roundingPoint < LONGWORD_DECIMAL_DIGITS) { + + // Lowest word gets integer rounding. + + // Factor includes scale. + final long roundPointFactor = powerOfTenTable[roundingPoint]; + + isRoundPortionAllZeroes = (fast0 % roundPointFactor == 0); + + } else if (roundingPoint < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Middle word gets integer rounding. + + final int adjustedRoundingPoint = roundingPoint - LONGWORD_DECIMAL_DIGITS; + + if (adjustedRoundingPoint == 0) { + isRoundPortionAllZeroes = (fast0 == 0); + } else { + + // Factor includes scale. + final long roundPointFactor = powerOfTenTable[adjustedRoundingPoint]; + + final long roundPortion = fast1 % roundPointFactor; + isRoundPortionAllZeroes = (roundPortion == 0 && fast0 == 0); + } + + } else { + + // High word gets integer rounding. + + final int adjustedRoundingPoint = roundingPoint - TWO_X_LONGWORD_DECIMAL_DIGITS; + + if (adjustedRoundingPoint == 0) { + isRoundPortionAllZeroes = (fast0 == 0 && fast1 == 0); + } else { + + // Factor includes scale. + final long roundPointFactor = powerOfTenTable[adjustedRoundingPoint]; + + final long roundPortion = fast2 % roundPointFactor; + isRoundPortionAllZeroes = (roundPortion == 0 && fast0 == 0 && fast1 == 0); + } + } + return isRoundPortionAllZeroes; + } + + private static boolean isRoundPortionHalfUp( + long fast0, long fast1, long fast2, + int roundingPoint) { + + boolean isRoundPortionHalfUp; + if (roundingPoint < LONGWORD_DECIMAL_DIGITS) { + + // Lowest word gets integer rounding. + + // Divide down just before round point to get round digit. + final long withRoundDigit = fast0 / powerOfTenTable[roundingPoint - 1]; + final long roundDigit = withRoundDigit % 10; + + isRoundPortionHalfUp = (roundDigit >= 5); + + } else if (roundingPoint < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Middle word gets integer rounding. + + final int adjustedRoundingPoint = roundingPoint - LONGWORD_DECIMAL_DIGITS; + + long roundDigit; + if (adjustedRoundingPoint == 0) { + // Grab round digit from lowest word. + roundDigit = fast0 / (MULTIPLER_LONGWORD_DECIMAL / 10); + } else { + // Divide down just before scaleDown to get round digit. + final long withRoundDigit = fast1 / powerOfTenTable[adjustedRoundingPoint - 1]; + roundDigit = withRoundDigit % 10; + } + + isRoundPortionHalfUp = (roundDigit >= 5); + + } else { + + // High word gets integer rounding. + + final int adjustedRoundingPoint = roundingPoint - TWO_X_LONGWORD_DECIMAL_DIGITS; + + long roundDigit; + if (adjustedRoundingPoint == 0) { + // Grab round digit from middle word. + roundDigit = fast1 / (MULTIPLER_LONGWORD_DECIMAL / 10); + } else { + // Divide down just before scaleDown to get round digit. + final long withRoundDigit = fast2 / powerOfTenTable[adjustedRoundingPoint - 1]; + roundDigit = withRoundDigit % 10; + } + + isRoundPortionHalfUp = (roundDigit >= 5); + + } + return isRoundPortionHalfUp; + } + + private static boolean isRoundPortionHalfEven( + long fast0, long fast1, long fast2, + int roundingPoint) { + + boolean isRoundPortionHalfEven; + if (roundingPoint < LONGWORD_DECIMAL_DIGITS) { + + // Lowest word gets integer rounding. + + // Divide down just before scaleDown to get round digit. + final long roundDivisor = powerOfTenTable[roundingPoint - 1]; + final long withRoundDigit = fast0 / roundDivisor; + final long roundDigit = withRoundDigit % 10; + final long fast0Scaled = withRoundDigit / 10; + + if (roundDigit > 5) { + isRoundPortionHalfEven = true; + } else if (roundDigit == 5) { + boolean exactlyOneHalf; + if (roundingPoint - 1 == 0) { + // Fraction below 0.5 is implicitly 0. + exactlyOneHalf = true; + } else { + exactlyOneHalf = (fast0 % roundDivisor == 0); + } + + // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. + if (exactlyOneHalf) { + isRoundPortionHalfEven = (fast0Scaled % 2 == 1); + } else { + isRoundPortionHalfEven = true; + } + } else { + isRoundPortionHalfEven = false; + } + + } else if (roundingPoint < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Middle word gets integer rounding. + + final int adjustedRoundingPoint = roundingPoint - LONGWORD_DECIMAL_DIGITS; + + long roundDigit; + long fast1Scaled; + if (adjustedRoundingPoint == 0) { + // Grab round digit from lowest word. + final long roundDivisor = MULTIPLER_LONGWORD_DECIMAL / 10; + roundDigit = fast0 / roundDivisor; + fast1Scaled = fast1; + if (roundDigit > 5) { + isRoundPortionHalfEven = true; + } else if (roundDigit == 5) { + boolean exactlyOneHalf = (fast0 % roundDivisor == 0); + + // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. + if (exactlyOneHalf) { + isRoundPortionHalfEven = (fast1Scaled % 2 == 1); + } else { + isRoundPortionHalfEven = true; + } + } else { + isRoundPortionHalfEven = false; + } + } else { + // Divide down just before scaleDown to get round digit. + final long roundDivisor = powerOfTenTable[adjustedRoundingPoint - 1]; + final long withRoundDigit = fast1 / roundDivisor; + roundDigit = withRoundDigit % 10; + fast1Scaled = withRoundDigit / 10; + if (roundDigit > 5) { + isRoundPortionHalfEven = true; + } else if (roundDigit == 5) { + boolean exactlyOneHalf; + if (adjustedRoundingPoint - 1 == 0) { + // Just examine the lower word. + exactlyOneHalf = (fast0 == 0); + } else { + exactlyOneHalf = (fast1 % roundDivisor == 0 && fast0 == 0); + } + + // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. + if (exactlyOneHalf) { + isRoundPortionHalfEven = (fast1Scaled % 2 == 1); + } else { + isRoundPortionHalfEven = true; + } + } else { + isRoundPortionHalfEven = false; + } + } + + } else { + + // High word gets integer rounding. + + final int adjustedRoundingPoint = roundingPoint - TWO_X_LONGWORD_DECIMAL_DIGITS; + + long roundDigit; + long fast2Scaled; + if (adjustedRoundingPoint == 0) { + // Grab round digit from middle word. + final long roundDivisor = MULTIPLER_LONGWORD_DECIMAL / 10; + roundDigit = fast1 / roundDivisor; + fast2Scaled = fast2; + if (roundDigit > 5) { + isRoundPortionHalfEven = true; + } else if (roundDigit == 5) { + boolean exactlyOneHalf = (fast1 % roundDivisor == 0 && fast0 == 0); + + // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. + if (exactlyOneHalf) { + isRoundPortionHalfEven = (fast2Scaled % 2 == 1); + } else { + isRoundPortionHalfEven = true; + } + } else { + isRoundPortionHalfEven = false; + } + } else { + // Divide down just before scaleDown to get round digit. + final long roundDivisor = powerOfTenTable[adjustedRoundingPoint - 1]; + final long withRoundDigit = fast2 / roundDivisor; + roundDigit = withRoundDigit % 10; + fast2Scaled = withRoundDigit / 10; + if (roundDigit > 5) { + isRoundPortionHalfEven = true; + } else if (roundDigit == 5) { + boolean exactlyOneHalf; + if (adjustedRoundingPoint - 1 == 0) { + // Just examine the middle and lower words. + exactlyOneHalf = (fast1 == 0 && fast0 == 0); + } else { + exactlyOneHalf = (fast2 % roundDivisor == 0 && fast1 == 0 && fast0 == 0); + } + + // When fraction is exactly 0.5 and lowest new digit is odd, go towards even. + if (exactlyOneHalf) { + isRoundPortionHalfEven = (fast2Scaled % 2 == 1); + } else { + isRoundPortionHalfEven = true; + } + } else { + isRoundPortionHalfEven = false; + } + } + } + return isRoundPortionHalfEven; + } + + private static void doClearRoundIntegerPortionAndAddOne( + long fast0, long fast1, long fast2, + int absRoundPower, + FastHiveDecimal fastResult) { + + long result0; + long result1; + long result2; + + if (absRoundPower < LONGWORD_DECIMAL_DIGITS) { + + // Lowest word gets integer rounding. + + // Clear rounding portion in lower longword and add 1 at right scale (roundMultiplyFactor). + + final long roundFactor = powerOfTenTable[absRoundPower]; + + final long r0 = + ((fast0 / roundFactor) * roundFactor) + + roundFactor; + result0 = r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fast1 + + r0 / MULTIPLER_LONGWORD_DECIMAL; + result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; + result2 = + fast2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + + } else if (absRoundPower < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Middle word gets integer rounding; lower longword is cleared. + + final int adjustedAbsPower = absRoundPower - LONGWORD_DECIMAL_DIGITS; + + // Clear rounding portion in middle longword and add 1 at right scale (roundMultiplyFactor); + // lower longword result is 0; + + final long roundFactor = powerOfTenTable[adjustedAbsPower]; + + result0 = 0; + final long r1 = + ((fast1 / roundFactor) * roundFactor) + + roundFactor; + result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; + result2 = + fast2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + + } else { + + // High word gets integer rounding; middle and lower longwords are cleared. + + final int adjustedAbsPower = absRoundPower - TWO_X_LONGWORD_DECIMAL_DIGITS; + + // Clear rounding portion in high longword and add 1 at right scale (roundMultiplyFactor); + // middle and lower longwords result is 0; + + final long roundFactor = powerOfTenTable[adjustedAbsPower]; + + result0 = 0; + result1 = 0; + result2 = + ((fast2 / roundFactor) * roundFactor) + + roundFactor; + + } + + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + } + + private static void doClearRoundIntegerPortion( + long fast0, long fast1, long fast2, + int absRoundPower, + FastHiveDecimal fastResult) { + + long result0; + long result1; + long result2; + + if (absRoundPower < LONGWORD_DECIMAL_DIGITS) { + + // Lowest word gets integer rounding. + + // Clear rounding portion in lower longword and add 1 at right scale (roundMultiplyFactor). + + final long roundFactor = powerOfTenTable[absRoundPower]; + // final long roundMultiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - absRoundPower]; + + result0 = + ((fast0 / roundFactor) * roundFactor); + result1 = fast1; + result2 = fast2; + + } else if (absRoundPower < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Middle word gets integer rounding; lower longword is cleared. + + final int adjustedAbsPower = absRoundPower - LONGWORD_DECIMAL_DIGITS; + + // Clear rounding portion in middle longword and add 1 at right scale (roundMultiplyFactor); + // lower longword result is 0; + + final long roundFactor = powerOfTenTable[adjustedAbsPower]; + + result0 = 0; + result1 = + ((fast1 / roundFactor) * roundFactor); + result2 = fast2; + + } else { + + // High word gets integer rounding; middle and lower longwords are cleared. + + final int adjustedAbsPower = absRoundPower - TWO_X_LONGWORD_DECIMAL_DIGITS; + + // Clear rounding portion in high longword and add 1 at right scale (roundMultiplyFactor); + // middle and lower longwords result is 0; + + final long roundFactor = powerOfTenTable[adjustedAbsPower]; + + result0 = 0; + result1 = 0; + result2 = + ((fast2 / roundFactor) * roundFactor); + + } + + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + } + + /** + * Fast decimal integer part rounding ROUND_UP. + * + * ceiling(12400.8302, -2) = 12500 // E.g. Positive case FAST_ROUND_CEILING + * rr rrrr + * + * @param fastSignum + * @param fast0 + * @param fast1 + * @param fast2 + * @param fastIntegerDigitCount + * @param fastScale + * @param roundPower + * @param fastResult + */ + public static boolean fastRoundIntegerUp( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int roundPower, + FastHiveDecimal fastResult) { + + /* + * Basic algorithm: + * + * 1. Determine if rounding part is non-zero for rounding. + * 2. Scale away fractional digits if present. + * 3. If rounding, clear integer rounding portion and add 1. + * + */ + + if (roundPower >= 0) { + throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")"); + } + + final int absRoundPower = -roundPower; + if (fastIntegerDigitCount < absRoundPower) { + + // Above decimal. + return false; + } + + final int roundingPoint = absRoundPower + fastScale; + if (roundingPoint > FAST_MAX_PRECISION) { + + // Value becomes null for rounding beyond. + return false; + } + + // First, determine whether rounding is necessary based on rounding point, which is inside + // integer part. And, get rid of any fractional digits. The result scale will be 0. + // + boolean isRoundPortionAllZeroes = + isRoundPortionAllZeroes( + fast0, fast1, fast2, + roundingPoint); + + // If necessary, divide and multiply to get rid of fractional digits. + if (fastScale == 0) { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } else { + doFastScaleDown( + fast0, fast1, fast2, + /* scaleDown */ fastScale, + fastResult); + } + + // The fractional digits are gone; when rounding, clear remaining round digits and add 1. + if (!isRoundPortionAllZeroes) { + + doClearRoundIntegerPortionAndAddOne( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + absRoundPower, + fastResult); + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = fastSignum; + fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult); + fastResult.fastScale = 0; + } + + return true; + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_DOWN. + * + * The fraction being scaled away is thrown away. + * + * The signum will be updated if the result is 0, otherwise the original sign is unchanged. + */ + public static boolean fastRoundIntegerDown( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int roundPower, + FastHiveDecimal fastResult) { + + /* + * Basic algorithm: + * + * 1. Scale away fractional digits if present. + * 2. Clear integer rounding portion. + * + */ + + if (roundPower >= 0) { + throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")"); + } + + final int absRoundPower = -roundPower; + if (fastIntegerDigitCount < absRoundPower) { + + // Zero result. + fastResult.fastReset(); + return true; + } + + final int roundingPoint = absRoundPower + fastScale; + if (roundingPoint > FAST_MAX_PRECISION) { + + // Value becomes zero for rounding beyond. + fastResult.fastReset(); + return true; + } + + // If necessary, divide and multiply to get rid of fractional digits. + if (fastScale == 0) { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } else { + doFastScaleDown( + fast0, fast1, fast2, + /* scaleDown */ fastScale, + fastResult); + } + + // The fractional digits are gone; clear remaining round digits. + doClearRoundIntegerPortion( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + absRoundPower, + fastResult); + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = 0; + fastResult.fastSignum = fastSignum; + fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult); + fastResult.fastScale = 0; + } + + return true; + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP. + * + * When the fraction being scaled away is >= 0.5, the add 1. + * + * @param fastSignum + * @param fast0 + * @param fast1 + * @param fast2 + * @param scaleDown + * @param fastResult + * @return + */ + public static boolean fastRoundIntegerHalfUp( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int roundPower, + FastHiveDecimal fastResult) { + + /* + * Basic algorithm: + * + * 1. Determine if rounding digit is >= 5 for rounding. + * 2. Scale away fractional digits if present. + * 3. If rounding, clear integer rounding portion and add 1. + * + */ + + if (roundPower >= 0) { + throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")"); + } + + final int absRoundPower = -roundPower; + if (fastIntegerDigitCount < absRoundPower) { + + // Zero result. + fastResult.fastReset(); + return true; + } + + final int roundingPoint = absRoundPower + fastScale; + if (roundingPoint > FAST_MAX_PRECISION) { + + // Value becomes zero for rounding beyond. + fastResult.fastReset(); + return true; + } + + // First, determine whether rounding is necessary based on rounding point, which is inside + // integer part. And, get rid of any fractional digits. The result scale will be 0. + // + boolean isRoundPortionHalfUp = + isRoundPortionHalfUp( + fast0, fast1, fast2, + roundingPoint); + + // If necessary, divide and multiply to get rid of fractional digits. + if (fastScale == 0) { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } else { + doFastScaleDown( + fast0, fast1, fast2, + /* scaleDown */ fastScale, + fastResult); + } + + // The fractional digits are gone; when rounding, clear remaining round digits and add 1. + if (isRoundPortionHalfUp) { + + doClearRoundIntegerPortionAndAddOne( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + absRoundPower, + fastResult); + } else { + + doClearRoundIntegerPortion( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + absRoundPower, + fastResult); + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = fastSignum; + fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult); + fastResult.fastScale = 0; + } + + return true; + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_EVEN. + * + * When the fraction being scaled away is exactly 0.5, then round and add 1 only if aaa. + * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1. + * Otherwise, throw away fraction. + * + * The signum will be updated if the result is 0, otherwise the original sign is unchanged. + */ + public static boolean fastRoundIntegerHalfEven( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int roundPower, + FastHiveDecimal fastResult) { + + /* + * Basic algorithm: + * + * 1. Determine if rounding part meets banker's rounding rules for rounding. + * 2. Scale away fractional digits if present. + * 3. If rounding, clear integer rounding portion and add 1. + * + */ + + if (roundPower >= 0) { + throw new IllegalArgumentException("Expecting roundPower < 0 (roundPower " + roundPower + ")"); + } + + final int absRoundPower = -roundPower; + if (fastIntegerDigitCount < absRoundPower) { + + // Zero result. + fastResult.fastReset(); + } + + final int roundingPoint = absRoundPower + fastScale; + if (roundingPoint > FAST_MAX_PRECISION) { + + // Value becomes zero for rounding beyond. + fastResult.fastReset(); + return true; + } + + // First, determine whether rounding is necessary based on rounding point, which is inside + // integer part. And, get rid of any fractional digits. The result scale will be 0. + // + boolean isRoundPortionHalfEven = + isRoundPortionHalfEven( + fast0, fast1, fast2, + roundingPoint); + + // If necessary, divide and multiply to get rid of fractional digits. + if (fastScale == 0) { + fastResult.fast0 = fast0; + fastResult.fast1 = fast1; + fastResult.fast2 = fast2; + } else { + doFastScaleDown( + fast0, fast1, fast2, + /* scaleDown */ fastScale, + fastResult); + } + + // The fractional digits are gone; when rounding, clear remaining round digits and add 1. + if (isRoundPortionHalfEven) { + + doClearRoundIntegerPortionAndAddOne( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + absRoundPower, + fastResult); + } else { + + doClearRoundIntegerPortion( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + absRoundPower, + fastResult); + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = fastSignum; + fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult); + fastResult.fastScale = 0; + } + + return true; + } + + /** + * Fast decimal scale down by factor of 10 and do not allow rounding. + * + * When the fraction being scaled away is non-zero, return false. + * + * The signum will be updated if the result is 0, otherwise the original sign is unchanged. + */ + public static boolean fastScaleDownNoRound( + int fastSignum, long fast0, long fast1, long fast2, + int scaleDown, + FastHiveDecimal fastResult) { + if (scaleDown < 1 || scaleDown >= THREE_X_LONGWORD_DECIMAL_DIGITS - 1) { + throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 3*16 - 1 (scaleDown " + scaleDown + ")"); + } + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + if (scaleDown < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[scaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; + + final long throwAwayFraction = fast0 % divideFactor; + + if (throwAwayFraction != 0) { + return false; + } + result0 = + fast0 / divideFactor + + ((fast1 % divideFactor) * multiplyFactor); + result1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result2 = + fast2 / divideFactor; + + } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + boolean isThrowAwayFractionZero; + if (adjustedScaleDown == 0) { + isThrowAwayFractionZero = (fast0 == 0); + } else { + final long throwAwayFraction = fast1 % divideFactor; + isThrowAwayFractionZero = (throwAwayFraction == 0 && fast0 == 0); + } + + if (!isThrowAwayFractionZero) { + return false; + } + result0 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result1 = + fast2 / divideFactor; + result2 = 0; + + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + + boolean isThrowAwayFractionZero; + if (adjustedScaleDown == 0) { + isThrowAwayFractionZero = (fast0 == 0 && fast1 == 0); + } else { + final long throwAwayFraction = fast2 % divideFactor; + isThrowAwayFractionZero = (throwAwayFraction == 0 && fast0 == 0 && fast1 == 0); + } + + if (!isThrowAwayFractionZero) { + return false; + } + result0 = + fast2 / divideFactor; + result1 = 0; + result2 = 0; + } + + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastReset(); + } else { + fastResult.fastSignum = fastSignum; + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + } + + return true; + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_UP. + * + * When the fraction being scaled away is non-zero, the add 1. + * + */ + public static boolean fastRoundFractionalUp( + int fastSignum, long fast0, long fast1, long fast2, + int scaleDown, + FastHiveDecimal fastResult) { + if (scaleDown < 1 || scaleDown > FAST_MAX_SCALE) { + throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 38 (scaleDown " + scaleDown + ")"); + } + + if (scaleDown == FAST_MAX_SCALE) { + + // Examine all digits being thrown away to determine if result is 0 or 1. + if (fast0 == 0 && fast1 == 0 && fast2 == 0) { + + // Zero result. + fastResult.fastReset(); + } else { + fastResult.fastSet(fastSignum, /* fast0 */ 1, 0, 0, /* fastIntegerDigitCount */ 1, 0); + } + return true; + } + + boolean isRoundPortionAllZeroes = + isRoundPortionAllZeroes( + fast0, fast1, fast2, + scaleDown); + + doFastScaleDown( + fast0, fast1, fast2, + scaleDown, + fastResult); + + if (!isRoundPortionAllZeroes) { + final long r0 = fastResult.fast0 + 1; + fastResult.fast0 = + r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fastResult.fast1 + + r0 / MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast2 = + fastResult.fast2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = fastSignum; + } + + return (fastResult.fast2 <= MAX_HIGHWORD_DECIMAL); + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_DOWN. + * + * The fraction being scaled away is thrown away. + * + */ + public static void fastRoundFractionalDown( + int fastSignum, long fast0, long fast1, long fast2, + int scaleDown, + FastHiveDecimal fastResult) { + if (scaleDown < 1 || scaleDown > FAST_MAX_SCALE) { + throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 38 (scaleDown " + scaleDown + ")"); + } + + if (scaleDown == FAST_MAX_SCALE) { + + // Complete fractional digits shear off. Zero result. + fastResult.fastReset(); + return; + } + + doFastScaleDown( + fast0, fast1, fast2, + scaleDown, + fastResult); + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = fastSignum; + } + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP. + * + * When the fraction being scaled away is >= 0.5, the add 1. + * + * @param fastSignum + * @param fast0 + * @param fast1 + * @param fast2 + * @param scaleDown + * @param fastResult + * @return + */ + public static boolean fastRoundFractionalHalfUp( + int fastSignum, long fast0, long fast1, long fast2, + int scaleDown, + FastHiveDecimal fastResult) { + if (fastSignum == 0) { + throw new IllegalArgumentException("Unexpected zero value"); + } + if (scaleDown < 1 || scaleDown > FAST_MAX_SCALE) { + throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 38 (scaleDown " + scaleDown + ")"); + } + + if (scaleDown == FAST_MAX_SCALE) { + + // Check highest digit for rounding. + final long roundDigit = fast2 / powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1]; + if (roundDigit < 5) { + + // Zero result. + fastResult.fastReset(); + } else { + fastResult.fastSet(fastSignum, /* fast0 */ 1, 0, 0, /* fastIntegerDigitCount */ 1, 0); + } + return true; + } + + boolean isRoundPortionHalfUp = + isRoundPortionHalfUp( + fast0, fast1, fast2, + scaleDown); + + doFastScaleDown( + fast0, fast1, fast2, + scaleDown, + fastResult); + + if (isRoundPortionHalfUp) { + final long r0 = fastResult.fast0 + 1; + fastResult.fast0 = + r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fastResult.fast1 + + r0 / MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast2 = + fastResult.fast2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = fastSignum; + } + + return (fastResult.fast2 <= MAX_HIGHWORD_DECIMAL); + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP. + * + * When the fraction being scaled away is >= 0.5, the add 1. + * + */ + public static boolean fastRoundFractionalHalfUp5Words( + int fastSignum, long fast0, long fast1, long fast2, long fast3, long fast4, + int scaleDown, + FastHiveDecimal fastResult) { + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + long result3; + long result4; + if (scaleDown < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + // Divide down just before scaleDown to get round digit. + final long withRoundDigit = fast0 / powerOfTenTable[scaleDown - 1]; + final long roundDigit = withRoundDigit % 10; + + final long divideFactor = powerOfTenTable[scaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; + + if (roundDigit < 5) { + result0 = + withRoundDigit / 10 + + ((fast1 % divideFactor) * multiplyFactor); + result1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result2 = + + fast2 / divideFactor + + ((fast3 % divideFactor) * multiplyFactor); + result3 = + fast3 / divideFactor + + ((fast4 % divideFactor) * multiplyFactor); + result4 = + fast4 / divideFactor; + } else { + // Add rounding and handle carry. + final long r0 = + withRoundDigit / 10 + + ((fast1 % divideFactor) * multiplyFactor) + + 1; + result0 = r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor) + + r0 / MULTIPLER_LONGWORD_DECIMAL; + result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; + final long r2 = + fast2 / divideFactor + + + ((fast3 % divideFactor) * multiplyFactor) + + r1 / MULTIPLER_LONGWORD_DECIMAL; + result2 = r2 % MULTIPLER_LONGWORD_DECIMAL; + final long r3 = + fast3 / divideFactor + + ((fast4 % divideFactor) * multiplyFactor) + + r2 / MULTIPLER_LONGWORD_DECIMAL; + result3 = r3 % MULTIPLER_LONGWORD_DECIMAL; + result4 = + fast4 / divideFactor + + r3 % MULTIPLER_LONGWORD_DECIMAL; + } + } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; + + long roundDigit; + long fast1Scaled; + if (adjustedScaleDown == 0) { + // Grab round digit from lowest word. + roundDigit = fast0 / (MULTIPLER_LONGWORD_DECIMAL / 10); + fast1Scaled = fast1; + } else { + // Divide down just before scaleDown to get round digit. + final long withRoundDigit = fast1 / powerOfTenTable[adjustedScaleDown - 1]; + roundDigit = withRoundDigit % 10; + fast1Scaled = withRoundDigit / 10; + } + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + if (roundDigit < 5) { + result0 = + fast1Scaled + + ((fast2 % divideFactor) * multiplyFactor); + result1 = + fast2 / divideFactor + + ((fast3 % divideFactor) * multiplyFactor); + result2 = + fast3 / divideFactor + + ((fast4 % divideFactor) * multiplyFactor); + result3 = + fast4 / divideFactor; + } else { + // Add rounding and handle carry. + final long r0 = + fast1Scaled + + ((fast2 % divideFactor) * multiplyFactor) + + 1; + result0 = r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fast2 / divideFactor + + ((fast3 % divideFactor) * multiplyFactor) + + r0 / MULTIPLER_LONGWORD_DECIMAL; + result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; + final long r2 = + fast3 / divideFactor + + ((fast4 % divideFactor) * multiplyFactor) + + r1 / MULTIPLER_LONGWORD_DECIMAL; + result2 = r2 % MULTIPLER_LONGWORD_DECIMAL; + result3 = + fast4 / divideFactor + + r2 / MULTIPLER_LONGWORD_DECIMAL; + } + result4 = 0; + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS; + + long roundDigit; + long fast2Scaled; + if (adjustedScaleDown == 0) { + // Grab round digit from middle word. + roundDigit = fast1 / (MULTIPLER_LONGWORD_DECIMAL / 10); + fast2Scaled = fast2; + } else { + // Divide down just before scaleDown to get round digit. + final long withRoundDigit = fast2 / powerOfTenTable[adjustedScaleDown - 1]; + roundDigit = withRoundDigit % 10; + fast2Scaled = withRoundDigit / 10; + } + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + if (roundDigit < 5) { + result0 = + fast2Scaled + + ((fast3 % divideFactor) * multiplyFactor); + result1 = + fast3 / divideFactor + + ((fast4 % divideFactor) * multiplyFactor); + result2 = + fast4 / divideFactor; + } else { + // Add rounding. + final long r0 = + fast2Scaled + + ((fast3 % divideFactor) * multiplyFactor) + + 1; + result0 = r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fast3 / divideFactor + + ((fast4 % divideFactor) * multiplyFactor) + + r0 / MULTIPLER_LONGWORD_DECIMAL; + result1 = r1 % MULTIPLER_LONGWORD_DECIMAL; + result2 = + fast4 / divideFactor + + r1 / MULTIPLER_LONGWORD_DECIMAL; + } + result3 = 0; + result4 = 0; + } + + if (result4 != 0 || result3 != 0) { + throw new RuntimeException("Unexpected overflow into result3 or result4"); + } + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastReset(); + } + fastResult.fastSignum = fastSignum; + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + + return (result2 <= MAX_HIGHWORD_DECIMAL); + } + + /** + * Fast decimal scale down by factor of 10 with rounding ROUND_HALF_EVEN. + * + * When the fraction being scaled away is exactly 0.5, then round and add 1 only if aaa. + * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1. + * Otherwise, throw away fraction. + * + */ + public static boolean fastRoundFractionalHalfEven( + int fastSignum, long fast0, long fast1, long fast2, + int scaleDown, + FastHiveDecimal fastResult) { + if (scaleDown < 1 || scaleDown > FAST_MAX_SCALE) { + throw new IllegalArgumentException("Expecting scaleDown > 0 and scaleDown < 38 (scaleDown " + scaleDown + ")"); + } + + if (scaleDown == FAST_MAX_SCALE) { + + // Check for rounding. + final long roundDivisor = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - 1]; + final long withRoundDigit = fast2 / roundDivisor; + final long roundDigit = withRoundDigit % 10; + final long fast2Scaled = withRoundDigit / 10; + boolean shouldRound; + if (roundDigit > 5) { + shouldRound = true; + } else if (roundDigit == 5) { + boolean exactlyOneHalf = (fast2Scaled == 0 && fast1 == 0 && fast0 == 0); + if (exactlyOneHalf) { + // Round to even 0. + shouldRound = false; + } else { + shouldRound = true; + } + } else { + shouldRound = false; + } + if (!shouldRound) { + + // Zero result. + fastResult.fastReset(); + } else { + fastResult.fastSet(fastSignum, /* fast0 */ 1, 0, 0, /* fastIntegerDigitCount */ 1, 0); + } + return true; + } + + boolean isRoundPortionHalfEven = + isRoundPortionHalfEven( + fast0, fast1, fast2, + scaleDown); + + doFastScaleDown( + fast0, fast1, fast2, + scaleDown, + fastResult); + + if (isRoundPortionHalfEven) { + final long r0 = fastResult.fast0 + 1; + fastResult.fast0 = + r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + fastResult.fast1 + + r0 / MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + fastResult.fast2 = + fastResult.fast2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + } + + if (fastResult.fast0 == 0 && fastResult.fast1 == 0 && fastResult.fast2 == 0) { + fastResult.fastSignum = 0; + fastResult.fastIntegerDigitCount = 0; + fastResult.fastScale = 0; + } else { + fastResult.fastSignum = fastSignum; + } + + return (fastResult.fast2 <= MAX_HIGHWORD_DECIMAL); + } + + public static void doFastScaleDown( + FastHiveDecimal fastDec, + int scaleDown, + FastHiveDecimal fastResult) { + doFastScaleDown( + fastDec.fast0, fastDec.fast1, fastDec.fast2, + scaleDown, + fastResult); + } + + //************************************************************************************************ + // Decimal Scale Up/Down. + + /** + * Fast decimal scale down by factor of 10 with NO rounding. + * + * The signum will be updated if the result is 0, otherwise the original sign is unchanged. + */ + public static void doFastScaleDown( + long fast0, long fast1, long fast2, + int scaleDown, + FastHiveDecimal fastResult) { + + // Adjust all longs using power 10 division/remainder. + long result0; + long result1; + long result2; + if (scaleDown < LONGWORD_DECIMAL_DIGITS) { + + // Part of lowest word survives. + + final long divideFactor = powerOfTenTable[scaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleDown]; + + result0 = + fast0 / divideFactor + + ((fast1 % divideFactor) * multiplyFactor); + result1 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result2 = + fast2 / divideFactor; + + } else if (scaleDown < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Throw away lowest word. + + final int adjustedScaleDown = scaleDown - LONGWORD_DECIMAL_DIGITS; + + final long divideFactor = powerOfTenTable[adjustedScaleDown]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - adjustedScaleDown]; + + result0 = + fast1 / divideFactor + + ((fast2 % divideFactor) * multiplyFactor); + result1 = + fast2 / divideFactor; + result2 = 0; + + } else { + + // Throw away middle and lowest words. + + final int adjustedScaleDown = scaleDown - 2*LONGWORD_DECIMAL_DIGITS; + + result0 = + fast2 / powerOfTenTable[adjustedScaleDown]; + result1 = 0; + result2 = 0; + + } + + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastSignum = 0; + } + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + } + + public static boolean fastScaleUp( + FastHiveDecimal fastDec, + int scaleUp, + FastHiveDecimal fastResult) { + return + fastScaleUp( + fastDec.fast0, fastDec.fast1, fastDec.fast2, + scaleUp, + fastResult); + } + + /** + * Fast decimal scale up by factor of 10. + */ + public static boolean fastScaleUp( + long fast0, long fast1, long fast2, + int scaleUp, + FastHiveDecimal fastResult) { + if (scaleUp < 1 || scaleUp >= FAST_MAX_SCALE) { + throw new IllegalArgumentException("Expecting scaleUp > 0 and scaleUp < 38"); + } + + long result0; + long result1; + long result2; + + // Each range checks for overflow first, then moves digits. + if (scaleUp < HIGHWORD_DECIMAL_DIGITS) { + // Need to check if there are overflow digits in the high word. + + final long overflowFactor = powerOfTenTable[HIGHWORD_DECIMAL_DIGITS - scaleUp]; + if (fast2 / overflowFactor != 0) { + return false; + } + + final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - scaleUp]; + final long multiplyFactor = powerOfTenTable[scaleUp]; + + result2 = + fast2 * multiplyFactor + + fast1 / divideFactor; + result1 = + (fast1 % divideFactor) * multiplyFactor + + fast0 / divideFactor; + result0 = + (fast0 % divideFactor) * multiplyFactor; + } else if (scaleUp < HIGHWORD_DECIMAL_DIGITS + LONGWORD_DECIMAL_DIGITS) { + // High word must be zero. Check for overflow digits in middle word. + + if (fast2 != 0) { + return false; + } + + final int adjustedScaleUp = scaleUp - HIGHWORD_DECIMAL_DIGITS; + + final int middleDigits = LONGWORD_DECIMAL_DIGITS - adjustedScaleUp; + final long overflowFactor = powerOfTenTable[middleDigits]; + if (fast1 / overflowFactor != 0) { + return false; + } + + if (middleDigits < HIGHWORD_DECIMAL_DIGITS) { + // Must fill high word from both middle and lower longs. + + final int highWordMoreDigits = HIGHWORD_DECIMAL_DIGITS - middleDigits; + final long multiplyFactor = powerOfTenTable[highWordMoreDigits]; + + final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - highWordMoreDigits]; + + result2 = + fast1 * multiplyFactor + + fast0 / divideFactor; + result1 = + (fast0 % divideFactor) * multiplyFactor; + result0 = 0; + } else if (middleDigits == HIGHWORD_DECIMAL_DIGITS) { + // Fill high long from middle long, and middle long from lower long. + + result2 = fast1; + result1 = fast0; + result0 = 0; + } else { + // Fill high long from some of middle long. + + final int keepMiddleDigits = middleDigits - HIGHWORD_DECIMAL_DIGITS; + final long divideFactor = powerOfTenTable[keepMiddleDigits]; + final long multiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - keepMiddleDigits]; + + result2 = + fast1 / divideFactor; + result1 = + (fast1 % divideFactor) * multiplyFactor + + fast0 / divideFactor; + result0 = + (fast0 % divideFactor) * multiplyFactor; + } + } else { + // High and middle word must be zero. Check for overflow digits in lower word. + + if (fast2 != 0 || fast1 != 0) { + return false; + } + + final int adjustedScaleUp = scaleUp - HIGHWORD_DECIMAL_DIGITS - LONGWORD_DECIMAL_DIGITS; + + final int lowerDigits = LONGWORD_DECIMAL_DIGITS - adjustedScaleUp; + final long overflowFactor = powerOfTenTable[lowerDigits]; + if (fast0 / overflowFactor != 0) { + return false; + } + + if (lowerDigits < HIGHWORD_DECIMAL_DIGITS) { + // Must fill high word from both middle and lower longs. + + final int highWordMoreDigits = HIGHWORD_DECIMAL_DIGITS - lowerDigits; + final long multiplyFactor = powerOfTenTable[highWordMoreDigits]; + + final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - highWordMoreDigits]; + + result2 = + fast0 * multiplyFactor; + result1 = 0; + result0 = 0; + } else if (lowerDigits == HIGHWORD_DECIMAL_DIGITS) { + // Fill high long from lower long. + + result2 = fast0; + result1 = 0; + result0 = 0; + } else { + // Fill high long and middle from some of lower long. + + final int keepLowerDigits = lowerDigits - HIGHWORD_DECIMAL_DIGITS; + final long keepLowerDivideFactor = powerOfTenTable[keepLowerDigits]; + final long keepLowerMultiplyFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - keepLowerDigits]; + + result2 = + fast0 / keepLowerDivideFactor; + result1 = + (fast0 % keepLowerDivideFactor) * keepLowerMultiplyFactor; + result0 = 0; + } + } + + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastSignum = 0; + } + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + + return true; + } + + //************************************************************************************************ + // Decimal Precision / Trailing Zeroes. + + public static int fastLongWordTrailingZeroCount( + long longWord) { + + if (longWord == 0) { + return LONGWORD_DECIMAL_DIGITS; + } + + long factor = 10; + for (int i = 0; i < LONGWORD_DECIMAL_DIGITS; i++) { + if (longWord % factor != 0) { + return i; + } + factor *= 10; + } + return 0; + } + + public static int fastHighWordTrailingZeroCount( + long longWord) { + + if (longWord == 0) { + return HIGHWORD_DECIMAL_DIGITS; + } + + long factor = 10; + for (int i = 0; i < HIGHWORD_DECIMAL_DIGITS; i++) { + if (longWord % factor != 0) { + return i; + } + factor *= 10; + } + return 0; + } + + public static int fastLongWordPrecision( + long longWord) { + + if (longWord == 0) { + return 0; + } + if (longWord > 99999999L) { + if (longWord > 999999999999L) { + if (longWord > 99999999999999L) { + if (longWord > 999999999999999L) { + return 16; + } else { + return 15; + } + } else { + if (longWord > 9999999999999L) { + return 14; + } else { + return 13; + } + } + } else { + if (longWord > 9999999999L) { + if (longWord > 99999999999L) { + return 12; + } else { + return 11; + } + } else { + if (longWord > 999999999L) { + return 10; + } else { + return 9; + } + } + } + } else { + if (longWord > 9999L) { + if (longWord > 999999L) { + if (longWord > 9999999L) { + return 8; + } else { + return 7; + } + } else { + if (longWord > 99999L) { + return 6; + } else { + return 5; + } + } + } else { + if (longWord > 99L) { + if (longWord > 999L) { + return 4; + } else { + return 3; + } + } else { + if (longWord > 9L) { + return 2; + } else { + return 1; + } + } + } + } + } + + public static int fastHighWordPrecision( + long longWord) { + + if (longWord == 0) { + return 0; + } + // 6 highword digits. + if (longWord > 999L) { + if (longWord > 9999L) { + if (longWord > 99999L) { + return 6; + } else { + return 5; + } + } else { + return 4; + } + } else { + if (longWord > 99L) { + return 3; + } else { + if (longWord > 9L) { + return 2; + } else { + return 1; + } + } + } + } + + public static int fastSqlPrecision( + FastHiveDecimal fastDec) { + return + fastSqlPrecision( + fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2, + fastDec.fastIntegerDigitCount, fastDec.fastScale); + } + + public static int fastSqlPrecision( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + + if (fastSignum == 0) { + return 1; + } + + final int rawPrecision = + fastRawPrecision(fastSignum, fast0, fast1, fast2); + + if (rawPrecision < fastScale) { + // This can happen for numbers less than 0.1 + // For 0.001234: rawPrecision=4, scale=6 + // In this case, we'll set the type to have the same precision as the scale. + return fastScale; + } + return rawPrecision; + } + + public static int fastRawPrecision( + FastHiveDecimal fastDec) { + return + fastRawPrecision( + fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2); + } + + public static int fastRawPrecision( + int fastSignum, long fast0, long fast1, long fast2) { + + if (fastSignum == 0) { + return 0; + } + + int precision; + + if (fast2 != 0) { + + // 6 highword digits. + precision = TWO_X_LONGWORD_DECIMAL_DIGITS + fastHighWordPrecision(fast2); + + } else if (fast1 != 0) { + + // Check fast1. + precision = LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fast1); + + + } else { + + // Check fast0. + precision = fastLongWordPrecision(fast0); + + } + + return precision; + } + + // Determine if all digits below a power is zero. + // The lowest digit is power = 0. + public static boolean isAllZeroesBelow( + int fastSignum, long fast0, long fast1, long fast2, + int power) { + if (power < 0 || power > FAST_MAX_SCALE) { + throw new IllegalArgumentException("Expecting power >= 0 and power <= 38"); + } + + if (fastSignum == 0) { + return true; + } + + if (power >= TWO_X_LONGWORD_DECIMAL_DIGITS) { + if (fast0 != 0 || fast1 != 0) { + return false; + } + final int adjustedPower = power - TWO_X_LONGWORD_DECIMAL_DIGITS; + if (adjustedPower == 0) { + return true; + } + long remainder = fast2 % powerOfTenTable[adjustedPower]; + return (remainder == 0); + } else if (power >= LONGWORD_DECIMAL_DIGITS) { + if (fast0 != 0) { + return false; + } + final int adjustedPower = power - LONGWORD_DECIMAL_DIGITS; + if (adjustedPower == 0) { + return true; + } + long remainder = fast1 % powerOfTenTable[adjustedPower]; + return (remainder == 0); + } else { + if (power == 0) { + return true; + } + long remainder = fast0 % powerOfTenTable[power]; + return (remainder == 0); + } + } + + public static boolean fastExceedsPrecision( + long fast0, long fast1, long fast2, + int precision) { + + if (precision <= 0) { + return true; + } else if (precision >= FAST_MAX_PRECISION) { + return false; + } + final int precisionLessOne = precision - 1; + + // 0 (lowest), 1 (middle), or 2 (high). + int wordNum = precisionLessOne / LONGWORD_DECIMAL_DIGITS; + + int digitInWord = precisionLessOne % LONGWORD_DECIMAL_DIGITS; + + final long overLimitInWord = powerOfTenTable[digitInWord + 1] - 1; + + if (wordNum == 0) { + if (digitInWord < LONGWORD_DECIMAL_DIGITS - 1) { + if (fast0 > overLimitInWord) { + return true; + } + } + return (fast1 != 0 || fast2 != 0); + } else if (wordNum == 1) { + if (digitInWord < LONGWORD_DECIMAL_DIGITS - 1) { + if (fast1 > overLimitInWord) { + return true; + } + } + return (fast2 != 0); + } else { + // We've eliminated the highest digit already with FAST_MAX_PRECISION check above. + return (fast2 > overLimitInWord); + } + } + + public static int fastTrailingDecimalZeroCount( + long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + if (fastScale < 0 || fastScale > FAST_MAX_SCALE) { + throw new IllegalArgumentException("Expecting scale >= 0 and scale <= 38"); + } + if (fastScale == 0) { + return 0; + } + + final int lowerLongwordDigits = Math.min(fastScale, LONGWORD_DECIMAL_DIGITS); + if (lowerLongwordDigits < LONGWORD_DECIMAL_DIGITS || fast0 != 0) { + long factor = 10; + for (int i = 0; i < lowerLongwordDigits; i++) { + if (fast0 % factor != 0) { + return i; + } + factor *= 10; + } + if (lowerLongwordDigits < LONGWORD_DECIMAL_DIGITS) { + return fastScale; + } + } + if (fastScale == LONGWORD_DECIMAL_DIGITS) { + return fastScale; + } + final int middleLongwordDigits = Math.min(fastScale - LONGWORD_DECIMAL_DIGITS, LONGWORD_DECIMAL_DIGITS); + if (middleLongwordDigits < LONGWORD_DECIMAL_DIGITS || fast1 != 0) { + long factor = 10; + for (int i = 0; i < middleLongwordDigits; i++) { + if (fast1 % factor != 0) { + return LONGWORD_DECIMAL_DIGITS + i; + } + factor *= 10; + } + if (middleLongwordDigits < LONGWORD_DECIMAL_DIGITS) { + return fastScale; + } + } + if (fastScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { + return fastScale; + } + final int highLongwordDigits = fastScale - TWO_X_LONGWORD_DECIMAL_DIGITS; + if (highLongwordDigits < HIGHWORD_DECIMAL_DIGITS || fast2 != 0) { + long factor = 10; + for (int i = 0; i < highLongwordDigits; i++) { + if (fast2 % factor != 0) { + return TWO_X_LONGWORD_DECIMAL_DIGITS + i; + } + factor *= 10; + } + } + return fastScale; + } + + public static FastCheckPrecisionScaleStatus fastCheckPrecisionScale( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int maxPrecision, int maxScale) { + + if (fastSignum == 0) { + return FastCheckPrecisionScaleStatus.NO_CHANGE; + } + final int maxIntegerDigitCount = maxPrecision - maxScale; + if (fastIntegerDigitCount > maxIntegerDigitCount) { + return FastCheckPrecisionScaleStatus.OVERFLOW; + } + if (fastScale > maxScale) { + return FastCheckPrecisionScaleStatus.UPDATE_SCALE_DOWN; + } + return FastCheckPrecisionScaleStatus.NO_CHANGE; + } + + public static boolean fastUpdatePrecisionScale( + final int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int maxPrecision, int maxScale, FastCheckPrecisionScaleStatus status, + FastHiveDecimal fastResult) { + + switch (status) { + case UPDATE_SCALE_DOWN: + { + fastResult.fastSignum = fastSignum; + + // Throw away lower digits. + if (!fastRoundFractionalHalfUp( + fastSignum, fast0, fast1, fast2, + fastScale - maxScale, + fastResult)) { + return false; + } + + fastResult.fastScale = maxScale; + + // CONSIDER: For now, recompute integerDigitCount... + fastResult.fastIntegerDigitCount = + Math.max(0, fastRawPrecision(fastResult) - fastResult.fastScale); + + // And, round up may cause us to exceed our precision/scale... + final int maxIntegerDigitCount = maxPrecision - maxScale; + if (fastResult.fastIntegerDigitCount > maxIntegerDigitCount) { + return false; + } + + // Scaling down may have opened up trailing zeroes... + final int trailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (trailingZeroCount > 0) { + // Scale down again. + doFastScaleDown( + fastResult, + trailingZeroCount, + fastResult); + fastResult.fastScale -= trailingZeroCount; + } + } + break; + default: + throw new RuntimeException("Unexpected fast check precision scale status " + status); + } + + /* + if (!fastResult.fastIsValid()) { + fastResult.fastRaiseInvalidException(); + } + */ + return true; + } + + //************************************************************************************************ + // Decimal Addition / Subtraction. + + public static boolean doAddSameScaleSameSign( + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + doAddSameScaleSameSign( + /* resultSignum */ fastLeft.fastSignum, + fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastResult); + } + + public static boolean doAddSameScaleSameSign( + int resultSignum, + long left0, long left1, long left2, + long right0, long right1, long right2, + FastHiveDecimal fastResult) { + + long result0; + long result1; + long result2; + + final long r0 = left0 + right0; + result0 = + r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + left1 + + right1 + + r0 / MULTIPLER_LONGWORD_DECIMAL; + result1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + result2 = + left2 + + right2 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastReset(); + } else { + fastResult.fastSignum = resultSignum; + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + } + + return (result2 <= MAX_HIGHWORD_DECIMAL); + } + + public static boolean doSubtractSameScaleNoUnderflow( + int resultSignum, + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + doSubtractSameScaleNoUnderflow( + resultSignum, + fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastResult); + } + + public static boolean doSubtractSameScaleNoUnderflow( + int resultSignum, + long left0, long left1, long left2, + long right0, long right1, long right2, + FastHiveDecimal fastResult) { + + long result0; + long result1; + long result2; + + final long r0 = left0 - right0; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = left1 - right1 - 1; + } else { + result0 = r0; + r1 = left1 - right1; + } + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + result2 = left2 - right2 - 1; + } else { + result1 = r1; + result2 = left2 - right2; + } + if (result2 < 0) { + return false; + } + + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastReset(); + } else { + fastResult.fastSignum = resultSignum; + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + } + + return true; + } + + public static boolean doSubtractSameScaleNoUnderflow( + long left0, long left1, long left2, + long right0, long right1, long right2, + long[] result) { + + long result0; + long result1; + long result2; + + final long r0 = left0 - right0; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = left1 - right1 - 1; + } else { + result0 = r0; + r1 = left1 - right1; + } + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + result2 = left2 - right2 - 1; + } else { + result1 = r1; + result2 = left2 - right2; + } + if (result2 < 0) { + return false; + } + + result[0] = result0; + result[1] = result1; + result[2] = result2; + + return true; + } + + private static boolean doAddSameScale( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int scale, + FastHiveDecimal fastResult) { + + if (leftSignum == rightSignum) { + if (!doAddSameScaleSameSign( + /* resultSignum */ leftSignum, + leftFast0, leftFast1, leftFast2, + rightFast0, rightFast1, rightFast2, + fastResult)) { + // Handle overflow precision issue. + if (scale > 0) { + if (!fastRoundFractionalHalfUp( + fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, + 1, + fastResult)) { + return false; + } + scale--; + } else { + // Overflow. + return false; + } + } + fastResult.fastScale = scale; + } else { + // Just compare the magnitudes (i.e. signums set to 1). + int compareTo = + fastCompareTo( + 1, + leftFast0, leftFast1, leftFast2, 0, + 1, + rightFast0, rightFast1, rightFast2, 0); + if (compareTo == 0) { + // They cancel each other. + fastResult.fastReset(); + return true; + } + + if (compareTo == 1) { + if (!doSubtractSameScaleNoUnderflow( + /* resultSignum */ leftSignum, + leftFast0, leftFast1, leftFast2, + rightFast0, rightFast1, rightFast2, + fastResult)) { + throw new RuntimeException("Unexpected underflow"); + } + } else { + if (!doSubtractSameScaleNoUnderflow( + /* resultSignum */ rightSignum, + rightFast0, rightFast1, rightFast2, + leftFast0, leftFast1, leftFast2, + fastResult)) { + throw new RuntimeException("Unexpected underflow"); + } + } + fastResult.fastScale = scale; + } + + if (fastResult.fastSignum != 0) { + final int precision = fastRawPrecision(fastResult); + fastResult.fastIntegerDigitCount = Math.max(0, precision - fastResult.fastScale); + } + + final int resultTrailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (resultTrailingZeroCount > 0) { + doFastScaleDown( + fastResult, + resultTrailingZeroCount, + fastResult); + if (fastResult.fastSignum == 0) { + fastResult.fastScale = 0; + } else { + fastResult.fastScale -= resultTrailingZeroCount; + } + } + + return true; + } + + private static boolean doFinishAddSubtractDifferentScale( + long result0, long result1, long result2, long result3, long result4, + int resultScale, + FastHiveDecimal fastResult) { + + int precision; + if (result4 != 0) { + precision = FOUR_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result4); + } else if (result3 != 0) { + precision = THREE_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result3); + } else if (result2 != 0) { + precision = TWO_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result2); + } else if (result1 != 0) { + precision = LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(result1); + } else { + precision = fastLongWordPrecision(result0); + } + + if (precision > FAST_MAX_PRECISION){ + final int scaleDown = precision - FAST_MAX_PRECISION; + + resultScale -= scaleDown; + if (resultScale < 0) { + // No room. + return false; + } + if (!fastRoundFractionalHalfUp5Words( + 1, result0, result1, result2, result3, result4, + scaleDown, + fastResult)) { + // Handle overflow precision issue. + if (resultScale > 0) { + if (!fastRoundFractionalHalfUp( + fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, + 1, + fastResult)) { + throw new RuntimeException("Unexpected overflow"); + } + if (fastResult.fastSignum == 0) { + return true; + } + resultScale--; + } else { + return false; + } + } + + precision = fastRawPrecision(1, fastResult.fast0, fastResult.fast1, fastResult.fast2); + + // Stick back into result variables... + result0 = fastResult.fast0; + result1 = fastResult.fast1; + result2 = fastResult.fast2; + } + + // Caller will set signum. + fastResult.fastSignum = 1; + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + fastResult.fastIntegerDigitCount = Math.max(0, precision - resultScale); + fastResult.fastScale = resultScale; + + final int resultTrailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (resultTrailingZeroCount > 0) { + doFastScaleDown( + fastResult, + resultTrailingZeroCount, + fastResult); + if (fastResult.fastSignum == 0) { + fastResult.fastScale = 0; + } else { + fastResult.fastScale -= resultTrailingZeroCount; + } + } + + return true; + } + + private static boolean fastSubtractDifferentScale( + long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + int diffScale; + int resultScale; + + long result0 = 0; + long result1 = 0; + long result2 = 0; + long result3 = 0; + long result4 = 0; + + // Since subtraction is not commutative, we can must subtract in the order passed in. + if (leftScale > rightScale) { + + // Since left has a longer digit tail and it doesn't move; we will shift the right digits + // as we do our addition and them into the result. + + diffScale = leftScale - rightScale; + resultScale = leftScale; + + if (diffScale < LONGWORD_DECIMAL_DIGITS) { + + final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale]; + + final long r0 = + leftFast0 + - (rightFast0 % divideFactor) * multiplyFactor; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = + leftFast1 + - rightFast0 / divideFactor + - (rightFast1 % divideFactor) * multiplyFactor + - 1; + } else { + result0 = r0; + r1 = + leftFast1 + - rightFast0 / divideFactor + - (rightFast1 % divideFactor) * multiplyFactor; + } + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + leftFast2 + - rightFast1 / divideFactor + - (rightFast2 % divideFactor) * multiplyFactor + - 1; + } else { + result1 = r1; + r2 = + leftFast2 + - rightFast1 / divideFactor + - (rightFast2 % divideFactor) * multiplyFactor; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + -(rightFast2 / divideFactor) + - 1; + } else { + result2 = r2; + r3 = + -(rightFast2 / divideFactor); + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = - 1; + } else { + result3 = r3; + r4 = 0; + } + if (r4 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + + } else if (diffScale == LONGWORD_DECIMAL_DIGITS){ + + result0 = leftFast0; + final long r1 = + leftFast1 + - rightFast0; + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + leftFast2 + - rightFast1 + - 1; + } else { + result1 = r1; + r2 = + leftFast2 + - rightFast1; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + -rightFast2 + - 1; + } else { + result2 = r2; + r3 = + -rightFast2; + } + if (r3 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + + + } else if (diffScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + final long divideFactor = powerOfTenTable[TWO_X_LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale - LONGWORD_DECIMAL_DIGITS]; + + result0 = leftFast0; + final long r1 = + leftFast1 + -(rightFast0 % divideFactor) * multiplyFactor; + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + leftFast2 + - rightFast0 / divideFactor + - (rightFast1 % divideFactor) * multiplyFactor + - 1; + } else { + result1 = r1; + r2 = + leftFast2 + - rightFast0 / divideFactor + - (rightFast1 % divideFactor) * multiplyFactor; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + - rightFast1 / divideFactor + - (rightFast2 % divideFactor) * multiplyFactor + - 1; + } else { + result2 = r2; + r3 = + - rightFast1 / divideFactor + - (rightFast2 % divideFactor) * multiplyFactor; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = + - rightFast2 / divideFactor + - 1; + } else { + result3 = r3; + r4 = + - rightFast2 / divideFactor; + } + long r5; + if (r4 < 0) { + result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; + r5 = - 1; + } else { + result4 = r4; + r5 = 0; + } + if (r5 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + + } else if (diffScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { + + result0 = leftFast0; + result1 = leftFast1; + final long r2 = + leftFast2 + - rightFast0; + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + - rightFast1 + - 1; + } else { + result2 = r2; + r3 = + - rightFast1; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = + -rightFast2 + - 1; + } else { + result3 = r3; + r4 = + -rightFast2; + } + long r5; + if (r4 < 0) { + result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; + r5 = - 1; + } else { + result4 = r4; + r5 = 0; + } + if (r5 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + + } else { + + final long divideFactor = powerOfTenTable[THREE_X_LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale - TWO_X_LONGWORD_DECIMAL_DIGITS]; + + result0 = leftFast0; + result1 = leftFast1; + final long r2 = + leftFast2 + - (rightFast0 % divideFactor) * multiplyFactor; + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + - (rightFast0 / divideFactor) + - (rightFast1 % divideFactor) * multiplyFactor + - 1; + } else { + result2 = r2; + r3 = + - (rightFast0 / divideFactor) + - (rightFast1 % divideFactor) * multiplyFactor; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = + - (rightFast1 / divideFactor) + - (rightFast2 % divideFactor) * multiplyFactor + - 1; + } else { + result3 = r3; + r4 = + - (rightFast1 / divideFactor) + - (rightFast2 % divideFactor) * multiplyFactor; + } + long r5; + if (r4 < 0) { + result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; + r5 = + - (rightFast2 / divideFactor) + - 1; + } else { + result4 = r4; + r5 = + - (rightFast2 / divideFactor); + } + if (r5 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + } + } else { + + // Since right has a longer digit tail and it doesn't move; we will shift the left digits + // as we do our addition and them into the result. + + diffScale = rightScale - leftScale; + resultScale = rightScale; + + if (diffScale < LONGWORD_DECIMAL_DIGITS) { + + final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale]; + + final long r0 = + (leftFast0 % divideFactor) * multiplyFactor + - rightFast0; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = + leftFast0 / divideFactor + + (leftFast1 % divideFactor) * multiplyFactor + - rightFast1 + - 1; + } else { + result0 = r0; + r1 = + leftFast0 / divideFactor + + (leftFast1 % divideFactor) * multiplyFactor + - rightFast1; + } + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + leftFast1 / divideFactor + + (leftFast2 % divideFactor) * multiplyFactor + - rightFast2 + - 1; + } else { + result1 = r1; + r2 = + leftFast1 / divideFactor + + (leftFast2 % divideFactor) * multiplyFactor + - rightFast2; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + leftFast2 / divideFactor + - 1; + } else { + result2 = r2; + r3 = + leftFast2 / divideFactor; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = - 1; + } else { + result3 = r3; + r4 = 0; + } + if (r4 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + + } else if (diffScale == LONGWORD_DECIMAL_DIGITS){ + + final long r0 = + - rightFast0; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = + leftFast0 + - rightFast1 + - 1; + } else { + result0 = r0; + r1 = + leftFast0 + - rightFast1; + } + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + leftFast1 + - rightFast2 + - 1; + } else { + result1 = r1; + r2 = + leftFast1 + - rightFast2; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + leftFast2 + - 1; + } else { + result2 = r2; + r3 = + leftFast2; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = - 1; + } else { + result3 = r3; + r4 = 0; + } + if (r4 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + + } else if (diffScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + final long divideFactor = powerOfTenTable[TWO_X_LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale - LONGWORD_DECIMAL_DIGITS]; + + final long r0 = + - rightFast0; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = + (leftFast0 % divideFactor) * multiplyFactor + - rightFast1 + - 1; + } else { + result0 = r0; + r1 = + (leftFast0 % divideFactor) * multiplyFactor + - rightFast1; + } + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + leftFast0 / divideFactor + + (leftFast1 % divideFactor) * multiplyFactor + - rightFast2 + - 1; + } else { + result1 = r1; + r2 = + leftFast0 / divideFactor + + (leftFast1 % divideFactor) * multiplyFactor + - rightFast2; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + leftFast1 / divideFactor + + (leftFast2 % divideFactor) * multiplyFactor + - 1; + } else { + result2 = r2; + r3 = + leftFast1 / divideFactor + + (leftFast2 % divideFactor) * multiplyFactor; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = + leftFast2 / divideFactor + - 1; + } else { + result3 = r3; + r4 = + leftFast2 / divideFactor; + } + if (r4 < 0) { + result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; + } else { + result4 = r4; + } + + } else if (diffScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { + + final long r0 = + - rightFast0; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = + - rightFast1 + - 1; + } else { + result0 = r0; + r1 = + - rightFast1; + } + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + leftFast0 + - rightFast2 + - 1; + } else { + result1 = r1; + r2 = + leftFast0 + - rightFast2; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + leftFast1 + - 1; + } else { + result2 = r2; + r3 = + leftFast1; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = + leftFast2 + - 1; + } else { + result3 = r3; + r4 = + leftFast2; + } + long r5; + if (r4 < 0) { + result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; + r5 = - 1; + } else { + result4 = r4; + r5 = 0; + } + if (r5 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + + } else { + + final long divideFactor = powerOfTenTable[THREE_X_LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale - TWO_X_LONGWORD_DECIMAL_DIGITS]; + + final long r0 = + - rightFast0; + long r1; + if (r0 < 0) { + result0 = r0 + MULTIPLER_LONGWORD_DECIMAL; + r1 = + - rightFast1 + - 1; + } else { + result0 = r0; + r1 = + - rightFast1; + } + long r2; + if (r1 < 0) { + result1 = r1 + MULTIPLER_LONGWORD_DECIMAL; + r2 = + (leftFast0 % divideFactor) * multiplyFactor + - rightFast2 + - 1; + } else { + result1 = r1; + r2 = + (leftFast0 % divideFactor) * multiplyFactor + - rightFast2; + } + long r3; + if (r2 < 0) { + result2 = r2 + MULTIPLER_LONGWORD_DECIMAL; + r3 = + leftFast0 / divideFactor + + (leftFast1 % divideFactor) * multiplyFactor + - 1; + } else { + result2 = r2; + r3 = + leftFast0 / divideFactor + + (leftFast1 % divideFactor) * multiplyFactor; + } + long r4; + if (r3 < 0) { + result3 = r3 + MULTIPLER_LONGWORD_DECIMAL; + r4 = + leftFast1 / divideFactor + + (leftFast2 % divideFactor) * multiplyFactor + - 1; + } else { + result3 = r3; + r4 = + leftFast1 / divideFactor + + (leftFast2 % divideFactor) * multiplyFactor; + } + long r5; + if (r4 < 0) { + result4 = r4 + MULTIPLER_LONGWORD_DECIMAL; + r5 = + leftFast2 / divideFactor + - 1; + } else { + result4 = r4; + r5 = + leftFast2 / divideFactor; + } + if (r5 != 0) { + throw new RuntimeException("Unexpected underflow"); + } + } + } + + return + doFinishAddSubtractDifferentScale( + result0, result1, result2, result3, result4, + resultScale, + fastResult); + } + + private static boolean fastAddDifferentScale( + long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + // Arrange so result* has a longer digit tail and it lines up; we will shift the shift* digits + // as we do our addition and them into the result. + long result0; + long result1; + long result2; + + long shift0; + long shift1; + long shift2; + + int diffScale; + int resultScale; + + // Since addition is commutative, we can add in any order. + if (leftScale > rightScale) { + + result0 = leftFast0; + result1 = leftFast1; + result2 = leftFast2; + + shift0 = rightFast0; + shift1 = rightFast1; + shift2 = rightFast2; + + diffScale = leftScale - rightScale; + resultScale = leftScale; + } else { + + result0 = rightFast0; + result1 = rightFast1; + result2 = rightFast2; + + shift0 = leftFast0; + shift1 = leftFast1; + shift2 = leftFast2; + + diffScale = rightScale - leftScale; + resultScale = rightScale; + } + + long result3 = 0; + long result4 = 0; + + if (diffScale < LONGWORD_DECIMAL_DIGITS) { + + final long divideFactor = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale]; + + final long r0 = + result0 + + (shift0 % divideFactor) * multiplyFactor; + result0 = + r0 % MULTIPLER_LONGWORD_DECIMAL; + final long r1 = + result1 + + shift0 / divideFactor + + (shift1 % divideFactor) * multiplyFactor + + r0 / MULTIPLER_LONGWORD_DECIMAL; + result1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + final long r2 = + result2 + + shift1 / divideFactor + + (shift2 % divideFactor) * multiplyFactor + + r1 / MULTIPLER_LONGWORD_DECIMAL; + result2 = + r2 % MULTIPLER_LONGWORD_DECIMAL; + final long r3 = + shift2 / divideFactor + + r2 / MULTIPLER_LONGWORD_DECIMAL; + result3 = + r3 % MULTIPLER_LONGWORD_DECIMAL; + + } else if (diffScale == LONGWORD_DECIMAL_DIGITS){ + + final long r1 = + result1 + + shift0; + result1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + final long r2 = + result2 + + shift1 + + r1 / MULTIPLER_LONGWORD_DECIMAL; + result2 = + r2 % MULTIPLER_LONGWORD_DECIMAL; + final long r3 = + shift2 + + r2 / MULTIPLER_LONGWORD_DECIMAL; + result3 = + r3 % MULTIPLER_LONGWORD_DECIMAL; + result4 = r3 / MULTIPLER_LONGWORD_DECIMAL; + + } else if (diffScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + + final long divideFactor = powerOfTenTable[TWO_X_LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale - LONGWORD_DECIMAL_DIGITS]; + + final long r1 = + result1 + + (shift0 % divideFactor) * multiplyFactor; + result1 = + r1 % MULTIPLER_LONGWORD_DECIMAL; + final long r2 = + result2 + + shift0 / divideFactor + + (shift1 % divideFactor) * multiplyFactor + + r1 / MULTIPLER_LONGWORD_DECIMAL; + result2 = + r2 % MULTIPLER_LONGWORD_DECIMAL; + final long r3 = + shift1 / divideFactor + + (shift2 % divideFactor) * multiplyFactor + + r2 / MULTIPLER_LONGWORD_DECIMAL; + result3 = + r3 % MULTIPLER_LONGWORD_DECIMAL; + final long r4 = + shift2 / divideFactor + + r3 / MULTIPLER_LONGWORD_DECIMAL; + result4 = + r4 % MULTIPLER_LONGWORD_DECIMAL; + + } else if (diffScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { + + final long r2 = + result2 + + shift0; + result2 = + r2 % MULTIPLER_LONGWORD_DECIMAL; + final long r3 = + shift1 + + r2 / MULTIPLER_LONGWORD_DECIMAL; + result3 = + r3 % MULTIPLER_LONGWORD_DECIMAL; + final long r4 = + shift2 + + r3 / MULTIPLER_LONGWORD_DECIMAL; + result4 = + r4 % MULTIPLER_LONGWORD_DECIMAL; + + } else { + + final long divideFactor = powerOfTenTable[THREE_X_LONGWORD_DECIMAL_DIGITS - diffScale]; + final long multiplyFactor = powerOfTenTable[diffScale - TWO_X_LONGWORD_DECIMAL_DIGITS]; + + final long r2 = + result2 + + (shift0 % divideFactor) * multiplyFactor; + result2 = + r2 % MULTIPLER_LONGWORD_DECIMAL; + final long r3 = + shift0 / divideFactor + + (shift1 % divideFactor) * multiplyFactor + + r2 / MULTIPLER_LONGWORD_DECIMAL; + result3 = + r3 % MULTIPLER_LONGWORD_DECIMAL; + final long r4 = + shift1 / divideFactor + + (shift2 % divideFactor) * multiplyFactor + + r3 / MULTIPLER_LONGWORD_DECIMAL; + result4 = + r4 % MULTIPLER_LONGWORD_DECIMAL; + if (shift2 / divideFactor != 0) { + throw new RuntimeException("Unexpected overflow"); + } + + } + + return + doFinishAddSubtractDifferentScale( + result0, result1, result2, result3, result4, + resultScale, + fastResult); + } + + private static boolean doAddDifferentScale( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + if (leftSignum == rightSignum) { + if (!fastAddDifferentScale( + leftFast0, leftFast1, leftFast2, + leftIntegerDigitCount, leftScale, + rightFast0, rightFast1, rightFast2, + rightIntegerDigitCount, rightScale, + fastResult)) { + return false; + } + // Sign stays the same. + fastResult.fastSignum = leftSignum; + } else { + + // Just compare the magnitudes (i.e. signums set to 1). + int compareTo = + fastCompareTo( + 1, + leftFast0, leftFast1, leftFast2, leftScale, + 1, + rightFast0, rightFast1, rightFast2, rightScale); + if (compareTo == 0) { + // They cancel each other. + fastResult.fastSignum = 0; + fastResult.fast0 = 0; + fastResult.fast1 = 0; + fastResult.fast2 = 0; + fastResult.fastScale = 0; + return true; + } + + if (compareTo == 1) { + if (!fastSubtractDifferentScale( + leftFast0, leftFast1, leftFast2, + leftIntegerDigitCount, leftScale, + rightFast0, rightFast1, rightFast2, + rightIntegerDigitCount, rightScale, + fastResult)) { + throw new RuntimeException("Unexpected overflow"); + } + fastResult.fastSignum = leftSignum; + } else { + if (!fastSubtractDifferentScale( + rightFast0, rightFast1, rightFast2, + rightIntegerDigitCount, rightScale, + leftFast0, leftFast1, leftFast2, + leftIntegerDigitCount, leftScale, + fastResult)) { + throw new RuntimeException("Unexpected overflow"); + } + fastResult.fastSignum = rightSignum; + } + } + + final int resultTrailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (resultTrailingZeroCount > 0) { + doFastScaleDown( + fastResult, + resultTrailingZeroCount, + fastResult); + if (fastResult.fastSignum == 0) { + fastResult.fastScale = 0; + } else { + fastResult.fastScale -= resultTrailingZeroCount; + } + } + + return true; + } + + public static boolean fastAdd( + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return fastAdd( + fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastLeft.fastIntegerDigitCount, fastLeft.fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + public static boolean fastAdd( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + if (rightSignum == 0) { + fastResult.fastSet(leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale); + return true; + } + if (leftSignum == 0) { + fastResult.fastSet(rightSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale); + return true; + } + + if (leftScale == rightScale) { + return doAddSameScale( + leftSignum, leftFast0, leftFast1, leftFast2, + rightSignum, rightFast0, rightFast1, rightFast2, + leftScale, + fastResult); + } else { + return doAddDifferentScale( + leftSignum, leftFast0, leftFast1, leftFast2, + leftIntegerDigitCount, leftScale, + rightSignum, rightFast0, rightFast1, rightFast2, + rightIntegerDigitCount, rightScale, + fastResult); + } + } + + public static boolean fastSubtract( + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return fastSubtract( + fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastLeft.fastIntegerDigitCount, fastLeft.fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + public static boolean fastSubtract( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + if (rightSignum == 0) { + fastResult.fastSet(leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale); + return true; + } + final int flippedDecSignum = (rightSignum == 1 ? -1 : 1); + if (leftSignum == 0) { + fastResult.fastSet(flippedDecSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale); + return true; + } + + if (leftScale == rightScale) { + return doAddSameScale( + leftSignum, leftFast0, leftFast1, leftFast2, + flippedDecSignum, rightFast0, rightFast1, rightFast2, + leftScale, + fastResult); + } else { + return doAddDifferentScale( + leftSignum, leftFast0, leftFast1, leftFast2, + leftIntegerDigitCount, leftScale, + flippedDecSignum, rightFast0, rightFast1, rightFast2, + rightIntegerDigitCount, rightScale, + fastResult); + } + } + + //************************************************************************************************ + // Decimal Multiply. + + private static boolean doMultiply( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + // Set signum before; if result is zero, fastMultiply will set signum to 0. + fastResult.fastSignum = (leftSignum == rightSignum ? 1 : -1); + int resultScale = leftScale + rightScale; + + /* + * For multiplicands with scale 0, trim trailing zeroes. + */ + if (leftScale == 0) { + + // Pretend like it has fractional digits so we can get the trailing zero count. + final int leftTrailingZeroCount = + fastTrailingDecimalZeroCount( + leftFast0, leftFast1, leftFast2, + 0, leftIntegerDigitCount); + if (leftTrailingZeroCount > 0) { + doFastScaleDown( + leftFast0, leftFast1, leftFast2, leftTrailingZeroCount, fastResult); + resultScale -= leftTrailingZeroCount; + leftFast0 = fastResult.fast0; + leftFast1 = fastResult.fast1; + leftFast2 = fastResult.fast2; + } + } + if (rightScale == 0) { + + // Pretend like it has fractional digits so we can get the trailing zero count. + final int rightTrailingZeroCount = + fastTrailingDecimalZeroCount( + rightFast0, rightFast1, rightFast2, + 0, rightIntegerDigitCount); + if (rightTrailingZeroCount > 0) { + doFastScaleDown( + rightFast0, rightFast1, rightFast2, rightTrailingZeroCount, fastResult); + resultScale -= rightTrailingZeroCount; + rightFast0 = fastResult.fast0; + rightFast1 = fastResult.fast1; + rightFast2 = fastResult.fast2; + } + } + + boolean largeOverflow = + !fastMultiply5x5HalfWords( + leftFast0, leftFast1, leftFast2, + rightFast0, rightFast1, rightFast2, + fastResult); + if (largeOverflow) { + return false; + } + + if (fastResult.fastSignum == 0) { + fastResult.fastScale = 0; + return true; + } + + if (resultScale < 0) { + if (-resultScale >= FAST_MAX_SCALE) { + return false; + } + if (!fastScaleUp( + fastResult.fast0, fastResult.fast1, fastResult.fast2, -resultScale, + fastResult)) { + return false; + } + resultScale = 0; + } + + int precision; + if (fastResult.fast2 != 0) { + precision = TWO_X_LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast2); + } else if (fastResult.fast1 != 0) { + precision = LONGWORD_DECIMAL_DIGITS + fastLongWordPrecision(fastResult.fast1); + } else { + precision = fastLongWordPrecision(fastResult.fast0); + } + + int integerDigitCount = Math.max(0, precision - resultScale); + if (integerDigitCount > FAST_MAX_PRECISION) { + // Integer is too large -- cannot recover by trimming fractional digits. + return false; + } + + if (precision > FAST_MAX_PRECISION || resultScale > FAST_MAX_SCALE) { + + // Trim off lower fractional digits but with NO ROUNDING. + + final int maxScale = FAST_MAX_SCALE - integerDigitCount; + final int scaleDown = resultScale - maxScale; + if (!fastScaleDownNoRound( + fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, + scaleDown, + fastResult)) { + // Round fractional must be 0. Not allowed to throw away digits. + return false; + } + resultScale -= scaleDown; + } + fastResult.fastScale = resultScale; + + // This assume no round up... + fastResult.fastIntegerDigitCount = integerDigitCount; + + if (fastResult.fastScale > FAST_MAX_SCALE) { + // We are not allowed to lose digits in multiply, so overflow. + return false; + } + final int resultTrailingZeroCount = + fastTrailingDecimalZeroCount( + fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale); + if (resultTrailingZeroCount > 0) { + doFastScaleDown( + fastResult, + resultTrailingZeroCount, + fastResult); + if (fastResult.fastSignum == 0) { + fastResult.fastScale = 0; + } else { + fastResult.fastScale -= resultTrailingZeroCount; + } + } + + return true; + } + + public static boolean fastMultiply5x5HalfWords( + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return + fastMultiply5x5HalfWords( + fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastResult); + } + + /** + * Fast decimal multiplication on two decimals that have been already scaled and whose results + * will fit in 38 digits. + * + * The caller is responsible checking for overflow within the highword and determining + * if scale down appropriate. + * + * @param left0 + * @param left1 + * @param left2 + * @param right0 + * @param right1 + * @param right2 + * @param result + * @return Returns false if the multiplication resulted in large overflow. Values in result are + * undefined in that case. + */ + public static boolean fastMultiply5x5HalfWords( + long left0, long left1, long left2, + long right0, long right1, long right2, + FastHiveDecimal fastResult) { + + long product; + + final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL; + + final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL; + + // v[0] + product = + halfRight0 * halfLeft0; + final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0]. + product = + halfRight0 + * halfLeft1 + + halfRight1 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[2] + product = + halfRight0 + * halfLeft2 + + halfRight1 + * halfLeft1 + + halfRight2 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[3] + product = + halfRight0 + * halfLeft3 + + halfRight1 + * halfLeft2 + + halfRight2 + * halfLeft1 + + halfRight3 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[4] + product = + halfRight0 + * halfLeft4 + + halfRight1 + * halfLeft3 + + halfRight2 + * halfLeft2 + + halfRight3 + * halfLeft1 + + halfRight4 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + + // v[5] is not calculated since high integer is always 0 for our decimals. + + // These remaining combinations below definitely result in overflow. + if ((halfRight4 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0 || halfLeft1 != 0)) + || (halfRight3 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0)) + || (halfRight2 != 0 && (halfLeft4 != 0 || halfLeft3 != 0)) + || (halfRight1 != 0 && halfLeft4 != 0)) { + return false; + } + + + final long result0 = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0; + final long result1 = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2; + final long result2 = product; + + if (result0 == 0 && result1 == 0 && result2 == 0) { + fastResult.fastSignum = 0; + } + fastResult.fast0 = result0; + fastResult.fast1 = result1; + fastResult.fast2 = result2; + + return true; + } + + public static boolean fastMultiplyFullInternal( + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + long[] result) { + return + fastMultiplyFullInternal( + fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastRight.fast0, fastRight.fast1, fastRight.fast2, + result); + } + + /** + * Fast decimal multiplication on two decimals that have been already scaled and whose results + * will fit in 38 digits. + * + * The caller is responsible checking for overflow within the highword and determining + * if scale down appropriate. + * + * @param left0 + * @param left1 + * @param left2 + * @param right0 + * @param right1 + * @param right2 + * @param result + * @return Returns false if the multiplication resulted in large overflow. Values in result are + * undefined in that case. + */ + public static boolean fastMultiply5x5HalfWords( + long left0, long left1, long left2, + long right0, long right1, long right2, + long[] result) { + + long product; + + final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL; + + final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL; + + // v[0] + product = + halfRight0 * halfLeft0; + final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0]. + product = + halfRight0 + * halfLeft1 + + halfRight1 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[2] + product = + halfRight0 + * halfLeft2 + + halfRight1 + * halfLeft1 + + halfRight2 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[3] + product = + halfRight0 + * halfLeft3 + + halfRight1 + * halfLeft2 + + halfRight2 + * halfLeft1 + + halfRight3 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[4] + product = + halfRight0 + * halfLeft4 + + halfRight1 + * halfLeft3 + + halfRight2 + * halfLeft2 + + halfRight3 + * halfLeft1 + + halfRight4 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + + // v[5] is not calculated since high integer is always 0 for our decimals. + + // These remaining combinations below definitely result in overflow. + if ((halfRight4 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0 || halfLeft1 != 0)) + || (halfRight3 != 0 && (halfLeft4 != 0 || halfLeft3 != 0 || halfLeft2 != 0)) + || (halfRight2 != 0 && (halfLeft4 != 0 || halfLeft3 != 0)) + || (halfRight1 != 0 && halfLeft4 != 0)) { + return false; + } + + result[0] = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0; + result[1] = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2; + result[2] = product; + + return true; + } + + /** + * Fast decimal multiplication on two decimals whose results are permitted to go beyond + * 38 digits to the maximum possible 76 digits. The caller is responsible for scaling and + * rounding the results back to 38 or fewer digits. + * + * The caller is responsible for determining the signum. + * + * @param left0 + * @param left1 + * @param left2 + * @param right0 + * @param right1 + * @param right2 + * @param result This full result has 5 longs. + * @return Returns false if the multiplication resulted in an overflow. Values in result are + * undefined in that case. + */ + public static boolean fastMultiplyFullInternal( + long left0, long left1, long left2, + long right0, long right1, long right2, + long[] result) { + assert (result.length == 5); + if (result.length != 5) { + throw new IllegalArgumentException("Expecting result array length = 5"); + } + + long product; + + final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL; + + final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL; + + // v[0] + product = + halfRight0 * halfLeft0; + final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0]. + product = + halfRight0 + * halfLeft1 + + halfRight1 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[2] + product = + halfRight0 + * halfLeft2 + + halfRight1 + * halfLeft1 + + halfRight2 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[3] + product = + halfRight0 + * halfLeft3 + + halfRight1 + * halfLeft2 + + halfRight2 + * halfLeft1 + + halfRight3 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[4] + product = + halfRight0 + * halfLeft4 + + halfRight1 + * halfLeft3 + + halfRight2 + * halfLeft2 + + halfRight3 + * halfLeft1 + + halfRight4 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z4 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[5] -- since integer #5 is always 0, some products here are not included. + product = + halfRight1 + * halfLeft4 + + halfRight2 + * halfLeft3 + + halfRight3 + * halfLeft2 + + halfRight4 + * halfLeft1 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z5 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[6] -- since integer #5 is always 0, some products here are not included. + product = + halfRight2 + * halfLeft4 + + halfRight3 + * halfLeft3 + + halfRight4 + * halfLeft2 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z6 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[7] -- since integer #5 is always 0, some products here are not included. + product = + halfRight3 + * halfLeft4 + + halfRight4 + * halfLeft3 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z7 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[8] -- since integer #5 is always 0, some products here are not included. + product = + halfRight4 + * halfLeft4 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z8 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[9] -- since integer #5 is always 0, some products here are not included. + product = + (product / MULTIPLER_INTWORD_DECIMAL); + if (product > FULL_MAX_HIGHWORD_DECIMAL) { + return false; + } + + result[0] = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0; + result[1] = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2; + result[2] = (long) z5 * MULTIPLER_INTWORD_DECIMAL + (long) z4; + result[3] = (long) z7 * MULTIPLER_INTWORD_DECIMAL + (long) z6; + result[4] = product * MULTIPLER_INTWORD_DECIMAL + (long) z8; + + return true; + } + + /** + * Fast decimal multiplication on two decimals whose results are permitted to go beyond + * 38 digits to the maximum possible 76 digits. The caller is responsible for scaling and + * rounding the results back to 38 or fewer digits. + * + * The caller is responsible for determining the signum. + * + * @param left0 + * @param left1 + * @param left2 + * @param right0 + * @param right1 + * @param right2 + * @param result This full result has 5 longs. + * @return Returns false if the multiplication resulted in an overflow. Values in result are + * undefined in that case. + */ + public static boolean fastMultiply5x6HalfWords( + long left0, long left1, long left2, + long right0, long right1, long right2, + long[] result) { + + if (result.length != 6) { + throw new RuntimeException("Expecting result array length = 6"); + } + + long product; + + final long halfRight0 = right0 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight1 = right0 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight2 = right1 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight3 = right1 / MULTIPLER_INTWORD_DECIMAL; + final long halfRight4 = right2 % MULTIPLER_INTWORD_DECIMAL; + final long halfRight5 = right2 / MULTIPLER_INTWORD_DECIMAL; + + final long halfLeft0 = left0 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft1 = left0 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft2 = left1 % MULTIPLER_INTWORD_DECIMAL; + final long halfLeft3 = left1 / MULTIPLER_INTWORD_DECIMAL; + final long halfLeft4 = left2 % MULTIPLER_INTWORD_DECIMAL; + + // v[0] + product = + halfRight0 * halfLeft0; + final int z0 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[1] where (product % MULTIPLER_INTWORD_DECIMAL) is the carry from v[0]. + product = + halfRight0 + * halfLeft1 + + halfRight1 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z1 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[2] + product = + halfRight0 + * halfLeft2 + + halfRight1 + * halfLeft1 + + halfRight2 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z2 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[3] + product = + halfRight0 + * halfLeft3 + + halfRight1 + * halfLeft2 + + halfRight2 + * halfLeft1 + + halfRight3 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z3 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[4] + product = + halfRight0 + * halfLeft4 + + halfRight1 + * halfLeft3 + + halfRight2 + * halfLeft2 + + halfRight3 + * halfLeft1 + + halfRight4 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z4 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[5] -- since left integer #5 is always 0, some products here are not included. + product = + halfRight1 + * halfLeft4 + + halfRight2 + * halfLeft3 + + halfRight3 + * halfLeft2 + + halfRight4 + * halfLeft1 + + halfRight5 + * halfLeft0 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z5 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[6] -- since left integer #5 is always 0, some products here are not included. + product = + halfRight2 + * halfLeft4 + + halfRight3 + * halfLeft3 + + halfRight4 + * halfLeft2 + + halfRight5 + * halfLeft1 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z6 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[7] -- since left integer #5 is always 0, some products here are not included. + product = + halfRight3 + * halfLeft4 + + halfRight4 + * halfLeft3 + + halfRight5 + * halfLeft2 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z7 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[8] -- since left integer #5 is always 0, some products here are not included. + product = + halfRight4 + * halfLeft4 + + halfRight5 + * halfLeft3 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z8 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[9] -- since left integer #5 is always 0, some products here are not included. + product = + halfRight5 + * halfLeft4 + + (product / MULTIPLER_INTWORD_DECIMAL); + final int z9 = (int) (product % MULTIPLER_INTWORD_DECIMAL); + + // v[10] -- since left integer #5 is always 0, some products here are not included. + product = + + (product / MULTIPLER_INTWORD_DECIMAL); + if (product > MULTIPLER_INTWORD_DECIMAL) { + return false; + } + + result[0] = (long) z1 * MULTIPLER_INTWORD_DECIMAL + (long) z0; + result[1] = (long) z3 * MULTIPLER_INTWORD_DECIMAL + (long) z2; + result[2] = (long) z5 * MULTIPLER_INTWORD_DECIMAL + (long) z4; + result[3] = (long) z7 * MULTIPLER_INTWORD_DECIMAL + (long) z6; + result[4] = (long) z9 * MULTIPLER_INTWORD_DECIMAL + (long) z8; + result[5] = product; + + return true; + } + + public static boolean fastMultiply( + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return fastMultiply( + fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastLeft.fastIntegerDigitCount, fastLeft.fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + public static boolean fastMultiply( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + if (leftSignum == 0 || rightSignum == 0) { + fastResult.fastReset(); + return true; + } + + return doMultiply( + leftSignum, leftFast0, leftFast1, leftFast2, + leftIntegerDigitCount, leftScale, + rightSignum, rightFast0, rightFast1, rightFast2, + rightIntegerDigitCount, rightScale, + fastResult); + } + + //************************************************************************************************ + // Decimal Division / Remainder. + + /** + * + * @param leftFast0 + * @param leftFast1 + * @param leftFast2 + * @param rightFast0 + * @param fastResult + * @return remainderSubexpr2 + */ + private static long doSingleWordQuotient( + long leftFast0, long leftFast1, long leftFast2, + long rightFast0, + FastHiveDecimal fastResult) { + + long quotient2; + long quotient1; + long quotient0; + + long remainderSubexpr2; + + if (leftFast2 == 0 && leftFast1 == 0) { + quotient2 = 0; + quotient1 = 0; + quotient0 = + leftFast0 / rightFast0; + final long k0 = + leftFast0 - quotient0 * rightFast0; + remainderSubexpr2 = + k0 * MULTIPLER_LONGWORD_DECIMAL; + } else if (leftFast2 == 0) { + // leftFast1 != 0. + quotient2 = 0; + quotient1 = + leftFast1 / rightFast0; + final long k1 = + leftFast1 - quotient1 * rightFast0; + final long quotientSubexpr0 = + k1 * MULTIPLER_LONGWORD_DECIMAL + + leftFast0; + quotient0 = + quotientSubexpr0 / rightFast0; + final long k0 = + quotientSubexpr0 - quotient0 * rightFast0; + remainderSubexpr2 = + k0 * MULTIPLER_LONGWORD_DECIMAL; + } else if (leftFast1 == 0){ + // leftFast2 != 0 && leftFast1 == 0. + quotient2 = + leftFast2 / rightFast0; + quotient1 = 0; + quotient0 = + leftFast0 / rightFast0; + final long k0 = + leftFast0 - quotient0 * rightFast0; + remainderSubexpr2 = + k0 * MULTIPLER_LONGWORD_DECIMAL; + } else { + quotient2 = + leftFast2 / rightFast0; + final long k2 = + leftFast2 - quotient2 * rightFast0; + final long quotientSubexpr1 = + k2 * MULTIPLER_LONGWORD_DECIMAL + + leftFast1; + quotient1 = + quotientSubexpr1 / rightFast0; + final long k1 = + quotientSubexpr1 - quotient1 * rightFast0; + final long quotientSubexpr0 = + k1 * MULTIPLER_LONGWORD_DECIMAL; + quotient0 = + quotientSubexpr0 / rightFast0; + final long k0 = + quotientSubexpr0 - quotient0 * rightFast0; + remainderSubexpr2 = + k0 * MULTIPLER_LONGWORD_DECIMAL; + } + + fastResult.fast0 = quotient0; + fastResult.fast1 = quotient1; + fastResult.fast2 = quotient2; + + return remainderSubexpr2; + } + + private static int doSingleWordRemainder( + long leftFast0, long leftFast1, long leftFast2, + long rightFast0, + long remainderSubexpr2, + FastHiveDecimal fastResult) { + + int remainderDigitCount; + + long remainder2; + long remainder1; + long remainder0; + + if (remainderSubexpr2 == 0) { + remainder2 = 0; + remainder1 = 0; + remainder0 = 0; + remainderDigitCount = 0; + } else { + remainder2 = + remainderSubexpr2 / rightFast0; + final long k2 = + remainderSubexpr2 - remainder2 * rightFast0; + if (k2 == 0) { + remainder1 = 0; + remainder0 = 0; + remainderDigitCount = + LONGWORD_DECIMAL_DIGITS - fastLongWordTrailingZeroCount(remainder2); + } else { + final long remainderSubexpr1 = + k2 * MULTIPLER_LONGWORD_DECIMAL; + long remainderSubexpr0; + remainder1 = + remainderSubexpr1 / rightFast0; + final long k1 = + remainderSubexpr1 - remainder1 * rightFast0; + if (k1 == 0) { + remainder0 = 0; + remainderDigitCount = + LONGWORD_DECIMAL_DIGITS + + LONGWORD_DECIMAL_DIGITS - fastLongWordTrailingZeroCount(remainder1); + } else { + remainderSubexpr0 = + k2 * MULTIPLER_LONGWORD_DECIMAL; + + remainder0 = + remainderSubexpr0 / rightFast0; + remainderDigitCount = + TWO_X_LONGWORD_DECIMAL_DIGITS + + LONGWORD_DECIMAL_DIGITS - fastLongWordTrailingZeroCount(remainder0); + } + } + } + + fastResult.fast0 = remainder0; + fastResult.fast1 = remainder1; + fastResult.fast2 = remainder2; + + return remainderDigitCount; + } + + // EXPERIMENT + private static boolean fastSingleWordDivision( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, int leftScale, + int rightSignum, long rightFast0, int rightScale, + FastHiveDecimal fastResult) { + + long remainderSubexpr2 = + doSingleWordQuotient( + leftFast0, leftFast1, leftFast2, + rightFast0, + fastResult); + + long quotient0 = fastResult.fast0; + long quotient1 = fastResult.fast1; + long quotient2 = fastResult.fast2; + + int quotientDigitCount; + if (quotient2 != 0) { + quotientDigitCount = fastLongWordPrecision(quotient2); + } else if (quotient1 != 0) { + quotientDigitCount = fastLongWordPrecision(quotient1); + } else { + quotientDigitCount = fastLongWordPrecision(quotient0); + } + + int remainderDigitCount = + doSingleWordRemainder( + leftFast0, leftFast1, leftFast2, + rightFast0, + remainderSubexpr2, + fastResult); + + long remainder0 = fastResult.fast0; + long remainder1 = fastResult.fast1; + long remainder2 = fastResult.fast2; + + fastResult.fast0 = quotient0; + fastResult.fast1 = quotient1; + fastResult.fast2 = quotient2; + + final int quotientScale = leftScale + rightScale; + + if (remainderDigitCount == 0) { + fastResult.fastScale = quotientScale; + } else { + int resultScale = quotientScale + remainderDigitCount; + + int adjustedQuotientDigitCount; + if (quotientScale > 0) { + adjustedQuotientDigitCount = Math.max(0, quotientDigitCount - quotientScale); + } else { + adjustedQuotientDigitCount = quotientDigitCount; + } + final int maxScale = FAST_MAX_SCALE - adjustedQuotientDigitCount; + + int scale = Math.min(resultScale, maxScale); + + int remainderScale; + remainderScale = Math.min(remainderDigitCount, maxScale - quotientScale); + if (remainderScale > 0) { + if (quotientDigitCount > 0) { + // Make room for remainder. + fastScaleUp( + fastResult, + remainderScale, + fastResult); + } + // Copy in remainder digits... which start at the top of remainder2. + if (remainderScale < LONGWORD_DECIMAL_DIGITS) { + final long remainderDivisor2 = powerOfTenTable[LONGWORD_DECIMAL_DIGITS - remainderScale]; + fastResult.fast0 += (remainder2 / remainderDivisor2); + } else if (remainderScale == LONGWORD_DECIMAL_DIGITS) { + fastResult.fast0 = remainder2; + } else if (remainderScale < TWO_X_LONGWORD_DECIMAL_DIGITS) { + final long remainderDivisor2 = powerOfTenTable[remainderScale - LONGWORD_DECIMAL_DIGITS]; + fastResult.fast1 += (remainder2 / remainderDivisor2); + fastResult.fast0 = remainder1; + } else if (remainderScale == TWO_X_LONGWORD_DECIMAL_DIGITS) { + fastResult.fast1 = remainder2; + fastResult.fast0 = remainder1; + } + } + + // UNDONE: Method is still under development. + fastResult.fastScale = scale; + + // UNDONE: Trim trailing zeroes... + } + + return true; + } + + public static boolean fastDivide( + FastHiveDecimal fastLeft, + FastHiveDecimal fastRight, + FastHiveDecimal fastResult) { + return fastDivide( + fastLeft.fastSignum, fastLeft.fast0, fastLeft.fast1, fastLeft.fast2, + fastLeft.fastIntegerDigitCount, fastLeft.fastScale, + fastRight.fastSignum, fastRight.fast0, fastRight.fast1, fastRight.fast2, + fastRight.fastIntegerDigitCount, fastRight.fastScale, + fastResult); + } + + public static boolean fastDivide( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + // Arithmetic operations reset the results. + fastResult.fastReset(); + + if (rightSignum == 0) { + // Division by 0. + return false; + } + if (leftSignum == 0) { + // Zero result. + return true; + } + + /* + if (rightFast1 == 0 && rightFast2 == 0) { + return fastSingleWordDivision( + leftSignum, leftFast0, leftFast1, leftFast2, leftScale, + rightSignum, rightFast0, rightScale, + fastResult); + } + */ + + BigDecimal denominator = + fastBigDecimalValue( + leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale); + BigDecimal divisor = + fastBigDecimalValue( + rightSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale); + BigDecimal quotient = + denominator.divide(divisor, FAST_MAX_SCALE, FAST_ROUND_HALF_UP); + + if (!fastSetFromBigDecimal( + quotient, + true, + fastResult)) { + return false; + } + /* + if (!fastIsValid(fastResult)) { + fastResult.fastRaiseInvalidException( + "denominator " + denominator + " divisor " + divisor + " quotient " + quotient); + } + */ + return true; + } + + public static boolean fastRemainder( + int leftSignum, long leftFast0, long leftFast1, long leftFast2, + int leftIntegerDigitCount, int leftScale, + int rightSignum, long rightFast0, long rightFast1, long rightFast2, + int rightIntegerDigitCount, int rightScale, + FastHiveDecimal fastResult) { + + // Arithmetic operations reset the results. + fastResult.fastReset(); + + if (rightSignum == 0) { + // Division by 0. + return false; + } + if (leftSignum == 0) { + // Zero result. + return true; + } + + BigDecimal denominator = + fastBigDecimalValue( + leftSignum, leftFast0, leftFast1, leftFast2, leftIntegerDigitCount, leftScale); + BigDecimal divisor = + fastBigDecimalValue( + rightSignum, rightFast0, rightFast1, rightFast2, rightIntegerDigitCount, rightScale); + BigDecimal remainder = + denominator.remainder(divisor); + fastResult.fastReset(); + if (!fastSetFromBigDecimal( + remainder, + true, + fastResult)) { + return false; + } + /* + if (!fastIsValid(fastResult)) { + fastResult.fastRaiseInvalidException( + "denominator " + denominator + " divisor " + divisor + " remainder " + remainder); + } + */ + return true; + } + + public static boolean fastPow( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int exponent, + FastHiveDecimal fastResult) { + + // Arithmetic operations (re)set the results. + fastResult.fastSet(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + + if (exponent < 0) { + // UNDONE: Currently, negative exponent is not supported. + return false; + } + + for (int e = 1; e < exponent; e++) { + if (!doMultiply( + fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale, + fastResult.fastSignum, fastResult.fast0, fastResult.fast1, fastResult.fast2, + fastResult.fastIntegerDigitCount, fastResult.fastScale, + fastResult)) { + return false; + } + } + return true; + } + + //************************************************************************************************ + // Decimal String Formatting. + + public static String fastToFormatString( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int formatScale) { + byte[] scratchBuffer = new byte[FAST_SCRATCH_BUFFER_LEN_TO_BYTES]; + final int index = + doFastToFormatBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + formatScale, + scratchBuffer); + return + new String(scratchBuffer, index, FAST_SCRATCH_BUFFER_LEN_TO_BYTES - index, StandardCharsets.UTF_8); + } + + //************************************************************************************************ + // Decimal String Formatting. + + public static int fastToFormatString( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int formatScale, + byte[] scratchBuffer) { + return + doFastToFormatBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + formatScale, + scratchBuffer); + } + + public static int fastToFormatBytes( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int formatScale, + byte[] scratchBuffer) { + return + doFastToFormatBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + formatScale, + scratchBuffer); + } + + public static int doFastToFormatBytes( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, + int formatScale, + byte[] scratchBuffer) { + + // NOTE: OldHiveDecimal.toFormatString returns decimal strings with more than > 38 digits! + + if (formatScale >= fastScale || formatScale == 0 || formatScale == FAST_MAX_SCALE) { + return + doFastToBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, formatScale, + scratchBuffer); + } else { + FastHiveDecimal fastTemp = new FastHiveDecimal(); + if (!fastRound( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, + formatScale, FAST_ROUND_HALF_UP, + fastTemp)) { + return 0; + } + return + doFastToBytes( + fastTemp.fastSignum, fastTemp.fast0, fastTemp.fast1, fastTemp.fast2, + fastTemp.fastIntegerDigitCount, fastTemp.fastScale, formatScale, + scratchBuffer); + } + } + + public static String fastToString( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale) { + return doFastToString( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, fastTrailingZeroesScale); + } + + public static String fastToString( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale, + byte[] scratchBuffer) { + return doFastToString( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, fastTrailingZeroesScale, + scratchBuffer); + } + + public static String fastToDigitsOnlyString( + long fast0, long fast1, long fast2, + int fastIntegerDigitCount) { + byte[] scratchBuffer = new byte[FAST_SCRATCH_BUFFER_LEN_TO_BYTES]; + final int index = + doFastToDigitsOnlyBytes( + fast0, fast1, fast2, + fastIntegerDigitCount, + scratchBuffer); + return + new String(scratchBuffer, index, FAST_SCRATCH_BUFFER_LEN_TO_BYTES - index, StandardCharsets.UTF_8); + } + + public static int fastToBytes( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale, + byte[] scratchBuffer) { + return doFastToBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, fastTrailingZeroesScale, + scratchBuffer); + } + + private static String doFastToString( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale) { + byte[] scratchBuffer = new byte[FAST_SCRATCH_BUFFER_LEN_TO_BYTES]; + final int index = + doFastToBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, fastTrailingZeroesScale, + scratchBuffer); + return + new String( + scratchBuffer, index, FAST_SCRATCH_BUFFER_LEN_TO_BYTES - index, StandardCharsets.UTF_8); + } + + private static String doFastToString( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale, + byte[] scratchBuffer) { + final int index = + doFastToBytes( + fastSignum, fast0, fast1, fast2, + fastIntegerDigitCount, fastScale, fastTrailingZeroesScale, + scratchBuffer); + return new String(scratchBuffer, index, scratchBuffer.length - index, StandardCharsets.UTF_8); + } + + private final static byte BYTE_BLANK = (byte) ' '; + private final static byte BYTE_DIGIT_ZERO = (byte) '0'; + private final static byte BYTE_DIGIT_NINE = (byte) '9'; + private final static byte BYTE_DOT = (byte) '.'; + private final static byte BYTE_MINUS = (byte) '-'; + private final static byte BYTE_PLUS = (byte) '+'; + private final static byte BYTE_EXPONENT_LOWER = (byte) 'e'; + private final static byte BYTE_EXPONENT_UPPER = (byte) 'E'; + + private static int doFastToBytes( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale, int fastTrailingZeroesScale, + byte[] scratchBuffer) { + + int index = scratchBuffer.length - 1; + + int trailingZeroCount = + (fastTrailingZeroesScale != -1 ? fastTrailingZeroesScale - fastScale : 0); + // Virtual trailing zeroes. + if (trailingZeroCount > 0) { + for (int i = 0; i < trailingZeroCount; i++) { + scratchBuffer[index--] = BYTE_DIGIT_ZERO; + } + } + + // Scale fractional digits, dot, integer digits. + + final int scale = fastScale; + + final boolean isZeroFast1AndFast2 = (fast1 == 0 && fast2 == 0); + final boolean isZeroFast2 = (fast2 == 0); + + int lowerLongwordScale = 0; + int middleLongwordScale = 0; + int highLongwordScale = 0; + long longWord = fast0; + if (scale > 0) { + + // Fraction digits from lower longword. + + lowerLongwordScale = Math.min(scale, LONGWORD_DECIMAL_DIGITS); + + for (int i = 0; i < lowerLongwordScale; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + longWord /= 10; + } + if (lowerLongwordScale == LONGWORD_DECIMAL_DIGITS) { + longWord = fast1; + } + + if (scale > LONGWORD_DECIMAL_DIGITS) { + + // Fraction digits continue into middle longword. + + middleLongwordScale = Math.min(scale - LONGWORD_DECIMAL_DIGITS, LONGWORD_DECIMAL_DIGITS); + for (int i = 0; i < middleLongwordScale; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + longWord /= 10; + } + if (middleLongwordScale == LONGWORD_DECIMAL_DIGITS) { + longWord = fast2; + } + + if (scale > TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Fraction digit continue into highest longword. + + highLongwordScale = scale - TWO_X_LONGWORD_DECIMAL_DIGITS; + for (int i = 0; i < highLongwordScale; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + longWord /= 10; + } + } + } + scratchBuffer[index--] = BYTE_DOT; + } else if (trailingZeroCount > 0) { + scratchBuffer[index--] = BYTE_DOT; + } + + // Integer digits; stop on zeroes above. + + boolean atLeastOneIntegerDigit = false; + if (scale <= LONGWORD_DECIMAL_DIGITS) { + + // Handle remaining lower long word digits as integer digits. + + final int remainingLowerLongwordDigits = LONGWORD_DECIMAL_DIGITS - lowerLongwordScale; + for (int i = 0; i < remainingLowerLongwordDigits; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + atLeastOneIntegerDigit = true; + longWord /= 10; + if (longWord == 0 && isZeroFast1AndFast2) { + // Suppress leading zeroes. + break; + } + } + if (isZeroFast1AndFast2) { + if (!atLeastOneIntegerDigit) { + scratchBuffer[index--] = BYTE_DIGIT_ZERO; + } + if (fastSignum == -1) { + scratchBuffer[index--] = BYTE_MINUS; + } + return index + 1; + } + longWord = fast1; + } + + if (scale <= TWO_X_LONGWORD_DECIMAL_DIGITS) { + + // Handle remaining middle long word digits. + + final int remainingMiddleLongwordDigits = LONGWORD_DECIMAL_DIGITS - middleLongwordScale; + + for (int i = 0; i < remainingMiddleLongwordDigits; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + atLeastOneIntegerDigit = true; + longWord /= 10; + if (longWord == 0 && isZeroFast2) { + // Suppress leading zeroes. + break; + } + } + if (isZeroFast2) { + if (!atLeastOneIntegerDigit) { + scratchBuffer[index--] = BYTE_DIGIT_ZERO; + } + if (fastSignum == -1) { + scratchBuffer[index--] = BYTE_MINUS; + } + return index + 1; + } + longWord = fast2; + } + + final int remainingHighwordDigits = HIGHWORD_DECIMAL_DIGITS - highLongwordScale; + + for (int i = 0; i < remainingHighwordDigits; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + atLeastOneIntegerDigit = true; + longWord /= 10; + if (longWord == 0) { + // Suppress leading zeroes. + break; + } + } + if (!atLeastOneIntegerDigit) { + scratchBuffer[index--] = BYTE_DIGIT_ZERO; + } + if (fastSignum == -1) { + scratchBuffer[index--] = BYTE_MINUS; + } + return index + 1; + } + + public static int fastToDigitsOnlyBytes( + long fast0, long fast1, long fast2, + int fastIntegerDigitCount, + byte[] scratchBuffer) { + return doFastToDigitsOnlyBytes( + fast0, fast1, fast2, + fastIntegerDigitCount, + scratchBuffer); + } + + private static int doFastToDigitsOnlyBytes( + long fast0, long fast1, long fast2, + int fastIntegerDigitCount, + byte[] scratchBuffer) { + + int index = scratchBuffer.length - 1; + + // Just digits. + + final boolean isZeroFast1AndFast2 = (fast1 == 0 && fast2 == 0); + final boolean isZeroFast2 = (fast2 == 0); + + boolean atLeastOneIntegerDigit = false; + long longWord = fast0; + for (int i = 0; i < LONGWORD_DECIMAL_DIGITS; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + atLeastOneIntegerDigit = true; + longWord /= 10; + if (longWord == 0 && isZeroFast1AndFast2) { + // Suppress leading zeroes. + break; + } + } + if (isZeroFast1AndFast2) { + if (!atLeastOneIntegerDigit) { + scratchBuffer[index--] = BYTE_DIGIT_ZERO; + } + return index + 1; + } + + longWord = fast1; + + for (int i = 0; i < LONGWORD_DECIMAL_DIGITS; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + atLeastOneIntegerDigit = true; + longWord /= 10; + if (longWord == 0 && isZeroFast2) { + // Suppress leading zeroes. + break; + } + } + if (isZeroFast2) { + if (!atLeastOneIntegerDigit) { + scratchBuffer[index--] = BYTE_DIGIT_ZERO; + } + return index + 1; + } + + longWord = fast2; + + for (int i = 0; i < HIGHWORD_DECIMAL_DIGITS; i++) { + scratchBuffer[index--] = (byte) (BYTE_DIGIT_ZERO + longWord % 10); + atLeastOneIntegerDigit = true; + longWord /= 10; + if (longWord == 0) { + // Suppress leading zeroes. + break; + } + } + if (!atLeastOneIntegerDigit) { + scratchBuffer[index--] = BYTE_DIGIT_ZERO; + } + return index + 1; + } + + //************************************************************************************************ + // Decimal Validation. + + public static boolean fastIsValid(FastHiveDecimal fastDec) { + return fastIsValid( + fastDec.fastSignum, fastDec.fast0, fastDec.fast1, fastDec.fast2, + fastDec.fastIntegerDigitCount, fastDec.fastScale); + } + + public static boolean fastIsValid( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + boolean isValid; + if (fastSignum == 0) { + isValid = (fast0 == 0 && fast1 == 0 && fast2 == 0 && fastIntegerDigitCount == 0 && fastScale == 0); + if (!isValid) { + System.out.println("FAST_IS_VALID signum 0 but other fields not"); + } + } else { + isValid = ( + (fast0 >= 0 && fast0 <= MAX_LONGWORD_DECIMAL) && + (fast1 >= 0 && fast1 <= MAX_LONGWORD_DECIMAL) && + (fast2 >= 0 && fast2 <= MAX_HIGHWORD_DECIMAL)); + if (!isValid) { + System.out.println("FAST_IS_VALID fast0 .. fast2 out of range"); + } else { + if (fastScale < 0 || fastScale > FAST_MAX_SCALE) { + System.out.println("FAST_IS_VALID fastScale " + fastScale + " out of range"); + isValid = false; + } else if (fastIntegerDigitCount < 0 || fastIntegerDigitCount > FAST_MAX_PRECISION) { + System.out.println("FAST_IS_VALID fastIntegerDigitCount " + fastIntegerDigitCount + " out of range"); + isValid = false; + } else if (fastIntegerDigitCount + fastScale > FAST_MAX_PRECISION) { + System.out.println("FAST_IS_VALID exceeds max precision: fastIntegerDigitCount " + fastIntegerDigitCount + " and fastScale " + fastScale); + isValid = false; + } else { + // Verify integerDigitCount given fastScale. + final int rawPrecision = fastRawPrecision(fastSignum, fast0, fast1, fast2); + if (fastIntegerDigitCount > 0) { + if (rawPrecision != fastIntegerDigitCount + fastScale) { + System.out.println("FAST_IS_VALID integer case: rawPrecision " + rawPrecision + + " fastIntegerDigitCount " + fastIntegerDigitCount + + " fastScale " + fastScale); + isValid = false; + } + } else { + if (rawPrecision > fastScale) { + System.out.println("FAST_IS_VALID fraction only case: rawPrecision " + rawPrecision + + " fastIntegerDigitCount " + fastIntegerDigitCount + + " fastScale " + fastScale); + isValid = false; + } + } + if (isValid) { + final int trailingZeroCount = + fastTrailingDecimalZeroCount( + fast0, fast1, fast2, + fastIntegerDigitCount, fastScale); + if (trailingZeroCount != 0) { + System.out.println("FAST_IS_VALID exceeds max precision: trailingZeroCount != 0"); + isValid = false; + } + } + } + } + } + + if (!isValid) { + System.out.println("FAST_IS_VALID fast0 " + fast0); + System.out.println("FAST_IS_VALID fast1 " + fast1); + System.out.println("FAST_IS_VALID fast2 " + fast2); + System.out.println("FAST_IS_VALID fastIntegerDigitCount " + fastIntegerDigitCount); + System.out.println("FAST_IS_VALID fastScale " + fastScale); + } + return isValid; + } + + public static void fastRaiseInvalidException( + FastHiveDecimal fastResult) { + throw new RuntimeException( + "Invalid fast decimal " + + " fastSignum " + fastResult.fastSignum + " fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2 + + " fastIntegerDigitCount " + fastResult.fastIntegerDigitCount + " fastScale " + fastResult.fastScale + + " stack trace: " + getStackTraceAsSingleLine(Thread.currentThread().getStackTrace())); + } + + public static void fastRaiseInvalidException( + FastHiveDecimal fastResult, + String parameters) { + throw new RuntimeException( + "Parameters: " + parameters + " --> " + + "Invalid fast decimal " + + " fastSignum " + fastResult.fastSignum + " fast0 " + fastResult.fast0 + " fast1 " + fastResult.fast1 + " fast2 " + fastResult.fast2 + + " fastIntegerDigitCount " + fastResult.fastIntegerDigitCount + " fastScale " + fastResult.fastScale + + " stack trace: " + getStackTraceAsSingleLine(Thread.currentThread().getStackTrace())); + } + + //************************************************************************************************ + // Decimal Debugging. + + static int STACK_LENGTH_LIMIT = 20; + public static String getStackTraceAsSingleLine(StackTraceElement[] stackTrace) { + StringBuilder sb = new StringBuilder(); + sb.append("Stack trace: "); + int length = stackTrace.length; + boolean isTruncated = false; + if (length > STACK_LENGTH_LIMIT) { + length = STACK_LENGTH_LIMIT; + isTruncated = true; + } + for (int i = 0; i < length; i++) { + if (i > 0) { + sb.append(", "); + } + sb.append(stackTrace[i]); + } + if (isTruncated) { + sb.append(", ..."); + } + + return sb.toString(); + } + + public static String displayBytes(byte[] bytes, int start, int length) { + StringBuilder sb = new StringBuilder(); + for (int i = start; i < start + length; i++) { + sb.append(String.format("\\%03d", (int) (bytes[i] & 0xff))); + } + return sb.toString(); + } +} \ No newline at end of file diff --git storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java index 674400c..5573988 100644 --- storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java +++ storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java @@ -17,17 +17,66 @@ */ package org.apache.hadoop.hive.common.type; +import java.util.Arrays; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.math.BigDecimal; import java.math.BigInteger; -import java.math.RoundingMode; /** + * HiveDecimal is a decimal data type with a maximum precision and scale. * - * HiveDecimal. Simple wrapper for BigDecimal. Adds fixed max precision and non scientific string - * representation + * It is the Hive DECIMAL data type. * + * The scale is the number of fractional decimal digits. The digits after the dot. It is limited + * to 38 (MAX_SCALE). + * + * The precision is the integer (or whole-number) decimal digits plus fractional decimal digits. + * It is limited to a total of 38 digits (MAX_PRECISION). + * + * Hive syntax for declaring DECIMAL has 3 forms: + * DECIMAL // Use the default precision/scale. + * DECIMAL(precision) // Use the default scale. + * DECIMAL(precision, scale) + * + * The declared scale must be <= precision. + * + * Use DECIMAL instead of DOUBLE when exact numeric accuracy is required. Not all decimal numbers + * are exactly representable in the binary floating point type DOUBLE and cause accuracy + * anomalies (i.e. wrong results). + * + * HiveDecimal is implemented as a classic Java immutable object. All operations on HiveDecimal + * that produce a different value will create a new HiveDecimal object. + * + * For good performance we do not represent the decimal using a BigDecimal object like the previous + * version of this class did (now renamed OldHiveDecimal). + * + * Decimals are physically stored without any extra leading or trailing zeroes. The scale, then, + * of the decimal is really the number of non-trailing zero fractional digits in the decimal. + * + * Math operations on decimals typically cause the scale to change as a result of the math and + * from trailing fractional digit elimination. + * + * Typically, Hive, when it wants to make sure a result decimal fits in the column decimal's + * precision/scale it calls enforcePrecisionScale. That method will scale down or trim off + * result fractional digits if necessary with rounding when the column has a smaller scale. + * And, it will also indicate overflow when the decimal has exceeded the column's maximum precision. + * + * NOTE: When Hive gets ready to display or serialize a decimal, it usually wants + * trailing fractional zeroes. See the special notes for toFormatString for details. */ -public class HiveDecimal implements Comparable { +public class HiveDecimal extends FastHiveDecimal implements Comparable { + + /* + * IMPLEMENTATION NOTE: + * We implement HiveDecimal with the mutable FastHiveDecimal class. That class uses + * protected on all its methods so they will not be visible in the HiveDecimal class. + * + * So even if one casts to FastHiveDecimal, you shouldn't be able to violate the immutability + * of a HiveDecimal class. + */ + public static final int MAX_PRECISION = 38; public static final int MAX_SCALE = 38; @@ -45,86 +94,675 @@ public static final int SYSTEM_DEFAULT_PRECISION = 38; public static final int SYSTEM_DEFAULT_SCALE = 18; - public static final HiveDecimal ZERO = new HiveDecimal(BigDecimal.ZERO); - public static final HiveDecimal ONE = new HiveDecimal(BigDecimal.ONE); + /** + * Common values. + */ + public static final HiveDecimal ZERO = HiveDecimal.create(0); + public static final HiveDecimal ONE = HiveDecimal.create(1); + /** + * ROUND_FLOOR: + * + * Round towards negative infinity. + * + * The Hive function is FLOOR. + * + * Positive numbers: The round fraction is thrown away. + * + * (Example here rounds at scale 0) + * Value FLOOR + * 0.3 0 + * 2 2 + * 2.1 2 + * + * Negative numbers: If there is a round fraction, throw it away and subtract 1. + * + * (Example here rounds at scale 0) + * Value FLOOR + * -0.3 -1 + * -2 -2 + * -2.1 -3 + */ public static final int ROUND_FLOOR = BigDecimal.ROUND_FLOOR; + + /** + * ROUND_CEILING: + * + * Round towards positive infinity. + * + * The Hive function is CEILING. + * + * Positive numbers: If there is a round fraction, throw it away and add 1 + * + * (Example here rounds at scale 0) + * Value CEILING + * 0.3 1 + * 2 2 + * 2.1 3 + * + * Negative numbers: The round fraction is thrown away. + * + * (Example here rounds at scale 0) + * Value CEILING + * -0.3 0 + * -2 -2 + * -2.1 -2 + */ public static final int ROUND_CEILING = BigDecimal.ROUND_CEILING; + + /** + * ROUND_HALF_UP: + * + * Round towards "nearest neighbor" unless both neighbors are equidistant then round up. + * + * The Hive function is ROUND. + * + * For result, throw away round fraction. If the round fraction is >= 0.5, then add 1 when + * positive and subtract 1 when negative. So, the sign is irrelevant. + * + * (Example here rounds at scale 0) + * Value ROUND Value ROUND + * 0.3 0 -0.3 0 + * 2 2 -2 -2 + * 2.1 2 -2.1 -2 + * 2.49 2 -2.49 -2 + * 2.5 3 -2.5 -3 + * + */ public static final int ROUND_HALF_UP = BigDecimal.ROUND_HALF_UP; + + /** + * ROUND_HALF_EVEN: + * Round towards the "nearest neighbor" unless both neighbors are equidistant, then round + * towards the even neighbor. + * + * The Hive function is BROUND. + * + * Known as Banker’s Rounding. + * + * When you add values rounded with ROUND_HALF_UP you have a bias that grows as you add more + * numbers. Banker's Rounding is a way to minimize that bias. It rounds toward the nearest + * even number when the fraction is 0.5 exactly. In table below, notice that 2.5 goes DOWN to + * 2 (even) but 3.5 goes UP to 4 (even), etc. + * + * So, the sign is irrelevant. + * + * (Example here rounds at scale 0) + * Value BROUND Value BROUND + * 0.49 0 -0.49 0 + * 0.5 0 -0.5 0 + * 0.51 1 -0.51 -1 + * 1.5 2 -1.5 -2 + * 2.5 2 -2.5 -2 + * 2.51 3 -2.51 -3 + * 3.5 4 -3.5 -4 + * 4.5 4 -4.5 -4 + * 4.51 5 -4.51 -5 + * + */ public static final int ROUND_HALF_EVEN = BigDecimal.ROUND_HALF_EVEN; - private BigDecimal bd = BigDecimal.ZERO; + //----------------------------------------------------------------------------------------------- + // Constructors are marked private; use create methods. + //----------------------------------------------------------------------------------------------- + + private HiveDecimal() { + super(); + } + + private HiveDecimal(HiveDecimal dec) { + super(dec); + } + + private HiveDecimal(FastHiveDecimal fastDec) { + super(fastDec); + } + + private HiveDecimal(int fastSignum, FastHiveDecimal fastDec) { + super(fastSignum, fastDec); + } + + private HiveDecimal( + int fastSignum, long fast0, long fast1, long fast2, + int fastIntegerDigitCount, int fastScale) { + super(fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale); + } + + //----------------------------------------------------------------------------------------------- + // Create methods. + //----------------------------------------------------------------------------------------------- + + /** + * Create a HiveDecimal from a FastHiveDecimal object. Used by HiveDecimalWritable. + * @param fastDec + * @return + */ + public static HiveDecimal createFromFast(FastHiveDecimal fastDec) { + return new HiveDecimal(fastDec); + } + + /** + * Create a HiveDecimal from BigDecimal object. + * + * A BigDecimal object has a decimal scale. + * + * We will have overflow if BigDecimal's integer part exceed MAX_PRECISION digits or + * 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1. + * + * When the BigDecimal value's precision exceeds MAX_PRECISION and there are fractional digits + * because of scale > 0, then lower digits are trimmed off with rounding to meet the + * MAX_PRECISION requirement. + * + * Also, BigDecimal supports negative scale -- which means multiplying the value by 10^abs(scale). + * And, BigDecimal allows for a non-zero scale for zero. We normalize that so zero always has + * scale 0. + * + * @param bigDecimal + * @return The HiveDecimal with the BigDecimal's value adjusted down to a maximum precision. + * Otherwise, null is returned for overflow. + */ + public static HiveDecimal create(BigDecimal bigDecimal) { + return create(bigDecimal, true); + } + + /** + * Same as the above create method, except fractional digit rounding can be turned off. + * @param bigDecimal + * @param allowRounding + * @return + */ + public static HiveDecimal create(BigDecimal bigDecimal, boolean allowRounding) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBigDecimal( + bigDecimal, allowRounding)) { + return null; + } + return result; + } + + /** + * Creates a HiveDecimal from a BigInteger's value with a scale of 0. + * + * We will have overflow if BigInteger exceed MAX_PRECISION digits or + * 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1. + * + * @param bigInteger + * @return A HiveDecimal object with the exact BigInteger's value. + * Otherwise, null is returned on overflow. + */ + public static HiveDecimal create(BigInteger bigInteger) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBigInteger( + bigInteger)) { + return null; + } + return result; + } + + /** + * Creates a HiveDecimal from a BigInteger's value with a specified scale. + * + * We will have overflow if BigInteger exceed MAX_PRECISION digits or + * 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1. + * + * The resulting decimal will have fractional digits when the specified scale is greater than 0. + * + * When the BigInteger's value's precision exceeds MAX_PRECISION and there are fractional digits + * because of scale > 0, then lower digits are trimmed off with rounding to meet the + * MAX_PRECISION requirement. + * + * @param bigInteger + * @param scale + * @return A HiveDecimal object with the BigInteger's value adjusted for scale. + * Otherwise, null is returned on overflow. + */ + public static HiveDecimal create(BigInteger bigInteger, int scale) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBigIntegerAndScale( + bigInteger, scale)) { + return null; + } + return result; + } + + /** + * Create a HiveDecimal by parsing a whole string. + * + * We support parsing a decimal with an exponent because the previous version + * (i.e. OldHiveDecimal) uses the BigDecimal parser and was able to. + * + * @param string + * @return + */ + public static HiveDecimal create(String string) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromString( + string, true)) { + return null; + } + /* + if (!result.fastIsValid()) { + result.fastRaiseInvalidException(); + } + */ + return result; + } + + /** + * Same as the method above, except blanks before and after are tolerated. + * @param string + * @param trimBlanks + * @return + */ + public static HiveDecimal create(String string, boolean trimBlanks) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromString( + string, trimBlanks)) { + return null; + } + return result; + } + + /** + * Create a HiveDecimal by parsing the characters in a whole byte array. + * + * Same rules as create(String string) above. + * + * @param bytes + * @return + */ + public static HiveDecimal create(byte[] bytes) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBytes( + bytes, 0, bytes.length, false)) { + return null; + } + return result; + } - private HiveDecimal(BigDecimal bd) { - this.bd = bd; + /** + * Same as the method above, except blanks before and after are tolerated. + * + * @param bytes + * @param trimBlanks + * @return + */ + public static HiveDecimal create(byte[] bytes, boolean trimBlanks) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBytes( + bytes, 0, bytes.length, trimBlanks)) { + return null; + } + return result; } - public static HiveDecimal create(BigDecimal b) { - return create(b, true); + public static HiveDecimal create(boolean isNegative, byte[] bytes, int scale) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromDigitsOnlyBytesAndScale( + isNegative, bytes, 0, bytes.length, scale)) { + return null; + } + if (isNegative) { + result.fastNegate(); + } + return result; } - public static HiveDecimal create(BigDecimal b, boolean allowRounding) { - BigDecimal bd = normalize(b, allowRounding); - return bd == null ? null : new HiveDecimal(bd); + /** + * Create a HiveDecimal by parsing the characters in a slice of a byte array. + * + * Same rules as create(String string) above. + * + * @param bytes + * @param offset + * @param length + * @return + */ + public static HiveDecimal create(byte[] bytes, int offset, int length) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBytes( + bytes, offset, length, false)) { + return null; + } + return result; } - public static HiveDecimal create(BigInteger unscaled, int scale) { - BigDecimal bd = normalize(new BigDecimal(unscaled, scale), true); - return bd == null ? null : new HiveDecimal(bd); + /** + * Same as the method above, except blanks before and after are tolerated. + * + * @param bytes + * @param offset + * @param length + * @param trimBlanks + * @return + */ + public static HiveDecimal create( + byte[] bytes, int offset, int length, boolean trimBlanks) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBytes( + bytes, offset, length, trimBlanks)) { + return null; + } + return result; } - public static HiveDecimal create(String dec) { - BigDecimal bd; - try { - bd = new BigDecimal(dec.trim()); - } catch (NumberFormatException ex) { + public static HiveDecimal create( + boolean isNegative, byte[] bytes, int offset, int length, int scale) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromDigitsOnlyBytesAndScale( + isNegative, bytes, offset, length, scale)) { return null; } - bd = normalize(bd, true); - return bd == null ? null : new HiveDecimal(bd); + return result; } - public static HiveDecimal create(BigInteger bi) { - BigDecimal bd = normalize(new BigDecimal(bi), true); - return bd == null ? null : new HiveDecimal(bd); + /** + * Create a HiveDecimal object from an int. + * @param intValue + * @return + */ + public static HiveDecimal create(int intValue) { + HiveDecimal result = new HiveDecimal(); + result.fastSetFromInt(intValue); + return result; } - public static HiveDecimal create(int i) { - return new HiveDecimal(new BigDecimal(i)); + /** + * Create a HiveDecimal object from a long. + * @param longValue + * @return + */ + public static HiveDecimal create(long longValue) { + HiveDecimal result = new HiveDecimal(); + result.fastSetFromLong(longValue); + return result; } - public static HiveDecimal create(long l) { - return new HiveDecimal(new BigDecimal(l)); + /** + * Create a HiveDecimal object from a long with a specified scale. + * @param longValue + * @return + */ + public static HiveDecimal create(long longValue, int scale) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromLongAndScale( + longValue, scale)) { + return null; + } + return result; } + /** + * Create a HiveDecimal object from a float. + * @param floatValue + * @return + */ + public static HiveDecimal create(float floatValue) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromFloat(floatValue)) { + return null; + } + return result; + } + + /** + * Create a HiveDecimal object from a double. + * @param doubleValue + * @return + */ + public static HiveDecimal create(double doubleValue) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromDouble(doubleValue)) { + return null; + } + return result; + } + + //----------------------------------------------------------------------------------------------- + // Serialization methods. + //----------------------------------------------------------------------------------------------- + + // The byte length of the scratch byte array that needs to be passed to serializationUtilsRead. + public static final int SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ = + FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ; + + /** + * Deserialize data written in the format used by the SerializationUtils methods + * readBigInteger/writeBigInteger and create a decimal using the supplied scale. + * + * ORC uses those SerializationUtils methods for its serialization. + * + * A scratch bytes array is necessary to do the binary to decimal conversion for better + * performance. Pass a SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ byte array for scratchBytes. + * + * @param inputStream + * @param scale + * @param scratchBytes + * @return + */ + public static HiveDecimal serializationUtilsRead( + InputStream inputStream, int scale, + byte[] scratchBytes) + throws IOException { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSerializationUtilsRead( + inputStream, scale, + scratchBytes)) { + return null; + } + return result; + } + + /** + * Convert bytes in the format used by BigInteger's toByteArray format (and accepted by its + * constructor) into a decimal using the specified scale. + * + * Our bigIntegerBytes methods create bytes in this format, too. + * + * This method is designed for high performance and does not create an actual BigInteger during + * binary to decimal conversion. + * + * @param bytes + * @param scale + * @return + */ + public static HiveDecimal createFromBigIntegerBytesAndScale( + byte[] bytes, int scale) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBigIntegerBytesAndScale( + bytes, 0, bytes.length, scale)) { + return null; + } + return result; + } + + public static HiveDecimal createFromBigIntegerBytesAndScale( + byte[] bytes, int offset, int length, int scale) { + HiveDecimal result = new HiveDecimal(); + if (!result.fastSetFromBigIntegerBytesAndScale( + bytes, offset, length, scale)) { + return null; + } + return result; + } + + public static final int SCRATCH_LONGS_LEN = FAST_SCRATCH_LONGS_LEN; + + /** + * Serialize this decimal's BigInteger equivalent unscaled value using the format that the + * SerializationUtils methods readBigInteger/writeBigInteger use. + * + * ORC uses those SerializationUtils methods for its serialization. + * + * Scratch objects necessary to do the decimal to binary conversion without actually creating a + * BigInteger object are passed for better performance. + * + * Allocate scratchLongs with SCRATCH_LONGS_LEN longs. + * + * @param outputStream + * @param scratchLongs + * @param fastScratch + */ + public boolean serializationUtilsWrite( + OutputStream outputStream, + long[] scratchLongs) + throws IOException { + return + fastSerializationUtilsWrite( + outputStream, + scratchLongs); + } + + public static final int SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES = FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES; + + /** + * Return binary representation of this decimal's BigInteger equivalent unscaled value using + * the format that the BigInteger's toByteArray method returns (and the BigInteger constructor + * accepts). + * + * Scratch objects necessary to do the decimal to binary conversion without actually creating a + * BigInteger object are passed for better performance. + * + * Allocate scratchLongs with SCRATCH_LONGS_LEN longs. + * And, allocate buffer with SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes. + * + * @param scratchLongs + * @param buffer + * @return The number of bytes used for the binary result in buffer. Otherwise, 0 if the + * conversion failed. + */ + public int bigIntegerBytes( + long[] scratchLongs, byte[] buffer) { + return + fastBigIntegerBytes( + scratchLongs, buffer); + } + + + + public byte[] bigIntegerBytes() { + long[] scratchLongs = new long[SCRATCH_LONGS_LEN]; + byte[] buffer = new byte[SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; + int byteLength = + fastBigIntegerBytes( + scratchLongs, buffer); + return Arrays.copyOfRange(buffer, 0, byteLength); + } + + // NOTE: This emulates the OldHiveDecimal setScale / OldHiveDecimal internalStorage + public int bigIntegerBytesScaled( + int scale, + long[] scratchLongs, byte[] buffer) { + return + fastBigIntegerBytesScaled( + scale, + scratchLongs, buffer); + } + + // NOTE: This emulates the OldHiveDecimal setScale / OldHiveDecimal internalStorage + public byte[] bigIntegerBytesScaled(int scale) { + long[] scratchLongs = new long[SCRATCH_LONGS_LEN]; + byte[] buffer = new byte[SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; + int byteLength = + fastBigIntegerBytesScaled( + scale, + scratchLongs, buffer); + return Arrays.copyOfRange(buffer, 0, byteLength); + } + + //----------------------------------------------------------------------------------------------- + // Convert to string methods. + //----------------------------------------------------------------------------------------------- + + /** + * Return a string representation of the decimal. + * + * If setScale(int newScale) was used to create the object, then trailing fractional digits can + * appear at the end of the result string. See the comments for that method. + * + * @return + */ @Override public String toString() { - return bd.toPlainString(); + return + fastToString(); + } + + public String toString( + byte[] scratchBuffer) { + return + fastToString( + scratchBuffer); } - + /** - * Return a string representation of the number with the number of decimal digits as - * the given scale. Please note that this is different from toString(). - * @param scale the number of digits after the decimal point - * @return the string representation of exact number of decimal digits + * Return a string representation of the decimal using the specified scale. + * + * This method is designed to ALWAYS SUCCEED (unless the newScale parameter is out of range). + * + * Is does the equivalent of a setScale(int newScale). So, more than 38 digits may be returned. + * See that method for more details on how this can happen. + * + * @param scale The number of digits after the decimal point + * @return The scaled decimal representation string representation. */ - public String toFormatString(int scale) { - return (bd.scale() == scale ? bd : - bd.setScale(scale, RoundingMode.HALF_UP)).toPlainString(); + public String toFormatString(int formatScale) { + return + fastToFormatString( + formatScale); + } + + public String toDigitsOnlyString() { + return + fastToDigitsOnlyString(); + } + + public final static int SCRATCH_BUFFER_LEN_TO_BYTES = FAST_SCRATCH_BUFFER_LEN_TO_BYTES; + + public int toBytes( + byte[] scratchBuffer) { + return + fastToBytes( + scratchBuffer); + } + + public int toFormatBytes( + int formatScale, + byte[] scratchBuffer) { + return + fastToFormatBytes( + formatScale, + scratchBuffer); } - public HiveDecimal setScale(int i) { - return new HiveDecimal(bd.setScale(i, RoundingMode.HALF_UP)); + public int toDigitsOnlyBytes( + byte[] scratchBuffer) { + return + fastToDigitsOnlyBytes( + scratchBuffer); } + //----------------------------------------------------------------------------------------------- + // Comparison methods. + //----------------------------------------------------------------------------------------------- + @Override public int compareTo(HiveDecimal dec) { - return bd.compareTo(dec.bd); + /* + if (!fastIsValid()) { + fastRaiseInvalidException(); + } + */ + return fastCompareTo(dec); } @Override public int hashCode() { - return bd.hashCode(); + return fastHashCode(); + } + + public int oldHiveDecimalHashCode() { + return fastOldHiveDecimalHashCode(); } @Override @@ -132,207 +770,533 @@ public boolean equals(Object obj) { if (obj == null || obj.getClass() != getClass()) { return false; } - return bd.equals(((HiveDecimal) obj).bd); + HiveDecimal rightDec = (HiveDecimal) obj; + return fastEquals(rightDec); } + + //----------------------------------------------------------------------------------------------- + // Attribute methods. + //----------------------------------------------------------------------------------------------- + + // NOTE: Include trailing zeroes when object produces by setScale. public int scale() { - return bd.scale(); + return fastScale(); + } + + public int integerDigitCount() { + return fastIntegerDigitCount(); } /** * Returns the number of digits (integer and fractional) in the number, which is equivalent - * to SQL decimal precision. Note that this is different from BigDecimal.precision(), - * which returns the precision of the unscaled value (BigDecimal.valueOf(0.01).precision() = 1, - * whereas HiveDecimal.create("0.01").precision() = 2). - * If you want the BigDecimal precision, use HiveDecimal.bigDecimalValue().precision() + * to SQL decimal precision. + * + * Note that this method is different from rawPrecision(), which returns the number of digits + * ignoring the scale. Note that rawPrecision returns 0 when the value is 0. + * + * Decimal sqlPrecision rawPrecision + * 0 1 0 + * 1 1 1 + * -7 1 1 + * 0.1 1 1 + * 0.04 2 1 + * 0.00380 5 3 + * 104.0009 7 7 + * + * If you just want the actual number of digits, use rawPrecision(). + * * @return */ - public int precision() { - int bdPrecision = bd.precision(); - int bdScale = bd.scale(); + public int sqlPrecision() { + return fastSqlPrecision(); + } - if (bdPrecision < bdScale) { - // This can happen for numbers less than 0.1 - // For 0.001234: bdPrecision=4, bdScale=6 - // In this case, we'll set the type to have the same precision as the scale. - return bdScale; - } - return bdPrecision; + // See comments for sqlPrecision. + public int rawPrecision() { + return fastRawPrecision(); } - /** Note - this method will corrupt the value if it doesn't fit. */ - public int intValue() { - return bd.intValue(); + /** + * Get the sign of the decimal. + * @return 0 if the decimal is equal to 0, -1 if less than zero, and 1 if greater than 0 + */ + public int signum() { + return fastSignum(); } - public double doubleValue() { - return bd.doubleValue(); + //----------------------------------------------------------------------------------------------- + // Value conversion methods. + //----------------------------------------------------------------------------------------------- + + /** + * Is the decimal value a byte? Range -128 to 127. + * Byte.MIN_VALUE Byte.MAX_VALUE + * + * Emulates testing for no value corruption: + * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().byteValue())) + * + * NOTE: Fractional digits are ignored in the test since shortValue() will + * remove them (no rounding). + * @return True when shortValue() will return a correct short. + */ + public boolean isByte() { + return fastIsByte(); } - /** Note - this method will corrupt the value if it doesn't fit. */ - public long longValue() { - return bd.longValue(); + // A byte variation of longValue(). + // This method will return a corrupted value unless isByte() is true. + public byte byteValue() { + return fastByteValueClip(); } - /** Note - this method will corrupt the value if it doesn't fit. */ + /** + * Is the decimal value a short? Range -32,768 to 32,767. + * Short.MIN_VALUE Short.MAX_VALUE + * + * Emulates testing for no value corruption: + * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().shortValue())) + * + * NOTE: Fractional digits are ignored in the test since shortValue() will + * remove them (no rounding). + * @return True when shortValue() will return a correct short. + */ + public boolean isShort() { + return fastIsShort(); + } + + // A short variation of longValue(). + // This method will return a corrupted value unless isShort() is true. public short shortValue() { - return bd.shortValue(); + return fastShortValueClip(); } - public float floatValue() { - return bd.floatValue(); + /** + * Is the decimal value a int? Range -2,147,483,648 to 2,147,483,647. + * Integer.MIN_VALUE Integer.MAX_VALUE + * + * Emulates testing for no value corruption: + * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().intValue())) + * + * NOTE: Fractional digits are ignored in the test since intValue() will + * remove them (no rounding). + * @return True when intValue() will return a correct int. + */ + public boolean isInt() { + return fastIsInt(); } - public BigDecimal bigDecimalValue() { - return bd; + // An int variation of longValue(). + // This method will return a corrupted value unless isInt() is true. + public int intValue() { + return fastIntValueClip(); } - public byte byteValue() { - return bd.byteValue(); + /** + * Is the decimal value a long? Range -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807. + * Long.MIN_VALUE Long.MAX_VALUE + * + * Emulates testing for no value corruption: + * bigDecimalValue().setScale(0).equals(BigDecimal.valueOf(bigDecimalValue().longValue())) + * + * NOTE: Fractional digits are ignored in the test since longValue() will + * remove them (no rounding). + * @return True when longValue() will return a correct long. + */ + public boolean isLong() { + return fastIsLong(); } - public HiveDecimal setScale(int adjustedScale, int rm) { - return create(bd.setScale(adjustedScale, rm)); + // This method will return a corrupted value unless isLong() is true. + public long longValue() { + return fastLongValueClip(); } - public HiveDecimal subtract(HiveDecimal dec) { - return create(bd.subtract(dec.bd)); + /** + * Return a float representing the decimal. Due the limitations of float, some values will not + * be accurate. + * @return + */ + public float floatValue() { + return fastFloatValue(); + } + + /** + * Return a double representing the decimal. Due the limitations of double, some values will not + * be accurate. + * @return + */ + public double doubleValue() { + return fastDoubleValue(); } - public HiveDecimal multiply(HiveDecimal dec) { - return create(bd.multiply(dec.bd), false); + /** + * Return a BigDecimal representing the decimal. The BigDecimal class is able to accurately + * represent the decimal. + * + * NOTE: We are not representing our decimal as BigDecimal now as OldHiveDecimal did, so this + * is now slower. + * + * @return + */ + public BigDecimal bigDecimalValue() { + return fastBigDecimalValue(); } + /** + * Get a BigInteger representing the decimal's digits without a dot. + * + * @return Returns a signed BigInteger. + */ public BigInteger unscaledValue() { - return bd.unscaledValue(); + return fastBigIntegerValue(); } - public HiveDecimal scaleByPowerOfTen(int n) { - return create(bd.scaleByPowerOfTen(n)); + /** + * Return a decimal with only the fractional digits. + * + * Zero is returned when there are no fractional digits (i.e. scale is 0). + * @return + */ + public HiveDecimal fractionPortion() { + HiveDecimal result = new HiveDecimal(); + result.fastFractionPortion(); + return result; } - public HiveDecimal abs() { - return create(bd.abs()); + /** + * Return a decimal with only the integer digits. + * + * Any fractional digits are removed. E.g. 2.083 scale 3 returns as 2 scale 0. + * @return + */ + public HiveDecimal integerPortion() { + HiveDecimal result = new HiveDecimal(); + result.fastIntegerPortion(); + return result; } - public HiveDecimal negate() { - return create(bd.negate()); - } + //----------------------------------------------------------------------------------------------- + // Math methods. + //----------------------------------------------------------------------------------------------- + /** + * Add the current decimal and another decimal and return the result. + * + * NOTE: Scale Determination for Addition/Subtraction + * + * One could take the Math.min of the scales and adjust the operand with the lower scale have a + * scale = higher scale. + * + * But this does not seem to work with decimals with widely varying scales as these: + * + * 598575157855521918987423259.94094 dec1 (int digits 27,scale 5) + * + 0.0000000000006711991169422033 dec2 (int digits 0, scale 28) + * + * Trying to make dec1 to have a scale of 28 (i.e. by adding trailing zeroes) would exceed + * MAX_PRECISION (int digits 27 + 28 > 38). + * + * In this example we need to make sure we have enough integer digit room in the result to + * handle dec1's digits. In order to maintain that, we will need to get rid of lower + * fractional digits of dec2. But when do we do that? + * + * OldHiveDecimal.add does the full arithmetic add with all the digits using BigDecimal and + * then adjusts the result to fit in MAX_PRECISION, etc. + * + * If we try to do pre-rounding dec2 it is problematic. We'd need to know if there is a carry in + * the arithmetic in order to know at which scale to do the rounding. This gets complicated. + * + * So, the simplest thing is to emulate what OldHiveDecimal does and do the full digit addition + * and then fit the result afterwards. + * + * @param dec + * @return + */ public HiveDecimal add(HiveDecimal dec) { - return create(bd.add(dec.bd)); + HiveDecimal result = new HiveDecimal(); + if (!fastAdd( + dec, + result)) { + return null; + } + return result; } - public HiveDecimal pow(int n) { - BigDecimal result = normalize(bd.pow(n), false); - return result == null ? null : new HiveDecimal(result); + /** + * Subtract from the current decimal another decimal and return the result. + * @param dec + * @return + */ + public HiveDecimal subtract(HiveDecimal dec) { + HiveDecimal result = new HiveDecimal(); + if (!fastSubtract( + dec, + result)) { + return null; + } + return result; } - public HiveDecimal remainder(HiveDecimal dec) { - return create(bd.remainder(dec.bd)); + /** + * Multiply two decimals. + * + * NOTE: Overflow Determination for Multiply + * + * OldDecimal.multiply performs the multiply with BigDecimal but DOES NOT ALLOW ROUNDING + * (i.e. no throwing away lower fractional digits). + * CONSIDER: Allowing rounding. + * + * IMPLEMENTATION NOTE: OldDecimal code does this: + * return create(bd.multiply(dec.bd), false); + * + * @param dec + * @return + */ + public HiveDecimal multiply(HiveDecimal dec) { + HiveDecimal result = new HiveDecimal(); + if (!fastMultiply( + dec, + result)) { + return null; + } + return result; } - public HiveDecimal divide(HiveDecimal dec) { - return create(bd.divide(dec.bd, MAX_SCALE, RoundingMode.HALF_UP), true); + /** + * Multiplies a decimal by a power of 10. + * + * The decimal 19350 scale 0 will return 193.5 scale 1 when power is -2 (negative). + * + * The decimal 1.000923 scale 6 will return 10009.23 scale 2 when power is 4 (positive). + * + * @param power + * @return Returns a HiveDecimal whose value is value * 10^power. + */ + public HiveDecimal scaleByPowerOfTen(int power) { + if (power == 0 || fastSignum() == 0) { + // No change for multiply by 10^0 or value 0. + return this; + } + HiveDecimal result = new HiveDecimal(); + if (!fastScaleByPowerOfTen( + power, + result)) { + return null; + } + return result; } /** - * Get the sign of the underlying decimal. - * @return 0 if the decimal is equal to 0, -1 if less than zero, and 1 if greater than 0 + * Take the absolute value of a decimal. + * @return When the decimal is negative, returns a new HiveDecimal with the positive value. + * Otherwise, returns the current 0 or positive value object; */ - public int signum() { - return bd.signum(); - } - - private static BigDecimal trim(BigDecimal d) { - if (d.compareTo(BigDecimal.ZERO) == 0) { - // Special case for 0, because java doesn't strip zeros correctly on that number. - d = BigDecimal.ZERO; - } else { - d = d.stripTrailingZeros(); - if (d.scale() < 0) { - // no negative scale decimals - d = d.setScale(0); - } + public HiveDecimal abs() { + if (fastSignum() != -1) { + return this; } - return d; + HiveDecimal result = new HiveDecimal(this); + result.fastAbs(); + return result; } - private static BigDecimal normalize(BigDecimal bd, boolean allowRounding) { - if (bd == null) { - return null; + /** + * Reverse the sign. + * @return Returns a new decimal with the sign flipped. When the value is 0, the current + * object is returned. + */ + public HiveDecimal negate() { + if (fastSignum() == 0) { + return this; } + HiveDecimal result = new HiveDecimal(this); + result.fastNegate(); + return result; + } - bd = trim(bd); + //----------------------------------------------------------------------------------------------- + // Rounding methods. + //----------------------------------------------------------------------------------------------- - int intDigits = bd.precision() - bd.scale(); + /** + * When the roundingPoint is 0 or positive, we round away lower fractional digits if the + * roundingPoint is less than current scale. In this case, we will round the result using the + * specified rounding mode. + * + * When the roundingPoint is negative, the rounding will occur within the integer digits. Integer + * digits below the roundPoint will be cleared. If the rounding occurred, a one will be added + * just above the roundingPoint. Note this may cause overflow. + * + * No effect when the roundingPoint equals the current scale. The current object is returned. + * + * @param roundingPoint + * @param roundingMode + * @return + */ + public HiveDecimal round( + int roundingPoint, int roundingMode) { + if (fastScale() == roundingPoint) { + // No change. + return this; + } - if (intDigits > MAX_PRECISION) { + // Even if we are just setting the scale when newScale is greater than the current scale, + // we need a new object to obey our immutable behavior. + HiveDecimal result = new HiveDecimal(); + if (!fastRound( + roundingPoint, roundingMode, + result)) { return null; } + return result; + } - int maxScale = Math.min(MAX_SCALE, Math.min(MAX_PRECISION - intDigits, bd.scale())); - if (bd.scale() > maxScale ) { - if (allowRounding) { - bd = bd.setScale(maxScale, RoundingMode.HALF_UP); - // Trimming is again necessary, because rounding may introduce new trailing 0's. - bd = trim(bd); - } else { - bd = null; - } + /** + * Return the result of decimal^exponent + * + * CONSIDER: Currently, negative exponent is not supported. + * CONSIDER: Does anybody use this method? + * + * @param exponent + * @return + */ + public HiveDecimal pow(int exponent) { + HiveDecimal result = new HiveDecimal(this); + if (!fastPow( + exponent, result)) { + return null; } - - return bd; + return result; } - private static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale) { - if (bd == null) { + /** + * Divides this decimal by another decimal and returns a new decimal with the result. + * @param dec + * @return + */ + public HiveDecimal divide(HiveDecimal divisor) { + HiveDecimal result = new HiveDecimal(); + if (!fastDivide( + divisor, + result)) { return null; } + return result; + } - /** - * Specially handling the case that bd=0, and we are converting it to a type where precision=scale, - * such as decimal(1, 1). - */ - if (bd.compareTo(BigDecimal.ZERO) == 0 && bd.scale() == 0 && maxPrecision == maxScale) { - return bd.setScale(maxScale); + /** + * Divides this decimal by another decimal and returns a new decimal with the remainder of the + * division. + * + * value is (decimal % divisor) + * + * The remainder is equivalent to BigDecimal: + * bigDecimalValue().subtract(bigDecimalValue().divideToIntegralValue(divisor).multiply(divisor)) + * + * @param dec + * @return + */ + public HiveDecimal remainder(HiveDecimal divisor) { + HiveDecimal result = new HiveDecimal(); + if (!fastRemainder( + divisor, + result)) { + return null; } + return result; + } - bd = trim(bd); + //----------------------------------------------------------------------------------------------- + // Precision/scale enforcement methods. + //----------------------------------------------------------------------------------------------- - if (bd.scale() > maxScale) { - bd = bd.setScale(maxScale, RoundingMode.HALF_UP); + /** + * Determine if a decimal fits within a specified maxPrecision and maxScale, and round + * off fractional digits if necessary to make the decimal fit. + * + * The relationship between the enforcement maxPrecision and maxScale is restricted. The + * specified maxScale must be less than or equal to the maxPrecision. + * + * Normally, decimals that result from creation operation, arithmetic operations, etc are + * "free range" up to MAX_PRECISION and MAX_SCALE. Each operation checks if the result decimal + * is beyond MAX_PRECISION and MAX_SCALE. If so the result decimal is rounded off using + * ROUND_HALF_UP. If the round digit is 5 or more, one is added to the lowest remaining digit. + * The round digit is the digit just below the round point. Result overflow can occur if a + * result decimal's integer portion exceeds MAX_PRECISION. + * + * This method supports enforcing to a declared Hive DECIMAL's precision/scale. + * E.g. DECIMAL(10,4) + * + * Here are the enforcement/rounding checks of this method: + * + * 1) Maximum integer digits = maxPrecision - maxScale + * + * If the decimal's integer digit count exceeds this, the decimal does not fit (overflow). + * + * 2) If decimal's scale is greater than maxScale, then excess fractional digits are + * rounded off. When rounding increases the remaining decimal, it may exceed the + * limits and overflow. + * + * @param dec + * @param maxPrecision + * @param setScale + * @return The original decimal if no adjustment is necessary. + * A rounded off decimal if adjustment was necessary. + * Otherwise, null if the decimal doesn't fit within maxPrecision / maxScale or rounding + * caused a result that exceeds the specified limits or MAX_PRECISION integer digits. + */ + public static HiveDecimal enforcePrecisionScale( + HiveDecimal dec, int maxPrecision, int maxScale) { + + if (maxPrecision < 1 || maxPrecision > MAX_PRECISION) { + throw new IllegalArgumentException(STRING_ENFORCE_PRECISION_OUT_OF_RANGE); } - int maxIntDigits = maxPrecision - maxScale; - int intDigits = bd.precision() - bd.scale(); - if (intDigits > maxIntDigits) { - return null; + if (maxScale < 0 || maxScale > HiveDecimal.MAX_SCALE) { + throw new IllegalArgumentException(STRING_ENFORCE_SCALE_OUT_OF_RANGE); } - return bd; - } + if (maxPrecision < maxScale) { + throw new IllegalArgumentException(STRING_ENFORCE_SCALE_LESS_THAN_EQUAL_PRECISION); + } - public static HiveDecimal enforcePrecisionScale(HiveDecimal dec, int maxPrecision, int maxScale) { if (dec == null) { return null; } - // Minor optimization, avoiding creating new objects. - if (dec.precision() - dec.scale() <= maxPrecision - maxScale && - dec.scale() <= maxScale) { + FastCheckPrecisionScaleStatus status = + dec.fastCheckPrecisionScale( + maxPrecision, maxScale); + switch (status) { + case NO_CHANGE: return dec; - } - - BigDecimal bd = enforcePrecisionScale(dec.bd, maxPrecision, maxScale); - if (bd == null) { + case OVERFLOW: return null; + case UPDATE_SCALE_DOWN: + { + HiveDecimal result = new HiveDecimal(); + if (!dec.fastUpdatePrecisionScale( + maxPrecision, maxScale, status, + result)) { + return null; + } + return result; + } + default: + throw new RuntimeException("Unknown fast decimal check precision and scale status " + status); } - - return HiveDecimal.create(bd); } - public long longValueExact() { - return bd.longValueExact(); + //----------------------------------------------------------------------------------------------- + // Validation methods. + //----------------------------------------------------------------------------------------------- + + /** + * Throws an exception if the current decimal value is invalid. + */ + public void validate() { + if (!fastIsValid()) { + fastRaiseInvalidException(); + } } -} +} \ No newline at end of file diff --git storage-api/src/java/org/apache/hadoop/hive/common/type/OldHiveDecimal.java storage-api/src/java/org/apache/hadoop/hive/common/type/OldHiveDecimal.java new file mode 100644 index 0000000..07333bf --- /dev/null +++ storage-api/src/java/org/apache/hadoop/hive/common/type/OldHiveDecimal.java @@ -0,0 +1,338 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.type; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; + +/** + * + * HiveDecimal. Simple wrapper for BigDecimal. Adds fixed max precision and non scientific string + * representation + * + */ +public class OldHiveDecimal implements Comparable { + public static final int MAX_PRECISION = 38; + public static final int MAX_SCALE = 38; + + /** + * Default precision/scale when user doesn't specify in the column metadata, such as + * decimal and decimal(8). + */ + public static final int USER_DEFAULT_PRECISION = 10; + public static final int USER_DEFAULT_SCALE = 0; + + /** + * Default precision/scale when system is not able to determine them, such as in case + * of a non-generic udf. + */ + public static final int SYSTEM_DEFAULT_PRECISION = 38; + public static final int SYSTEM_DEFAULT_SCALE = 18; + + public static final OldHiveDecimal ZERO = new OldHiveDecimal(BigDecimal.ZERO); + public static final OldHiveDecimal ONE = new OldHiveDecimal(BigDecimal.ONE); + + public static final int ROUND_FLOOR = BigDecimal.ROUND_FLOOR; + public static final int ROUND_CEILING = BigDecimal.ROUND_CEILING; + public static final int ROUND_HALF_UP = BigDecimal.ROUND_HALF_UP; + public static final int ROUND_HALF_EVEN = BigDecimal.ROUND_HALF_EVEN; + + private BigDecimal bd = BigDecimal.ZERO; + + private OldHiveDecimal(BigDecimal bd) { + this.bd = bd; + } + + public static OldHiveDecimal create(BigDecimal b) { + return create(b, true); + } + + public static OldHiveDecimal create(BigDecimal b, boolean allowRounding) { + BigDecimal bd = normalize(b, allowRounding); + return bd == null ? null : new OldHiveDecimal(bd); + } + + public static OldHiveDecimal create(BigInteger unscaled, int scale) { + BigDecimal bd = normalize(new BigDecimal(unscaled, scale), true); + return bd == null ? null : new OldHiveDecimal(bd); + } + + public static OldHiveDecimal create(String dec) { + BigDecimal bd; + try { + bd = new BigDecimal(dec.trim()); + } catch (NumberFormatException ex) { + return null; + } + bd = normalize(bd, true); + return bd == null ? null : new OldHiveDecimal(bd); + } + + public static OldHiveDecimal create(BigInteger bi) { + BigDecimal bd = normalize(new BigDecimal(bi), true); + return bd == null ? null : new OldHiveDecimal(bd); + } + + public static OldHiveDecimal create(int i) { + return new OldHiveDecimal(new BigDecimal(i)); + } + + public static OldHiveDecimal create(long l) { + return new OldHiveDecimal(new BigDecimal(l)); + } + + @Override + public String toString() { + return bd.toPlainString(); + } + + /** + * Return a string representation of the number with the number of decimal digits as + * the given scale. Please note that this is different from toString(). + * @param scale the number of digits after the decimal point + * @return the string representation of exact number of decimal digits + */ + public String toFormatString(int scale) { + return (bd.scale() == scale ? bd : + bd.setScale(scale, RoundingMode.HALF_UP)).toPlainString(); + } + + public OldHiveDecimal setScale(int i) { + return new OldHiveDecimal(bd.setScale(i, RoundingMode.HALF_UP)); + } + + @Override + public int compareTo(OldHiveDecimal dec) { + return bd.compareTo(dec.bd); + } + + @Override + public int hashCode() { + return bd.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + return bd.equals(((OldHiveDecimal) obj).bd); + } + + public int scale() { + return bd.scale(); + } + + /** + * Returns the number of digits (integer and fractional) in the number, which is equivalent + * to SQL decimal precision. Note that this is different from BigDecimal.precision(), + * which returns the precision of the unscaled value (BigDecimal.valueOf(0.01).precision() = 1, + * whereas HiveDecimal.create("0.01").precision() = 2). + * If you want the BigDecimal precision, use HiveDecimal.bigDecimalValue().precision() + * @return + */ + public int precision() { + int bdPrecision = bd.precision(); + int bdScale = bd.scale(); + + if (bdPrecision < bdScale) { + // This can happen for numbers less than 0.1 + // For 0.001234: bdPrecision=4, bdScale=6 + // In this case, we'll set the type to have the same precision as the scale. + return bdScale; + } + return bdPrecision; + } + + /** Note - this method will corrupt the value if it doesn't fit. */ + public int intValue() { + return bd.intValue(); + } + + public double doubleValue() { + return bd.doubleValue(); + } + + /** Note - this method will corrupt the value if it doesn't fit. */ + public long longValue() { + return bd.longValue(); + } + + /** Note - this method will corrupt the value if it doesn't fit. */ + public short shortValue() { + return bd.shortValue(); + } + + public float floatValue() { + return bd.floatValue(); + } + + public BigDecimal bigDecimalValue() { + return bd; + } + + public byte byteValue() { + return bd.byteValue(); + } + + public OldHiveDecimal setScale(int adjustedScale, int rm) { + return create(bd.setScale(adjustedScale, rm)); + } + + public OldHiveDecimal subtract(OldHiveDecimal dec) { + return create(bd.subtract(dec.bd)); + } + + public OldHiveDecimal multiply(OldHiveDecimal dec) { + return create(bd.multiply(dec.bd), false); + } + + public BigInteger unscaledValue() { + return bd.unscaledValue(); + } + + public OldHiveDecimal scaleByPowerOfTen(int n) { + return create(bd.scaleByPowerOfTen(n)); + } + + public OldHiveDecimal abs() { + return create(bd.abs()); + } + + public OldHiveDecimal negate() { + return create(bd.negate()); + } + + public OldHiveDecimal add(OldHiveDecimal dec) { + return create(bd.add(dec.bd)); + } + + public OldHiveDecimal pow(int n) { + BigDecimal result = normalize(bd.pow(n), false); + return result == null ? null : new OldHiveDecimal(result); + } + + public OldHiveDecimal remainder(OldHiveDecimal dec) { + return create(bd.remainder(dec.bd)); + } + + public OldHiveDecimal divide(OldHiveDecimal dec) { + return create(bd.divide(dec.bd, MAX_SCALE, RoundingMode.HALF_UP), true); + } + + /** + * Get the sign of the underlying decimal. + * @return 0 if the decimal is equal to 0, -1 if less than zero, and 1 if greater than 0 + */ + public int signum() { + return bd.signum(); + } + + private static BigDecimal trim(BigDecimal d) { + if (d.compareTo(BigDecimal.ZERO) == 0) { + // Special case for 0, because java doesn't strip zeros correctly on that number. + d = BigDecimal.ZERO; + } else { + d = d.stripTrailingZeros(); + if (d.scale() < 0) { + // no negative scale decimals + d = d.setScale(0); + } + } + return d; + } + + private static BigDecimal normalize(BigDecimal bd, boolean allowRounding) { + if (bd == null) { + return null; + } + + bd = trim(bd); + + int intDigits = bd.precision() - bd.scale(); + + if (intDigits > MAX_PRECISION) { + return null; + } + + int maxScale = Math.min(MAX_SCALE, Math.min(MAX_PRECISION - intDigits, bd.scale())); + if (bd.scale() > maxScale ) { + if (allowRounding) { + bd = bd.setScale(maxScale, RoundingMode.HALF_UP); + // Trimming is again necessary, because rounding may introduce new trailing 0's. + bd = trim(bd); + } else { + bd = null; + } + } + + return bd; + } + + private static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, int maxScale) { + if (bd == null) { + return null; + } + + /** + * Specially handling the case that bd=0, and we are converting it to a type where precision=scale, + * such as decimal(1, 1). + */ + if (bd.compareTo(BigDecimal.ZERO) == 0 && bd.scale() == 0 && maxPrecision == maxScale) { + return bd.setScale(maxScale); + } + + bd = trim(bd); + + if (bd.scale() > maxScale) { + bd = bd.setScale(maxScale, RoundingMode.HALF_UP); + } + + int maxIntDigits = maxPrecision - maxScale; + int intDigits = bd.precision() - bd.scale(); + if (intDigits > maxIntDigits) { + return null; + } + + return bd; + } + + public static OldHiveDecimal enforcePrecisionScale(OldHiveDecimal dec, int maxPrecision, int maxScale) { + if (dec == null) { + return null; + } + + // Minor optimization, avoiding creating new objects. + if (dec.precision() - dec.scale() <= maxPrecision - maxScale && + dec.scale() <= maxScale) { + return dec; + } + + BigDecimal bd = enforcePrecisionScale(dec.bd, maxPrecision, maxScale); + if (bd == null) { + return null; + } + + return OldHiveDecimal.create(bd); + } + + public long longValueExact() { + return bd.longValueExact(); + } +} diff --git storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java index 53a7823..8d950a2 100644 --- storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java +++ storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java @@ -57,19 +57,7 @@ public static String getRandString(Random r, String characters, int length) { private static final String DECIMAL_CHARS = "0123456789"; - public static class HiveDecimalAndPrecisionScale { - public HiveDecimal hiveDecimal; - public int precision; - public int scale; - - HiveDecimalAndPrecisionScale(HiveDecimal hiveDecimal, int precision, int scale) { - this.hiveDecimal = hiveDecimal; - this.precision = precision; - this.scale = scale; - } - } - - public static HiveDecimalAndPrecisionScale getRandHiveDecimal(Random r) { + public static HiveDecimal getRandHiveDecimal(Random r) { int precision; int scale; while (true) { @@ -93,18 +81,7 @@ public static HiveDecimalAndPrecisionScale getRandHiveDecimal(Random r) { sb.append(getRandString(r, DECIMAL_CHARS, scale)); } - HiveDecimal bd = HiveDecimal.create(sb.toString()); - precision = bd.precision(); - scale = bd.scale(); - if (scale > precision) { - // Sometimes weird decimals are produced? - continue; - } - - // For now, punt. - precision = HiveDecimal.SYSTEM_DEFAULT_PRECISION; - scale = HiveDecimal.SYSTEM_DEFAULT_SCALE; - return new HiveDecimalAndPrecisionScale(bd, precision, scale); + return HiveDecimal.create(sb.toString()); } } diff --git storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java index 2488631..c9a5e34 100644 --- storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java +++ storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/DecimalColumnVector.java @@ -17,10 +17,12 @@ */ package org.apache.hadoop.hive.ql.exec.vector; + import java.math.BigInteger; import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.FastHiveDecimal; public class DecimalColumnVector extends ColumnVector { @@ -45,7 +47,7 @@ public DecimalColumnVector(int size, int precision, int scale) { this.scale = (short) scale; vector = new HiveDecimalWritable[size]; for (int i = 0; i < size; i++) { - vector[i] = new HiveDecimalWritable(HiveDecimal.ZERO); + vector[i] = new HiveDecimalWritable(0); // Initially zero. } } @@ -71,15 +73,14 @@ public void setElement(int outElementNum, int inputElementNum, ColumnVector inpu inputElementNum = 0; } if (inputVector.noNulls || !inputVector.isNull[inputElementNum]) { - HiveDecimal hiveDec = - ((DecimalColumnVector) inputVector).vector[inputElementNum] - .getHiveDecimal(precision, scale); - if (hiveDec == null) { + vector[outElementNum].set( + ((DecimalColumnVector) inputVector).vector[inputElementNum], + precision, scale); + if (!vector[outElementNum].isSet()) { isNull[outElementNum] = true; noNulls = false; } else { isNull[outElementNum] = false; - vector[outElementNum].set(hiveDec); } } else { isNull[outElementNum] = true; @@ -100,34 +101,38 @@ public void stringifyValue(StringBuilder buffer, int row) { } public void set(int elementNum, HiveDecimalWritable writeable) { - if (writeable == null) { + vector[elementNum].set(writeable, precision, scale); + if (!vector[elementNum].isSet()) { noNulls = false; isNull[elementNum] = true; } else { - HiveDecimal hiveDec = writeable.getHiveDecimal(precision, scale); - if (hiveDec == null) { - noNulls = false; - isNull[elementNum] = true; - } else { - vector[elementNum].set(hiveDec); - } + isNull[elementNum] = false; } } public void set(int elementNum, HiveDecimal hiveDec) { - HiveDecimal checkedDec = HiveDecimal.enforcePrecisionScale(hiveDec, precision, scale); - if (checkedDec == null) { + vector[elementNum].set(hiveDec, precision, scale); + if (!vector[elementNum].isSet()) { + noNulls = false; + isNull[elementNum] = true; + } else { + isNull[elementNum] = false; + } + } + + public void set(int elementNum, FastHiveDecimal fastHiveDec) { + vector[elementNum].set(fastHiveDec, precision, scale); + if (!vector[elementNum].isSet()) { noNulls = false; isNull[elementNum] = true; } else { - vector[elementNum].set(checkedDec); + isNull[elementNum] = false; } } public void setNullDataValue(int elementNum) { // E.g. For scale 2 the minimum is "0.01" - HiveDecimal minimumNonZeroValue = HiveDecimal.create(BigInteger.ONE, scale); - vector[elementNum].set(minimumNonZeroValue); + vector[elementNum].setFromLongAndScale(1L, scale); } @Override @@ -144,7 +149,7 @@ public void ensureSize(int size, boolean preserveData) { System.arraycopy(oldArray, 0, vector, 0 , oldArray.length); } for (int i = initPos; i < vector.length; ++i) { - vector[i] = new HiveDecimalWritable(HiveDecimal.ZERO); + vector[i] = new HiveDecimalWritable(0); // Initially zero. } } } diff --git storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java index 41db9ca..a302bce 100644 --- storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java +++ storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hive.ql.util; import org.apache.hadoop.hive.common.type.HiveDecimal; +import org.apache.hadoop.hive.common.type.OldHiveDecimal; +import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; import java.math.BigDecimal; import java.sql.Timestamp; @@ -68,9 +70,86 @@ public static Timestamp doubleToTimestamp(double f) { } } - public static Timestamp decimalToTimestamp(HiveDecimal d) { + /** + * Take a HiveDecimal and return the timestamp representation where the fraction part is the + * nanoseconds and integer part is the number of seconds. + * @param dec + * @return + */ + public static Timestamp decimalToTimestamp(HiveDecimal dec) { + + HiveDecimalWritable nanosWritable = new HiveDecimalWritable(dec); + nanosWritable.mutateFractionPortion(); // Clip off seconds portion. + nanosWritable.mutateScaleByPowerOfTen(9); // Bring nanoseconds into integer portion. + if (!nanosWritable.isSet() || !nanosWritable.isInt()) { + return null; + } + int nanos = nanosWritable.intValue(); + if (nanos < 0) { + nanos += 1000000000; + } + nanosWritable.setFromLong(nanos); + + HiveDecimalWritable nanoInstant = new HiveDecimalWritable(dec); + nanoInstant.mutateScaleByPowerOfTen(9); + + nanoInstant.mutateSubtract(nanosWritable); + nanoInstant.mutateScaleByPowerOfTen(-9); // Back to seconds. + if (!nanoInstant.isSet() || !nanoInstant.isLong()) { + return null; + } + long seconds = nanoInstant.longValue(); + Timestamp t = new Timestamp(seconds * 1000); + t.setNanos(nanos); + return t; + } + + /** + * Take a HiveDecimalWritable and return the timestamp representation where the fraction part + * is the nanoseconds and integer part is the number of seconds. + * + * This is a HiveDecimalWritable variation with supplied scratch objects. + * @param decdecWritable + * @param scratchDecWritable1 + * @param scratchDecWritable2 + * @return + */ + public static Timestamp decimalToTimestamp( + HiveDecimalWritable decWritable, + HiveDecimalWritable scratchDecWritable1, HiveDecimalWritable scratchDecWritable2) { + + HiveDecimalWritable nanosWritable = scratchDecWritable1; + nanosWritable.set(decWritable); + nanosWritable.mutateFractionPortion(); // Clip off seconds portion. + nanosWritable.mutateScaleByPowerOfTen(9); // Bring nanoseconds into integer portion. + if (!nanosWritable.isSet() || !nanosWritable.isInt()) { + return null; + } + int nanos = nanosWritable.intValue(); + if (nanos < 0) { + nanos += 1000000000; + } + nanosWritable.setFromLong(nanos); + + HiveDecimalWritable nanoInstant = scratchDecWritable2; + nanoInstant.set(decWritable); + nanoInstant.mutateScaleByPowerOfTen(9); + + nanoInstant.mutateSubtract(nanosWritable); + nanoInstant.mutateScaleByPowerOfTen(-9); // Back to seconds. + if (!nanoInstant.isSet() || !nanoInstant.isLong()) { + return null; + } + long seconds = nanoInstant.longValue(); + + Timestamp timestamp = new Timestamp(seconds * 1000L); + timestamp.setNanos(nanos); + return timestamp; + } + + public static Timestamp decimalToTimestamp(OldHiveDecimal dec) { try { - BigDecimal nanoInstant = d.bigDecimalValue().multiply(BILLION_BIG_DECIMAL); + BigDecimal nanoInstant = dec.bigDecimalValue().multiply(BILLION_BIG_DECIMAL); int nanos = nanoInstant.remainder(BILLION_BIG_DECIMAL).intValue(); if (nanos < 0) { nanos += 1000000000; diff --git storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java index 41452da..864432e 100644 --- storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java +++ storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java @@ -17,63 +17,195 @@ */ package org.apache.hadoop.hive.serde2.io; +import java.util.Arrays; import java.io.DataInput; import java.io.DataOutput; +import java.io.InputStream; +import java.io.OutputStream; import java.io.IOException; import java.math.BigInteger; import org.apache.hadoop.hive.common.type.HiveDecimal; - +import org.apache.hadoop.hive.common.type.FastHiveDecimal; +import org.apache.hadoop.hive.common.type.FastHiveDecimalImpl; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.WritableUtils; -public class HiveDecimalWritable implements WritableComparable { +public class HiveDecimalWritable extends FastHiveDecimal + implements WritableComparable { + + private boolean isSet; - private byte[] internalStorage = new byte[0]; - private int scale; + private long[] internalScratchLongs; + private byte[] internalScratchBuffer; public HiveDecimalWritable() { + super(); + isSet = false; } - public HiveDecimalWritable(String value) { - set(HiveDecimal.create(value)); + public HiveDecimalWritable(String string) { + super(); + isSet = fastSetFromString(string, false); + if (!isSet) { + fastReset(); + } } - public HiveDecimalWritable(byte[] bytes, int scale) { - set(bytes, scale); + public HiveDecimalWritable(byte[] bigIntegerBytes, int scale) { + super(); + setFromBigIntegerBytesAndScale(bigIntegerBytes, scale); } public HiveDecimalWritable(HiveDecimalWritable writable) { - set(writable.getHiveDecimal()); + super(); + set(writable); } public HiveDecimalWritable(HiveDecimal value) { + super(); set(value); } public HiveDecimalWritable(long value) { - set((HiveDecimal.create(value))); + super(); + setFromLong(value); } public void set(HiveDecimal value) { - set(value.unscaledValue().toByteArray(), value.scale()); + if (value == null) { + fastReset(); + isSet = false; + } else { + fastSet(value); + isSet = true; + } } public void set(HiveDecimal value, int maxPrecision, int maxScale) { - set(HiveDecimal.enforcePrecisionScale(value, maxPrecision, maxScale)); + set(value); + if (isSet) { + isSet = fastEnforcePrecisionScale(maxPrecision, maxScale); + if (!isSet) { + fastReset(); + } + } } public void set(HiveDecimalWritable writable) { - set(writable.getHiveDecimal()); + if (writable == null || !writable.isSet()) { + fastReset(); + isSet = false; + } else { + fastSet(writable); + isSet = true; + } + } + + public void set(HiveDecimalWritable writable, int maxPrecision, int maxScale) { + set(writable); + if (isSet) { + isSet = fastEnforcePrecisionScale(maxPrecision, maxScale); + if (!isSet) { + fastReset(); + } + } + } + + public void setFromLong(long longValue) { + fastReset(); + fastSetFromLong(longValue); + isSet = true; + } + + public void setFromDouble(double doubleValue) { + fastReset(); + isSet = fastSetFromDouble(doubleValue); + if (!isSet) { + fastReset(); + } + } + + public void setFromBytes(byte[] bytes, int offset, int length) { + fastReset(); + isSet = fastSetFromBytes(bytes, offset, length, false); + if (!isSet) { + fastReset(); + } + } + + public void setFromBytes(byte[] bytes, int offset, int length, boolean trimBlanks) { + fastReset(); + isSet = fastSetFromBytes(bytes, offset, length, trimBlanks); + if (!isSet) { + fastReset(); + } + } + + public void setFromDigitsOnlyBytesWithScale( + boolean isNegative, byte[] bytes, int offset, int length, int scale) { + fastReset(); + isSet = fastSetFromDigitsOnlyBytesAndScale(isNegative, bytes, offset, length, scale); + if (!isSet) { + fastReset(); + } + } + + public void setFromBigIntegerBytesAndScale(byte[] bigIntegerBytes, int scale) { + fastReset(); + isSet = fastSetFromBigIntegerBytesAndScale(bigIntegerBytes, 0, bigIntegerBytes.length, scale); + if (!isSet) { + fastReset(); + } } - public void set(byte[] bytes, int scale) { - this.internalStorage = bytes; - this.scale = scale; + public void setFromBigIntegerBytesAndScale( + byte[] bigIntegerBytes, int offset, int length, int scale) { + fastReset(); + isSet = fastSetFromBigIntegerBytesAndScale(bigIntegerBytes, offset, length, scale); + if (!isSet) { + fastReset(); + } + } + + public void setFromLongAndScale(long longValue, int scale) { + fastReset(); + isSet = fastSetFromLongAndScale(longValue, scale); + if (!isSet) { + fastReset(); + } + } + + public void set(FastHiveDecimal fastValue) { + if (fastValue == null) { + fastReset(); + isSet = false; + } else { + fastSet(fastValue); + isSet = true; + } + } + + public void set(FastHiveDecimal fastValue, int maxPrecision, int maxScale) { + set(fastValue); + if (isSet) { + isSet = fastEnforcePrecisionScale(maxPrecision, maxScale); + if (!isSet) { + fastReset(); + } + } + } + + public boolean isSet() { + return isSet; } public HiveDecimal getHiveDecimal() { - return HiveDecimal.create(new BigInteger(internalStorage), scale); + if (!isSet) { + return null; + } + HiveDecimal result = HiveDecimal.createFromFast(this); + return result; } /** @@ -84,87 +216,516 @@ public HiveDecimal getHiveDecimal() { * @return HiveDecimal instance */ public HiveDecimal getHiveDecimal(int maxPrecision, int maxScale) { - return HiveDecimal.enforcePrecisionScale(HiveDecimal. - create(new BigInteger(internalStorage), scale), - maxPrecision, maxScale); + if (!isSet) { + return null; + } + HiveDecimal dec = HiveDecimal.createFromFast(this); + HiveDecimal result = HiveDecimal.enforcePrecisionScale(dec, maxPrecision, maxScale); + return result; } @Override public void readFields(DataInput in) throws IOException { - scale = WritableUtils.readVInt(in); + int scale = WritableUtils.readVInt(in); int byteArrayLen = WritableUtils.readVInt(in); - if (internalStorage.length != byteArrayLen) { - internalStorage = new byte[byteArrayLen]; + byte[] bytes = new byte[byteArrayLen]; + in.readFully(bytes); + + fastReset(); + if (!fastSetFromBigIntegerBytesAndScale(bytes, 0, bytes.length, scale)) { + throw new IOException("Couldn't convert decimal"); } - in.readFully(internalStorage); + isSet = true; } @Override public void write(DataOutput out) throws IOException { - WritableUtils.writeVInt(out, scale); - WritableUtils.writeVInt(out, internalStorage.length); - out.write(internalStorage); + if (!isSet()) { + throw new RuntimeException("no value set"); + } + + if (internalScratchLongs == null) { + internalScratchLongs = new long[FastHiveDecimal.FAST_SCRATCH_LONGS_LEN]; + internalScratchBuffer = new byte[FastHiveDecimal.FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; + } + + write(out, internalScratchLongs, internalScratchBuffer); + } + + + /** + * + * Allocate scratchLongs with HiveDecimal.SCRATCH_LONGS_LEN longs. + * And, allocate scratch buffer with HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes. + * + * @param out + * @param scratchLongs + * @param buffer + * @throws IOException + */ + public void write( + DataOutput out, + long[] scratchLongs, byte[] scratchBuffer) throws IOException { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + + WritableUtils.writeVInt(out, fastScale()); + + int byteLength = + fastBigIntegerBytes( + scratchLongs, scratchBuffer); + if (byteLength == 0) { + throw new RuntimeException("Couldn't convert decimal to binary"); + } + + WritableUtils.writeVInt(out, byteLength); + out.write(scratchBuffer, 0, byteLength); + } + + public boolean serializationUtilsRead( + InputStream inputStream, int scale, + byte[] scratchBytes) + throws IOException { + fastReset(); + isSet = + fastSerializationUtilsRead( + inputStream, + scale, + scratchBytes); + return isSet; + } + + public boolean serializationUtilsWrite( + OutputStream outputStream, + long[] scratchLongs) + throws IOException { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return + fastSerializationUtilsWrite( + outputStream, + scratchLongs); + } + + /** + * Returns the length of the decimal converted to bytes. + * Call bigIntegerBytesBuffer() to get a reference to the converted bytes. + * @return + */ + public int bigIntegerBytesInternalScratch() { + + if (!isSet()) { + throw new RuntimeException("no value set"); + } + + if (internalScratchLongs == null) { + internalScratchLongs = new long[FastHiveDecimal.FAST_SCRATCH_LONGS_LEN]; + internalScratchBuffer = new byte[FastHiveDecimal.FAST_SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES]; + } + + int byteLength = + fastBigIntegerBytes( + internalScratchLongs, internalScratchBuffer); + if (byteLength == 0) { + throw new RuntimeException("Couldn't convert decimal to binary"); + } + return byteLength; + } + + public byte[] bigIntegerBytesInternalScratchBuffer() { + return internalScratchBuffer; + } + + /** + * + * Allocate scratchLongs with HiveDecimal.SCRATCH_LONGS_LEN longs. + * And, allocate scratch scratchBuffer with HiveDecimal.SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes. + * + * @param scratchLongs + * @param scratchBuffer + * @throws IOException + */ + public byte[] bigIntegerBytesCopy( + long[] scratchLongs, byte[] scratchBuffer) { + + if (!isSet()) { + throw new RuntimeException("no value set"); + } + + int byteLength = + fastBigIntegerBytes( + scratchLongs, scratchBuffer); + if (byteLength == 0) { + throw new RuntimeException("Couldn't convert decimal to binary"); + } + return Arrays.copyOf(scratchBuffer, byteLength); + } + + public int bigIntegerBytes( + long[] scratchLongs, byte[] scratchBuffer) { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + int byteLength = + fastBigIntegerBytes( + scratchLongs, scratchBuffer); + return byteLength; + } + + public int signum() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastSignum(); + } + + public int sqlPrecision() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastSqlPrecision(); + } + + public int rawPrecision() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastRawPrecision(); + } + + public int scale() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastScale(); + } + + public boolean isByte() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastIsByte(); + } + + public byte byteValue() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastByteValueClip(); + } + + public boolean isShort() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastIsShort(); + } + + public short shortValue() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastShortValueClip(); + } + + public boolean isInt() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastIsInt(); + } + + public int intValue() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastIntValueClip(); + } + + public boolean isLong() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastIsLong(); + } + + public long longValue() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastLongValueClip(); + } + + public float floatValue() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastFloatValue(); + } + + public double doubleValue() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastDoubleValue(); + } + + public void mutateAbs() { + if (!isSet) { + return; + } + fastAbs(); + } + + public void mutateNegate() { + if (!isSet) { + return; + } + fastNegate(); + } + + public void mutateAdd(HiveDecimalWritable decWritable) { + if (!isSet || !decWritable.isSet) { + isSet = false; + return; + } + isSet = + fastAdd(decWritable, this); + } + + public void mutateAdd(HiveDecimal dec) { + if (!isSet) { + return; + } + isSet = + fastAdd(dec, this); + } + + public void mutateSubtract(HiveDecimalWritable decWritable) { + if (!isSet || !decWritable.isSet) { + isSet = false; + return; + } + isSet = + fastSubtract(decWritable, this); + } + + public void mutateSubtract(HiveDecimal dec) { + if (!isSet) { + return; + } + isSet = + fastSubtract(dec, this); + } + + public void mutateMultiply(HiveDecimalWritable decWritable) { + if (!isSet || !decWritable.isSet) { + isSet = false; + return; + } + isSet = + fastMultiply(decWritable, this); + } + + public void mutateMultiply(HiveDecimal dec) { + if (!isSet) { + return; + } + isSet = + fastMultiply(dec, this); + } + + public void mutateDivide(HiveDecimalWritable decWritable) { + if (!isSet || !decWritable.isSet) { + isSet = false; + return; + } + isSet = + fastDivide(decWritable, this); + } + + public void mutateDivide(HiveDecimal dec) { + if (!isSet) { + return; + } + isSet = + fastDivide(dec, this); + + } + + public void mutateRemainder(HiveDecimalWritable decWritable) { + if (!isSet || !decWritable.isSet) { + isSet = false; + return; + } + isSet = + fastRemainder(decWritable, this); + } + + public void mutateRemainder(HiveDecimal dec) { + if (!isSet) { + return; + } + isSet = + fastRemainder(dec, this); + } + + public void mutateScaleByPowerOfTen(int power) { + if (!isSet) { + return; + } + isSet = fastScaleByPowerOfTen(power, this); + } + + public void mutateFractionPortion() { + if (!isSet) { + return; + } + fastFractionPortion(); + } + + public void mutateIntegerPortion() { + if (!isSet) { + return; + } + fastIntegerPortion(); } @Override - public int compareTo(HiveDecimalWritable that) { - return getHiveDecimal().compareTo(that.getHiveDecimal()); + public int compareTo(HiveDecimalWritable writable) { + if (!isSet() || writable == null || !writable.isSet()) { + throw new RuntimeException("Invalid comparision operand(s)"); + } + return fastCompareTo(writable); + } + + public int compareTo(HiveDecimal dec) { + if (!isSet() || dec == null) { + throw new RuntimeException("Invalid comparision operand(s)"); + } + return fastCompareTo(dec); + } + + public static int compareTo(HiveDecimal dec, HiveDecimalWritable writable) { + if (dec == null || !writable.isSet()) { + throw new RuntimeException("Invalid comparision operand(s)"); + } + return FastHiveDecimal.fastCompareTo(dec, writable); + } + + public int toBytes(byte[] scratchBuffer) { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastToBytes(scratchBuffer); } @Override public String toString() { - return getHiveDecimal().toString(); + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastToString(); + } + + public String toFormatString( + int formatScale) { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return + fastToFormatString( + formatScale); + } + + public int toFormatBytes( + int formatScale, + byte[] scratchBuffer) { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return + fastToFormatBytes( + formatScale, + scratchBuffer); + } + + public int toDigitsOnlyBytes( + byte[] scratchBuffer) { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return + fastToDigitsOnlyBytes( + scratchBuffer); } @Override public boolean equals(Object other) { + if (!isSet) { + return false; + } if (this == other) { return true; } if (other == null || getClass() != other.getClass()) { return false; } - HiveDecimalWritable bdw = (HiveDecimalWritable) other; + HiveDecimalWritable otherHiveDecWritable = (HiveDecimalWritable) other; + if (!otherHiveDecWritable.isSet()) { + return false; + } + return fastEquals((FastHiveDecimal) otherHiveDecWritable); - // 'equals' and 'compareTo' are not compatible with HiveDecimals. We want - // compareTo which returns true iff the numbers are equal (e.g.: 3.14 is - // the same as 3.140). 'Equals' returns true iff equal and the same scale - // is set in the decimals (e.g.: 3.14 is not the same as 3.140) - return getHiveDecimal().compareTo(bdw.getHiveDecimal()) == 0; } @Override public int hashCode() { - return getHiveDecimal().hashCode(); + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastHashCode(); } - /* (non-Javadoc) - * In order to update a Decimal128 fast (w/o allocation) we need to expose access to the - * internal storage bytes and scale. - * @return - */ - public byte[] getInternalStorage() { - return internalStorage; + public int oldHiveDecimalHashCode() { + if (!isSet()) { + throw new RuntimeException("no value set"); + } + return fastOldHiveDecimalHashCode(); } - /* (non-Javadoc) - * In order to update a Decimal128 fast (w/o allocation) we need to expose access to the - * internal storage bytes and scale. - */ - public int getScale() { - return scale; + public void mutateRound(int newScale, int roundingMode) { + if (!isSet) { + return; + } + isSet = fastRound(newScale, roundingMode, this); + if (!isSet) { + fastReset(); + } } - public static - HiveDecimalWritable enforcePrecisionScale(HiveDecimalWritable writable, - int precision, int scale) { - if (writable == null) { - return null; + public boolean mutateEnforcePrecisionScale(int precision, int scale) { + if (!isSet) { + return false; } + isSet = fastEnforcePrecisionScale(precision, scale); + if (!isSet) { + fastReset(); + } + return isSet; + } - HiveDecimal dec = - HiveDecimal.enforcePrecisionScale(writable.getHiveDecimal(), precision, - scale); - return dec == null ? null : new HiveDecimalWritable(dec); + public static HiveDecimalWritable enforcePrecisionScale(HiveDecimalWritable writable, int precision, int scale) { + if (!writable.isSet) { + return null; + } + HiveDecimalWritable result = new HiveDecimalWritable(writable); + result.mutateEnforcePrecisionScale(precision, scale); + if (!result.isSet()) { + return null; + } + return result; } } diff --git storage-api/src/java/org/apache/hadoop/hive/serde2/io/OldHiveDecimalWritable.java storage-api/src/java/org/apache/hadoop/hive/serde2/io/OldHiveDecimalWritable.java new file mode 100644 index 0000000..efad66b --- /dev/null +++ storage-api/src/java/org/apache/hadoop/hive/serde2/io/OldHiveDecimalWritable.java @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.serde2.io; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.math.BigInteger; + +import org.apache.hadoop.hive.common.type.OldHiveDecimal; + +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableUtils; + +public class OldHiveDecimalWritable implements WritableComparable { + + private byte[] internalStorage = new byte[0]; + private int scale; + + public OldHiveDecimalWritable() { + } + + public OldHiveDecimalWritable(String value) { + set(OldHiveDecimal.create(value)); + } + + public OldHiveDecimalWritable(byte[] bytes, int scale) { + set(bytes, scale); + } + + public OldHiveDecimalWritable(OldHiveDecimalWritable writable) { + set(writable.getHiveDecimal()); + } + + public OldHiveDecimalWritable(OldHiveDecimal value) { + set(value); + } + + public OldHiveDecimalWritable(long value) { + set((OldHiveDecimal.create(value))); + } + + public void set(OldHiveDecimal value) { + set(value.unscaledValue().toByteArray(), value.scale()); + } + + public void set(OldHiveDecimal value, int maxPrecision, int maxScale) { + set(OldHiveDecimal.enforcePrecisionScale(value, maxPrecision, maxScale)); + } + + public void set(OldHiveDecimalWritable writable) { + set(writable.getHiveDecimal()); + } + + public void set(byte[] bytes, int scale) { + this.internalStorage = bytes; + this.scale = scale; + } + + public OldHiveDecimal getHiveDecimal() { + return OldHiveDecimal.create(new BigInteger(internalStorage), scale); + } + + /** + * Get a OldHiveDecimal instance from the writable and constraint it with maximum precision/scale. + * + * @param maxPrecision maximum precision + * @param maxScale maximum scale + * @return OldHiveDecimal instance + */ + public OldHiveDecimal getHiveDecimal(int maxPrecision, int maxScale) { + return OldHiveDecimal.enforcePrecisionScale(OldHiveDecimal. + create(new BigInteger(internalStorage), scale), + maxPrecision, maxScale); + } + + @Override + public void readFields(DataInput in) throws IOException { + scale = WritableUtils.readVInt(in); + int byteArrayLen = WritableUtils.readVInt(in); + if (internalStorage.length != byteArrayLen) { + internalStorage = new byte[byteArrayLen]; + } + in.readFully(internalStorage); + } + + @Override + public void write(DataOutput out) throws IOException { + WritableUtils.writeVInt(out, scale); + WritableUtils.writeVInt(out, internalStorage.length); + out.write(internalStorage); + } + + @Override + public int compareTo(OldHiveDecimalWritable that) { + return getHiveDecimal().compareTo(that.getHiveDecimal()); + } + + @Override + public String toString() { + return getHiveDecimal().toString(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + OldHiveDecimalWritable bdw = (OldHiveDecimalWritable) other; + + // 'equals' and 'compareTo' are not compatible with HiveDecimals. We want + // compareTo which returns true iff the numbers are equal (e.g.: 3.14 is + // the same as 3.140). 'Equals' returns true iff equal and the same scale + // is set in the decimals (e.g.: 3.14 is not the same as 3.140) + return getHiveDecimal().compareTo(bdw.getHiveDecimal()) == 0; + } + + @Override + public int hashCode() { + return getHiveDecimal().hashCode(); + } + + /* (non-Javadoc) + * In order to update a Decimal128 fast (w/o allocation) we need to expose access to the + * internal storage bytes and scale. + * @return + */ + public byte[] getInternalStorage() { + return internalStorage; + } + + /* (non-Javadoc) + * In order to update a Decimal128 fast (w/o allocation) we need to expose access to the + * internal storage bytes and scale. + */ + public int getScale() { + return scale; + } + + public static + OldHiveDecimalWritable enforcePrecisionScale(OldHiveDecimalWritable writable, + int precision, int scale) { + if (writable == null) { + return null; + } + + OldHiveDecimal dec = + OldHiveDecimal.enforcePrecisionScale(writable.getHiveDecimal(), precision, + scale); + return dec == null ? null : new OldHiveDecimalWritable(dec); + } +}