diff --git common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java index bd3e997..d4cc32d 100644 --- common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java +++ common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java @@ -103,13 +103,6 @@ private short scale; /** - * This is the actual scale detected from the value passed to this Decimal128. - * The value is always equals or less than #scale. It is used to return the correct - * decimal string from {@link #getHiveDecimalString()}. - */ - private short actualScale; - - /** * -1 means negative, 0 means zero, 1 means positive. * * @serial @@ -134,7 +127,6 @@ public Decimal128() { this.unscaledValue = new UnsignedInt128(); this.scale = 0; this.signum = 0; - this.actualScale = 0; } /** @@ -147,7 +139,6 @@ public Decimal128(Decimal128 o) { this.unscaledValue = new UnsignedInt128(o.unscaledValue); this.scale = o.scale; this.signum = o.signum; - this.actualScale = o.actualScale; } /** @@ -187,7 +178,6 @@ public Decimal128(UnsignedInt128 unscaledVal, short scale, boolean negative) { checkScaleRange(scale); this.unscaledValue = new UnsignedInt128(unscaledVal); this.scale = scale; - this.actualScale = scale; if (unscaledValue.isZero()) { this.signum = 0; } else { @@ -274,7 +264,6 @@ public Decimal128 update(Decimal128 o) { this.unscaledValue.update(o.unscaledValue); this.scale = o.scale; this.signum = o.signum; - this.actualScale = o.actualScale; return this; } @@ -303,7 +292,7 @@ public Decimal128 update(long val) { /** * Update the value of this object with the given {@code long} with the given - * scale. + * scal. * * @param val * {@code long} value to be set to {@code Decimal128}. @@ -325,8 +314,6 @@ public Decimal128 update(long val, short scale) { if (scale != 0) { changeScaleDestructive(scale); } - // set actualScale to 0 because there is no fractional digits on integer values - this.actualScale = 0; return this; } @@ -354,11 +341,6 @@ public Decimal128 update(double val, short scale) { checkScaleRange(scale); this.scale = scale; - // Obtains the scale of the double value to keep a record of the original - // scale. This will be used to print the HiveDecimal string with the - // correct value scale. - this.actualScale = (short) BigDecimal.valueOf(val).scale(); - // Translate the double into sign, exponent and significand, according // to the formulae in JLS, Section 20.10.22. long valBits = Double.doubleToLongBits(val); @@ -382,10 +364,6 @@ public Decimal128 update(double val, short scale) { exponent++; } - // Calculate the real number of fractional digits from the double value - this.actualScale -= (exponent > 0) ? exponent : 0; - this.actualScale = (this.actualScale < 0) ? 0 : this.actualScale; - // so far same as java.math.BigDecimal, but the scaling below is // specific to ANSI SQL Numeric. @@ -448,7 +426,6 @@ public Decimal128 update(double val, short scale) { public Decimal128 update(IntBuffer buf, int precision) { int scaleAndSignum = buf.get(); this.scale = (short) (scaleAndSignum >> 16); - this.actualScale = this.scale; this.signum = (byte) (scaleAndSignum & 0xFF); this.unscaledValue.update(buf, precision); assert ((signum == 0) == unscaledValue.isZero()); @@ -465,7 +442,6 @@ public Decimal128 update(IntBuffer buf, int precision) { public Decimal128 update128(IntBuffer buf) { int scaleAndSignum = buf.get(); this.scale = (short) (scaleAndSignum >> 16); - this.actualScale = this.scale; this.signum = (byte) (scaleAndSignum & 0xFF); this.unscaledValue.update128(buf); assert ((signum == 0) == unscaledValue.isZero()); @@ -482,7 +458,6 @@ public Decimal128 update128(IntBuffer buf) { public Decimal128 update96(IntBuffer buf) { int scaleAndSignum = buf.get(); this.scale = (short) (scaleAndSignum >> 16); - this.actualScale = this.scale; this.signum = (byte) (scaleAndSignum & 0xFF); this.unscaledValue.update96(buf); assert ((signum == 0) == unscaledValue.isZero()); @@ -499,7 +474,6 @@ public Decimal128 update96(IntBuffer buf) { public Decimal128 update64(IntBuffer buf) { int scaleAndSignum = buf.get(); this.scale = (short) (scaleAndSignum >> 16); - this.actualScale = this.scale; this.signum = (byte) (scaleAndSignum & 0xFF); this.unscaledValue.update64(buf); assert ((signum == 0) == unscaledValue.isZero()); @@ -516,7 +490,6 @@ public Decimal128 update64(IntBuffer buf) { public Decimal128 update32(IntBuffer buf) { int scaleAndSignum = buf.get(); this.scale = (short) (scaleAndSignum >> 16); - this.actualScale = this.scale; this.signum = (byte) (scaleAndSignum & 0xFF); this.unscaledValue.update32(buf); assert ((signum == 0) == unscaledValue.isZero()); @@ -537,7 +510,6 @@ public Decimal128 update32(IntBuffer buf) { public Decimal128 update(int[] array, int offset, int precision) { int scaleAndSignum = array[offset]; this.scale = (short) (scaleAndSignum >> 16); - this.actualScale = this.scale; this.signum = (byte) (scaleAndSignum & 0xFF); this.unscaledValue.update(array, offset + 1, precision); return this; @@ -555,7 +527,6 @@ public Decimal128 update(int[] array, int offset, int precision) { public Decimal128 update128(int[] array, int offset) { int scaleAndSignum = array[offset]; this.scale = (short) (scaleAndSignum >> 16); - this.actualScale = this.scale; this.signum = (byte) (scaleAndSignum & 0xFF); this.unscaledValue.update128(array, offset + 1); return this; @@ -573,7 +544,6 @@ public Decimal128 update128(int[] array, int offset) { public Decimal128 update96(int[] array, int offset) { int scaleAndSignum = array[offset]; this.scale = (short) (scaleAndSignum >> 16); - this.actualScale = this.scale; this.signum = (byte) (scaleAndSignum & 0xFF); this.unscaledValue.update96(array, offset + 1); return this; @@ -591,7 +561,6 @@ public Decimal128 update96(int[] array, int offset) { public Decimal128 update64(int[] array, int offset) { int scaleAndSignum = array[offset]; this.scale = (short) (scaleAndSignum >> 16); - this.actualScale = this.scale; this.signum = (byte) (scaleAndSignum & 0xFF); this.unscaledValue.update64(array, offset + 1); return this; @@ -609,7 +578,6 @@ public Decimal128 update64(int[] array, int offset) { public Decimal128 update32(int[] array, int offset) { int scaleAndSignum = array[offset]; this.scale = (short) (scaleAndSignum >> 16); - this.actualScale = this.scale; this.signum = (byte) (scaleAndSignum & 0xFF); this.unscaledValue.update32(array, offset + 1); return this; @@ -632,6 +600,7 @@ public Decimal128 update(BigDecimal bigDecimal) { * @param scale */ public Decimal128 update(BigInteger bigInt, short scale) { + this.scale = scale; this.signum = (byte) bigInt.compareTo(BigInteger.ZERO); if (signum == 0) { update(0); @@ -640,9 +609,6 @@ public Decimal128 update(BigInteger bigInt, short scale) { } else { unscaledValue.update(bigInt); } - this.scale = scale; - this.actualScale = scale; - return this; } @@ -765,9 +731,6 @@ public Decimal128 update(char[] str, int offset, int length, short scale) { this.unscaledValue.addDestructive(accumulated); } - this.actualScale = (short) (fractionalDigits - exponent); - this.actualScale = (this.actualScale < 0) ? 0 : this.actualScale; - int scaleAdjust = scale - fractionalDigits + exponent; if (scaleAdjust > 0) { this.unscaledValue.scaleUpTenDestructive((short) scaleAdjust); @@ -961,7 +924,6 @@ public void changeScaleDestructive(short scale) { this.unscaledValue.scaleUpTenDestructive((short) -scaleDown); } this.scale = scale; - this.actualScale = scale; this.unscaledValue.throwIfExceedsTenToThirtyEight(); } @@ -1163,7 +1125,6 @@ public void multiplyDestructiveNativeDecimal128(Decimal128 right, short newScale if (this.signum == 0 || right.signum == 0) { this.zeroClear(); this.scale = newScale; - this.actualScale = newScale; return; } @@ -1193,7 +1154,6 @@ public void multiplyDestructiveNativeDecimal128(Decimal128 right, short newScale } this.scale = newScale; - this.actualScale = newScale; this.signum = (byte) (this.signum * right.signum); if (this.unscaledValue.isZero()) { this.signum = 0; // because of scaling down, this could happen @@ -1284,7 +1244,6 @@ public void divideDestructiveNativeDecimal128(Decimal128 right, short newScale, } if (this.signum == 0) { this.scale = newScale; - this.actualScale = newScale; remainder.update(this); return; } @@ -1312,7 +1271,6 @@ public void divideDestructiveNativeDecimal128(Decimal128 right, short newScale, } this.scale = newScale; - this.actualScale = newScale; this.signum = (byte) (this.unscaledValue.isZero() ? 0 : (this.signum * right.signum)); remainder.scale = scale; @@ -1773,13 +1731,17 @@ private static void checkScaleRange(short scale) { private int [] tmpArray = new int[2]; /** - * Returns the string representation of this value. It returns the original - * {@code actualScale} fractional part when this value was created. However, + * Returns the string representation of this value. It discards the trailing zeros + * in the fractional part to match the HiveDecimal's string representation. However, * don't use this string representation for the reconstruction of the object. * * @return string representation of this value */ public String getHiveDecimalString() { + if (this.signum == 0) { + return "0"; + } + StringBuilder buf = new StringBuilder(50); if (this.signum < 0) { buf.append('-'); @@ -1790,40 +1752,32 @@ public String getHiveDecimalString() { int trailingZeros = tmpArray[1]; int numIntegerDigits = unscaledLength - this.scale; if (numIntegerDigits > 0) { + // write out integer part first // then write out fractional part for (int i=0; i < numIntegerDigits; i++) { buf.append(unscaled[i]); } - if (this.actualScale > 0) { + if (this.scale > trailingZeros) { buf.append('.'); - - if (trailingZeros > this.actualScale) { - for (int i=0; i < (trailingZeros - this.scale); i++) { - buf.append("0"); - } - } - - for (int i = numIntegerDigits; i < (numIntegerDigits + this.actualScale); i++) { + for (int i = numIntegerDigits; i < (unscaledLength - trailingZeros); i++) { buf.append(unscaled[i]); } } } else { + // no integer part buf.append('0'); - if (this.actualScale > 0) { + if (this.scale > trailingZeros) { + // fractional part has, starting with zeros buf.append('.'); - - if (this.actualScale > trailingZeros) { - for (int i = unscaledLength; i < this.scale; ++i) { - buf.append('0'); - } + for (int i = unscaledLength; i < this.scale; ++i) { + buf.append('0'); } - - for (int i = 0; i < (numIntegerDigits + this.actualScale); i++) { + for (int i = 0; i < (unscaledLength - trailingZeros); i++) { buf.append(unscaled[i]); } } @@ -1882,10 +1836,9 @@ public String toFormalString() { @Override public String toString() { - return toFormalString() + "(Decimal128: scale=" + scale + ", actualScale=" - + this.actualScale + ", signum=" + signum + ", BigDecimal.toString=" - + toBigDecimal().toString() + ", unscaledValue=[" + unscaledValue.toString() - + "])"; + return toFormalString() + "(Decimal128: scale=" + scale + ", signum=" + + signum + ", BigDecimal.toString=" + toBigDecimal().toString() + + ", unscaledValue=[" + unscaledValue.toString() + "])"; } /** @@ -2003,7 +1956,6 @@ public Decimal128 updateVarianceDestructive( */ public Decimal128 fastUpdateFromInternalStorage(byte[] internalStorage, short scale) { this.scale = scale; - this.actualScale = scale; this.signum = this.unscaledValue.fastUpdateFromInternalStorage(internalStorage); return this; diff --git common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java index 0bb84e1..c2bf6d7 100644 --- common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java +++ common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java @@ -30,6 +30,7 @@ public class HiveDecimal implements Comparable { public static final int MAX_PRECISION = 38; public static final int MAX_SCALE = 38; + /** * Default precision/scale when user doesn't specify in the column metadata, such as * decimal and decimal(8). @@ -112,7 +113,7 @@ public int compareTo(HiveDecimal dec) { @Override public int hashCode() { - return trim(bd).hashCode(); + return bd.hashCode(); } @Override @@ -168,7 +169,7 @@ public HiveDecimal subtract(HiveDecimal dec) { } public HiveDecimal multiply(HiveDecimal dec) { - return create(bd.multiply(dec.bd), true); + return create(bd.multiply(dec.bd), false); } public BigInteger unscaledValue() { @@ -201,7 +202,7 @@ public HiveDecimal remainder(HiveDecimal dec) { } public HiveDecimal divide(HiveDecimal dec) { - return create(trim(bd.divide(dec.bd, MAX_SCALE, RoundingMode.HALF_UP)), true); + return create(bd.divide(dec.bd, MAX_SCALE, RoundingMode.HALF_UP), true); } /** @@ -231,6 +232,8 @@ private static BigDecimal normalize(BigDecimal bd, boolean allowRounding) { return null; } + bd = trim(bd); + int intDigits = bd.precision() - bd.scale(); if (intDigits > MAX_PRECISION) { @@ -241,6 +244,8 @@ private static BigDecimal normalize(BigDecimal bd, boolean allowRounding) { if (bd.scale() > maxScale ) { if (allowRounding) { bd = bd.setScale(maxScale, RoundingMode.HALF_UP); + // Trimming is again necessary, because rounding may introduce new trailing 0's. + bd = trim(bd); } else { bd = null; } @@ -254,6 +259,8 @@ public static BigDecimal enforcePrecisionScale(BigDecimal bd, int maxPrecision, return null; } + bd = trim(bd); + if (bd.scale() > maxScale) { bd = bd.setScale(maxScale, RoundingMode.HALF_UP); } diff --git common/src/test/org/apache/hadoop/hive/common/type/TestDecimal128.java common/src/test/org/apache/hadoop/hive/common/type/TestDecimal128.java index 0786cca..46236a5 100644 --- common/src/test/org/apache/hadoop/hive/common/type/TestDecimal128.java +++ common/src/test/org/apache/hadoop/hive/common/type/TestDecimal128.java @@ -811,7 +811,7 @@ public void testToHiveDecimalString() { assertEquals("0.00923076923", d2.getHiveDecimalString()); Decimal128 d3 = new Decimal128("0.00923076000", (short) 15); - assertEquals("0.00923076000", d3.getHiveDecimalString()); + assertEquals("0.00923076", d3.getHiveDecimalString()); Decimal128 d4 = new Decimal128("4294967296.01", (short) 15); assertEquals("4294967296.01", d4.getHiveDecimalString()); @@ -849,37 +849,15 @@ public void testToHiveDecimalString() { d11.update(hd6.bigDecimalValue()); assertEquals(hd6.toString(), d11.getHiveDecimalString()); - // The trailing zeros from a double value are trimmed automatically - // by the double data type Decimal128 d12 = new Decimal128(27.000, (short)3); - HiveDecimal hd7 = HiveDecimal.create(new BigDecimal("27.0")); + HiveDecimal hd7 = HiveDecimal.create(new BigDecimal("27.000")); assertEquals(hd7.toString(), d12.getHiveDecimalString()); - assertEquals("27.0", d12.getHiveDecimalString()); + assertEquals("27", d12.getHiveDecimalString()); Decimal128 d13 = new Decimal128(1234123000, (short)3); HiveDecimal hd8 = HiveDecimal.create(new BigDecimal("1234123000")); assertEquals(hd8.toString(), d13.getHiveDecimalString()); assertEquals("1234123000", d13.getHiveDecimalString()); - - Decimal128 d14 = new Decimal128(1.33e4, (short)10); - HiveDecimal hd9 = HiveDecimal.create(new BigDecimal("1.33e4")); - assertEquals(hd9.toString(), d14.getHiveDecimalString()); - assertEquals("13300", d14.getHiveDecimalString()); - - Decimal128 d15 = new Decimal128(1.33e-4, (short)10); - HiveDecimal hd10 = HiveDecimal.create(new BigDecimal("1.33e-4")); - assertEquals(hd10.toString(), d15.getHiveDecimalString()); - assertEquals("0.000133", d15.getHiveDecimalString()); - - Decimal128 d16 = new Decimal128("1.33e4", (short)10); - HiveDecimal hd11 = HiveDecimal.create(new BigDecimal("1.33e4")); - assertEquals(hd11.toString(), d16.getHiveDecimalString()); - assertEquals("13300", d16.getHiveDecimalString()); - - Decimal128 d17 = new Decimal128("1.33e-4", (short)10); - HiveDecimal hd12 = HiveDecimal.create(new BigDecimal("1.33e-4")); - assertEquals(hd12.toString(), d17.getHiveDecimalString()); - assertEquals("0.000133", d17.getHiveDecimalString()); } @Test diff --git common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java index 959989a..438cc21 100644 --- common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java +++ common/src/test/org/apache/hadoop/hive/common/type/TestHiveDecimal.java @@ -57,35 +57,27 @@ public void testPrecisionScaleEnforcement() { Assert.assertEquals("-1786135888657847525803324040144343378.1", dec.toString()); dec = HiveDecimal.create("005.34000"); - Assert.assertEquals(dec.precision(), 6); - Assert.assertEquals(dec.scale(), 5); + Assert.assertEquals(dec.precision(), 3); + Assert.assertEquals(dec.scale(), 2); dec = HiveDecimal.create("178613588865784752580332404014434337809799306448796128931113691624"); Assert.assertNull(dec); - // Leaving trailing zeros - Assert.assertEquals("0.0", HiveDecimal.enforcePrecisionScale(new BigDecimal("0.0"), 2, 1).toString()); - Assert.assertEquals("0.00", HiveDecimal.enforcePrecisionScale(new BigDecimal("0.00"), 3, 2).toString()); - Assert.assertEquals("0.0000", HiveDecimal.enforcePrecisionScale(new BigDecimal("0.0000"), 10, 4).toString()); - Assert.assertEquals("100.00000", HiveDecimal.enforcePrecisionScale(new BigDecimal("100.00000"), 15, 5).toString()); - Assert.assertEquals("100.00", HiveDecimal.enforcePrecisionScale(new BigDecimal("100.00"), 15, 5).toString()); - - // Rounding numbers - Assert.assertEquals("0.01", HiveDecimal.enforcePrecisionScale(new BigDecimal("0.012"), 3, 2).toString()); - Assert.assertEquals("0.02", HiveDecimal.enforcePrecisionScale(new BigDecimal("0.015"), 3, 2).toString()); - Assert.assertEquals("0.01", HiveDecimal.enforcePrecisionScale(new BigDecimal("0.0145"), 3, 2).toString()); - // Rounding numbers that increase int digits Assert.assertEquals("10", HiveDecimal.enforcePrecisionScale(new BigDecimal("9.5"), 2, 0).toString()); Assert.assertNull(HiveDecimal.enforcePrecisionScale(new BigDecimal("9.5"), 1, 0)); Assert.assertEquals("9", HiveDecimal.enforcePrecisionScale(new BigDecimal("9.4"), 1, 0).toString()); + } - // Integers with no scale values are not modified (zeros are not null) - Assert.assertEquals("0", HiveDecimal.enforcePrecisionScale(new BigDecimal("0"), 1, 0).toString()); - Assert.assertEquals("30", HiveDecimal.enforcePrecisionScale(new BigDecimal("30"), 2, 0).toString()); - Assert.assertEquals("5", HiveDecimal.enforcePrecisionScale(new BigDecimal("5"), 3, 2).toString()); + @Test + @Concurrent(count=4) + @Repeating(repetition=100) + public void testTrailingZeroRemovalAfterEnforcement() { + String decStr = "8.090000000000000000000000000000000000000123456"; + HiveDecimal dec = HiveDecimal.create(decStr); + Assert.assertEquals("8.09", dec.toString()); } @Test diff --git ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java index 0802e3b..1aa1ae1 100644 --- ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java +++ ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorTypeCasts.java @@ -334,7 +334,7 @@ public void testCastDecimalToString() { StringExpr.compare(v, 0, v.length, r.vector[1], r.start[1], r.length[1])); - v = toBytes("9999999999999999.00"); + v = toBytes("9999999999999999"); Assert.assertEquals(0, StringExpr.compare(v, 0, v.length, r.vector[2], r.start[2], r.length[2])); diff --git ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java index 831ef8c..c91644c 100644 --- ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java +++ ql/src/test/org/apache/hadoop/hive/ql/io/sarg/TestSearchArgumentImpl.java @@ -2830,7 +2830,7 @@ public void testBuilderComplexTypes() throws Exception { .build(); assertEquals("leaf-0 = (LESS_THAN x 1970-01-11)\n" + "leaf-1 = (LESS_THAN_EQUALS y hi)\n" + - "leaf-2 = (EQUALS z 1.0)\n" + + "leaf-2 = (EQUALS z 1)\n" + "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toString()); sarg = SearchArgumentFactory.newBuilder() diff --git ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPDivide.java ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPDivide.java index ce578a0..4c5b3a5 100644 --- ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPDivide.java +++ ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPDivide.java @@ -187,7 +187,7 @@ public void testDecimalDivideDecimal() throws HiveException { PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs); Assert.assertEquals(TypeInfoFactory.getDecimalTypeInfo(11, 7), oi.getTypeInfo()); HiveDecimalWritable res = (HiveDecimalWritable) udf.evaluate(args); - Assert.assertEquals(HiveDecimal.create("0.0617100"), res.getHiveDecimal()); + Assert.assertEquals(HiveDecimal.create("0.06171"), res.getHiveDecimal()); } @Test diff --git ql/src/test/queries/clientpositive/decimal_join2.q ql/src/test/queries/clientpositive/decimal_join2.q new file mode 100644 index 0000000..fb45367 --- /dev/null +++ ql/src/test/queries/clientpositive/decimal_join2.q @@ -0,0 +1,26 @@ +DROP TABLE IF EXISTS DECIMAL_3_txt; +DROP TABLE IF EXISTS DECIMAL_3; + +CREATE TABLE DECIMAL_3_txt(key decimal(38,18), value int) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE; + +LOAD DATA LOCAL INPATH '../../data/files/kv7.txt' INTO TABLE DECIMAL_3_txt; + +CREATE TABLE DECIMAL_3 STORED AS ORC AS SELECT * FROM DECIMAL_3_txt; + +set hive.auto.convert.join=false; +EXPLAIN +SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key) ORDER BY a.key, a.value, b.key, b.value; + +SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key) ORDER BY a.key, a.value, b.key, b.value; + +set hive.auto.convert.join=true; +EXPLAIN +SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key) ORDER BY a.key, a.value, b.key, b.value; + +SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key) ORDER BY a.key, a.value, b.key, b.value; + +DROP TABLE DECIMAL_3_txt; +DROP TABLE DECIMAL_3; diff --git ql/src/test/results/clientpositive/decimal_join2.q.out ql/src/test/results/clientpositive/decimal_join2.q.out new file mode 100644 index 0000000..f9c33b7 --- /dev/null +++ ql/src/test/results/clientpositive/decimal_join2.q.out @@ -0,0 +1,371 @@ +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_3_txt +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_3_txt +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP TABLE IF EXISTS DECIMAL_3 +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS DECIMAL_3 +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE DECIMAL_3_txt(key decimal(38,18), value int) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@DECIMAL_3_txt +POSTHOOK: query: CREATE TABLE DECIMAL_3_txt(key decimal(38,18), value int) +ROW FORMAT DELIMITED + FIELDS TERMINATED BY ' ' +STORED AS TEXTFILE +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DECIMAL_3_txt +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv7.txt' INTO TABLE DECIMAL_3_txt +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@decimal_3_txt +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv7.txt' INTO TABLE DECIMAL_3_txt +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@decimal_3_txt +PREHOOK: query: CREATE TABLE DECIMAL_3 STORED AS ORC AS SELECT * FROM DECIMAL_3_txt +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@decimal_3_txt +PREHOOK: Output: database:default +PREHOOK: Output: default@DECIMAL_3 +POSTHOOK: query: CREATE TABLE DECIMAL_3 STORED AS ORC AS SELECT * FROM DECIMAL_3_txt +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@decimal_3_txt +POSTHOOK: Output: database:default +POSTHOOK: Output: default@DECIMAL_3 +PREHOOK: query: EXPLAIN +SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key) ORDER BY a.key, a.value, b.key, b.value +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key) ORDER BY a.key, a.value, b.key, b.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-2 depends on stages: Stage-1 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: decimal(38,18)) + sort order: + + Map-reduce partition columns: key (type: decimal(38,18)) + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: int) + TableScan + alias: a + Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: key (type: decimal(38,18)) + sort order: + + Map-reduce partition columns: key (type: decimal(38,18)) + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE + value expressions: value (type: int) + Reduce Operator Tree: + Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {KEY.reducesinkkey0} {VALUE._col0} + 1 {KEY.reducesinkkey0} {VALUE._col0} + outputColumnNames: _col0, _col1, _col5, _col6 + Statistics: Num rows: 20 Data size: 2362 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: decimal(38,18)), _col1 (type: int), _col5 (type: decimal(38,18)), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 20 Data size: 2362 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + Reduce Output Operator + key expressions: _col0 (type: decimal(38,18)), _col1 (type: int), _col2 (type: decimal(38,18)), _col3 (type: int) + sort order: ++++ + Statistics: Num rows: 20 Data size: 2362 Basic stats: COMPLETE Column stats: NONE + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: decimal(38,18)), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: decimal(38,18)), KEY.reducesinkkey3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 20 Data size: 2362 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 2362 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key) ORDER BY a.key, a.value, b.key, b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key) ORDER BY a.key, a.value, b.key, b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +-1234567890.123456789 -1234567890 -1234567890.123456789 -1234567890 +-4400 4400 -4400 4400 +-1255.49 -1255 -1255.49 -1255 +-1.122 -11 -1.122 -11 +-1.12 -1 -1.12 -1 +-1.12 -1 -1.12 -1 +-1.12 -1 -1.12 -1 +-1.12 -1 -1.12 -1 +-0.333 0 -0.333 0 +-0.33 0 -0.33 0 +-0.3 0 -0.3 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0.01 0 0.01 0 +0.02 0 0.02 0 +0.1 0 0.1 0 +0.2 0 0.2 0 +0.3 0 0.3 0 +0.33 0 0.33 0 +0.333 0 0.333 0 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1.12 1 1.12 1 +1.122 1 1.122 1 +2 2 2 2 +2 2 2 2 +2 2 2 2 +2 2 2 2 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 4 +3.14 3 3.14 4 +3.14 3 3.14 4 +3.14 4 3.14 3 +3.14 4 3.14 3 +3.14 4 3.14 3 +3.14 4 3.14 4 +10 10 10 10 +20 20 20 20 +100 100 100 100 +124 124 124 124 +125.2 125 125.2 125 +200 200 200 200 +1234567890.12345678 1234567890 1234567890.12345678 1234567890 +PREHOOK: query: EXPLAIN +SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key) ORDER BY a.key, a.value, b.key, b.value +PREHOOK: type: QUERY +POSTHOOK: query: EXPLAIN +SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key) ORDER BY a.key, a.value, b.key, b.value +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-5 is a root stage + Stage-2 depends on stages: Stage-5 + Stage-0 depends on stages: Stage-2 + +STAGE PLANS: + Stage: Stage-5 + Map Reduce Local Work + Alias -> Map Local Tables: + a + Fetch Operator + limit: -1 + Alias -> Map Local Operator Tree: + a + TableScan + alias: a + Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE + HashTable Sink Operator + condition expressions: + 0 {value} + 1 {key} {value} + keys: + 0 key (type: decimal(38,18)) + 1 key (type: decimal(38,18)) + + Stage: Stage-2 + Map Reduce + Map Operator Tree: + TableScan + alias: b + Statistics: Num rows: 38 Data size: 4296 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 19 Data size: 2148 Basic stats: COMPLETE Column stats: NONE + Map Join Operator + condition map: + Inner Join 0 to 1 + condition expressions: + 0 {key} {value} + 1 {key} {value} + keys: + 0 key (type: decimal(38,18)) + 1 key (type: decimal(38,18)) + outputColumnNames: _col0, _col1, _col5, _col6 + Statistics: Num rows: 20 Data size: 2362 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: decimal(38,18)), _col1 (type: int), _col5 (type: decimal(38,18)), _col6 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 20 Data size: 2362 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: decimal(38,18)), _col1 (type: int), _col2 (type: decimal(38,18)), _col3 (type: int) + sort order: ++++ + Statistics: Num rows: 20 Data size: 2362 Basic stats: COMPLETE Column stats: NONE + Local Work: + Map Reduce Local Work + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: decimal(38,18)), KEY.reducesinkkey1 (type: int), KEY.reducesinkkey2 (type: decimal(38,18)), KEY.reducesinkkey3 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 20 Data size: 2362 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 20 Data size: 2362 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key) ORDER BY a.key, a.value, b.key, b.value +PREHOOK: type: QUERY +PREHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +POSTHOOK: query: SELECT * FROM DECIMAL_3 a JOIN DECIMAL_3 b ON (a.key = b.key) ORDER BY a.key, a.value, b.key, b.value +POSTHOOK: type: QUERY +POSTHOOK: Input: default@decimal_3 +#### A masked pattern was here #### +-1234567890.123456789 -1234567890 -1234567890.123456789 -1234567890 +-4400 4400 -4400 4400 +-1255.49 -1255 -1255.49 -1255 +-1.122 -11 -1.122 -11 +-1.12 -1 -1.12 -1 +-1.12 -1 -1.12 -1 +-1.12 -1 -1.12 -1 +-1.12 -1 -1.12 -1 +-0.333 0 -0.333 0 +-0.33 0 -0.33 0 +-0.3 0 -0.3 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0 0 0 0 +0.01 0 0.01 0 +0.02 0 0.02 0 +0.1 0 0.1 0 +0.2 0 0.2 0 +0.3 0 0.3 0 +0.33 0 0.33 0 +0.333 0 0.333 0 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1 1 1 1 +1.12 1 1.12 1 +1.122 1 1.122 1 +2 2 2 2 +2 2 2 2 +2 2 2 2 +2 2 2 2 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 3 +3.14 3 3.14 4 +3.14 3 3.14 4 +3.14 3 3.14 4 +3.14 4 3.14 3 +3.14 4 3.14 3 +3.14 4 3.14 3 +3.14 4 3.14 4 +10 10 10 10 +20 20 20 20 +100 100 100 100 +124 124 124 124 +125.2 125 125.2 125 +200 200 200 200 +1234567890.12345678 1234567890 1234567890.12345678 1234567890 +PREHOOK: query: DROP TABLE DECIMAL_3_txt +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_3_txt +PREHOOK: Output: default@decimal_3_txt +POSTHOOK: query: DROP TABLE DECIMAL_3_txt +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_3_txt +POSTHOOK: Output: default@decimal_3_txt +PREHOOK: query: DROP TABLE DECIMAL_3 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@decimal_3 +PREHOOK: Output: default@decimal_3 +POSTHOOK: query: DROP TABLE DECIMAL_3 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@decimal_3 +POSTHOOK: Output: default@decimal_3 diff --git serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java index 73ae2d9..2b7fba6 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java +++ serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java @@ -779,7 +779,7 @@ static void serialize(ByteStream.Output buffer, Object o, ObjectInspector oi, // get the scale factor to turn big decimal into a decimal < 1 int factor = dec.precision() - dec.scale(); - factor = sign != -1 ? factor : -factor; + factor = sign == 1 ? factor : -factor; // convert the absolute big decimal to string dec.scaleByPowerOfTen(Math.abs(dec.scale()));