diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java index c2bf6d7..a8215f2 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java @@ -128,8 +128,25 @@ public int scale() { return bd.scale(); } + /** + * Returns the number of digits (integer and fractional) in the number, which is equivalent + * to SQL decimal precision. Note that this is different from BigDecimal.precision(), + * which returns the precision of the unscaled value (BigDecimal.valueOf(0.01).precision() = 1, + * whereas HiveDecimal.create("0.01").precision() = 2). + * If you want the BigDecimal precision, use HiveDecimal.bigDecimalValue().precision() + * @return + */ public int precision() { - return bd.precision(); + int bdPrecision = bd.precision(); + int bdScale = bd.scale(); + + if (bdPrecision < bdScale) { + // This can happen for numbers less than 0.1 + // For 0.001234: bdPrecision=4, bdScale=6 + // In this case, we'll set the type to have the same precision as the scale. + return bdScale; + } + return bdPrecision; } public int intValue() { diff --git a/ql/src/test/queries/clientpositive/decimal_precision2.q b/ql/src/test/queries/clientpositive/decimal_precision2.q new file mode 100644 index 0000000..b5f6f6b --- /dev/null +++ b/ql/src/test/queries/clientpositive/decimal_precision2.q @@ -0,0 +1,15 @@ + +explain select 100.001BD; + +explain select 100.000BD; + +explain select 0.000BD; + +explain select 0.100BD; + +explain select 0.010BD; + +explain select cast(0.010 as decimal(6,3)); + +explain select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD; +select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD; diff --git a/ql/src/test/results/clientpositive/decimal_precision2.q.out b/ql/src/test/results/clientpositive/decimal_precision2.q.out new file mode 100644 index 0000000..16765db --- /dev/null +++ b/ql/src/test/results/clientpositive/decimal_precision2.q.out @@ -0,0 +1,163 @@ +PREHOOK: query: explain select 100.001BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 100.001BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 100.001 (type: decimal(6,3)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 100.000BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 100.000BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 100.000 (type: decimal(3,0)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 0.000BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 0.000BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 0.000 (type: decimal(1,0)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 0.100BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 0.100BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 0.100 (type: decimal(1,1)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 0.010BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 0.010BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 0.010 (type: decimal(2,2)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select cast(0.010 as decimal(6,3)) +PREHOOK: type: QUERY +POSTHOOK: query: explain select cast(0.010 as decimal(6,3)) +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 0.01 (type: decimal(6,3)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: explain select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD +PREHOOK: type: QUERY +POSTHOOK: query: explain select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: _dummy_table + Row Limit Per Split: 1 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + expressions: 69.0212249755859375 (type: decimal(27,20)) + outputColumnNames: _col0 + Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE + ListSink + +PREHOOK: query: select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +POSTHOOK: query: select 0.09765625BD * 0.09765625BD * 0.0125BD * 578992BD +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +#### A masked pattern was here #### +69.0212249755859375 diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java index a6ab7a7..313b5f6 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java @@ -674,8 +674,7 @@ static void serialize(ByteStream.Output buffer, Object o, ObjectInspector oi, case SHORT: { ShortObjectInspector spoi = (ShortObjectInspector) poi; short v = spoi.get(o); - writeByte(buffer, (byte) ((v >> 8) ^ 0x80), invert); - writeByte(buffer, (byte) v, invert); + serializeShort(buffer, v, invert); return; } case INT: { @@ -692,38 +691,12 @@ static void serialize(ByteStream.Output buffer, Object o, ObjectInspector oi, } case FLOAT: { FloatObjectInspector foi = (FloatObjectInspector) poi; - int v = Float.floatToIntBits(foi.get(o)); - if ((v & (1 << 31)) != 0) { - // negative number, flip all bits - v = ~v; - } else { - // positive number, flip the first bit - v = v ^ (1 << 31); - } - writeByte(buffer, (byte) (v >> 24), invert); - writeByte(buffer, (byte) (v >> 16), invert); - writeByte(buffer, (byte) (v >> 8), invert); - writeByte(buffer, (byte) v, invert); + serializeFloat(buffer, foi.get(o), invert); return; } case DOUBLE: { DoubleObjectInspector doi = (DoubleObjectInspector) poi; - long v = Double.doubleToLongBits(doi.get(o)); - if ((v & (1L << 63)) != 0) { - // negative number, flip all bits - v = ~v; - } else { - // positive number, flip the first bit - v = v ^ (1L << 63); - } - writeByte(buffer, (byte) (v >> 56), invert); - writeByte(buffer, (byte) (v >> 48), invert); - writeByte(buffer, (byte) (v >> 40), invert); - writeByte(buffer, (byte) (v >> 32), invert); - writeByte(buffer, (byte) (v >> 24), invert); - writeByte(buffer, (byte) (v >> 16), invert); - writeByte(buffer, (byte) (v >> 8), invert); - writeByte(buffer, (byte) v, invert); + serializeDouble(buffer, doi.get(o), invert); return; } case STRING: { @@ -768,26 +741,19 @@ static void serialize(ByteStream.Output buffer, Object o, ObjectInspector oi, case TIMESTAMP: { TimestampObjectInspector toi = (TimestampObjectInspector) poi; TimestampWritable t = toi.getPrimitiveWritableObject(o); - byte[] data = t.getBinarySortable(); - for (int i = 0; i < data.length; i++) { - writeByte(buffer, data[i], invert); - } + serializeTimestampWritable(buffer, t, invert); return; } case INTERVAL_YEAR_MONTH: { HiveIntervalYearMonthObjectInspector ioi = (HiveIntervalYearMonthObjectInspector) poi; HiveIntervalYearMonth intervalYearMonth = ioi.getPrimitiveJavaObject(o); - int totalMonths = intervalYearMonth.getTotalMonths(); - serializeInt(buffer, totalMonths, invert); + serializeHiveIntervalYearMonth(buffer, intervalYearMonth, invert); return; } case INTERVAL_DAY_TIME: { HiveIntervalDayTimeObjectInspector ioi = (HiveIntervalDayTimeObjectInspector) poi; HiveIntervalDayTime intervalDayTime = ioi.getPrimitiveJavaObject(o); - long totalSecs = intervalDayTime.getTotalSeconds(); - int nanos = intervalDayTime.getNanos(); - serializeLong(buffer, totalSecs, invert); - serializeInt(buffer, nanos, invert); + serializeHiveIntervalDayTime(buffer, intervalDayTime, invert); return; } case DECIMAL: { @@ -805,29 +771,7 @@ static void serialize(ByteStream.Output buffer, Object o, ObjectInspector oi, HiveDecimalObjectInspector boi = (HiveDecimalObjectInspector) poi; HiveDecimal dec = boi.getPrimitiveJavaObject(o); - - // get the sign of the big decimal - int sign = dec.compareTo(HiveDecimal.ZERO); - - // we'll encode the absolute value (sign is separate) - dec = dec.abs(); - - // get the scale factor to turn big decimal into a decimal < 1 - int factor = dec.precision() - dec.scale(); - factor = sign == 1 ? factor : -factor; - - // convert the absolute big decimal to string - dec.scaleByPowerOfTen(Math.abs(dec.scale())); - String digits = dec.unscaledValue().toString(); - - // finally write out the pieces (sign, scale, digits) - writeByte(buffer, (byte) ( sign + 1), invert); - writeByte(buffer, (byte) ((factor >> 24) ^ 0x80), invert); - writeByte(buffer, (byte) ( factor >> 16), invert); - writeByte(buffer, (byte) ( factor >> 8), invert); - writeByte(buffer, (byte) factor, invert); - serializeBytes(buffer, digits.getBytes(decimalCharSet), - digits.length(), sign == -1 ? !invert : invert); + serializeHiveDecimal(buffer, dec, invert); return; } @@ -918,6 +862,11 @@ public static void serializeBytes( writeByte(buffer, (byte) 0, invert); } + public static void serializeShort(ByteStream.Output buffer, short v, boolean invert) { + writeByte(buffer, (byte) ((v >> 8) ^ 0x80), invert); + writeByte(buffer, (byte) v, invert); + } + public static void serializeInt(ByteStream.Output buffer, int v, boolean invert) { writeByte(buffer, (byte) ((v >> 24) ^ 0x80), invert); writeByte(buffer, (byte) (v >> 16), invert); @@ -936,6 +885,88 @@ public static void serializeLong(ByteStream.Output buffer, long v, boolean inver writeByte(buffer, (byte) v, invert); } + public static void serializeFloat(ByteStream.Output buffer, float vf, boolean invert) { + int v = Float.floatToIntBits(vf); + if ((v & (1 << 31)) != 0) { + // negative number, flip all bits + v = ~v; + } else { + // positive number, flip the first bit + v = v ^ (1 << 31); + } + writeByte(buffer, (byte) (v >> 24), invert); + writeByte(buffer, (byte) (v >> 16), invert); + writeByte(buffer, (byte) (v >> 8), invert); + writeByte(buffer, (byte) v, invert); + } + + public static void serializeDouble(ByteStream.Output buffer, double vd, boolean invert) { + long v = Double.doubleToLongBits(vd); + if ((v & (1L << 63)) != 0) { + // negative number, flip all bits + v = ~v; + } else { + // positive number, flip the first bit + v = v ^ (1L << 63); + } + writeByte(buffer, (byte) (v >> 56), invert); + writeByte(buffer, (byte) (v >> 48), invert); + writeByte(buffer, (byte) (v >> 40), invert); + writeByte(buffer, (byte) (v >> 32), invert); + writeByte(buffer, (byte) (v >> 24), invert); + writeByte(buffer, (byte) (v >> 16), invert); + writeByte(buffer, (byte) (v >> 8), invert); + writeByte(buffer, (byte) v, invert); + } + + public static void serializeTimestampWritable(ByteStream.Output buffer, TimestampWritable t, boolean invert) { + byte[] data = t.getBinarySortable(); + for (int i = 0; i < data.length; i++) { + writeByte(buffer, data[i], invert); + } + } + + public static void serializeHiveIntervalYearMonth(ByteStream.Output buffer, + HiveIntervalYearMonth intervalYearMonth, boolean invert) { + int totalMonths = intervalYearMonth.getTotalMonths(); + serializeInt(buffer, totalMonths, invert); + } + + public static void serializeHiveIntervalDayTime(ByteStream.Output buffer, + HiveIntervalDayTime intervalDayTime, boolean invert) { + long totalSecs = intervalDayTime.getTotalSeconds(); + int nanos = intervalDayTime.getNanos(); + serializeLong(buffer, totalSecs, invert); + serializeInt(buffer, nanos, invert); + } + + public static void serializeHiveDecimal(ByteStream.Output buffer, HiveDecimal dec, boolean invert) { + // get the sign of the big decimal + int sign = dec.compareTo(HiveDecimal.ZERO); + + // we'll encode the absolute value (sign is separate) + dec = dec.abs(); + + // get the scale factor to turn big decimal into a decimal < 1 + // This relies on the BigDecimal precision value, which as of HIVE-10270 + // is now different from HiveDecimal.precision() + int factor = dec.bigDecimalValue().precision() - dec.bigDecimalValue().scale(); + factor = sign == 1 ? factor : -factor; + + // convert the absolute big decimal to string + dec.scaleByPowerOfTen(Math.abs(dec.scale())); + String digits = dec.unscaledValue().toString(); + + // finally write out the pieces (sign, scale, digits) + writeByte(buffer, (byte) ( sign + 1), invert); + writeByte(buffer, (byte) ((factor >> 24) ^ 0x80), invert); + writeByte(buffer, (byte) ( factor >> 16), invert); + writeByte(buffer, (byte) ( factor >> 8), invert); + writeByte(buffer, (byte) factor, invert); + serializeBytes(buffer, digits.getBytes(decimalCharSet), + digits.length(), sign == -1 ? !invert : invert); + } + @Override public SerDeStats getSerDeStats() { // no support for statistics diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableSerializeWrite.java b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableSerializeWrite.java index f62def8..6b2675c 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableSerializeWrite.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/fast/BinarySortableSerializeWrite.java @@ -147,8 +147,7 @@ public void writeShort(short v) throws IOException { // This field is not a null. BinarySortableSerDe.writeByte(output, (byte) 1, invert); - BinarySortableSerDe.writeByte(output, (byte) ((v >> 8) ^ 0x80), invert); - BinarySortableSerDe.writeByte(output, (byte) v, invert); + BinarySortableSerDe.serializeShort(output, v, invert); } /* @@ -174,15 +173,7 @@ public void writeLong(long v) throws IOException { // This field is not a null. BinarySortableSerDe.writeByte(output, (byte) 1, invert); - BinarySortableSerDe.writeByte(output, (byte) ((v >> 56) ^ 0x80), invert); - BinarySortableSerDe.writeByte(output, (byte) (v >> 48), invert); - BinarySortableSerDe.writeByte(output, (byte) (v >> 40), invert); - BinarySortableSerDe.writeByte(output, (byte) (v >> 32), invert); - BinarySortableSerDe.writeByte(output, (byte) (v >> 24), invert); - BinarySortableSerDe.writeByte(output, (byte) (v >> 16), invert); - BinarySortableSerDe.writeByte(output, (byte) (v >> 8), invert); - BinarySortableSerDe.writeByte(output, (byte) v, invert); - + BinarySortableSerDe.serializeLong(output, v, invert); } /* @@ -195,18 +186,7 @@ public void writeFloat(float vf) throws IOException { // This field is not a null. BinarySortableSerDe.writeByte(output, (byte) 1, invert); - int v = Float.floatToIntBits(vf); - if ((v & (1 << 31)) != 0) { - // negative number, flip all bits - v = ~v; - } else { - // positive number, flip the first bit - v = v ^ (1 << 31); - } - BinarySortableSerDe.writeByte(output, (byte) (v >> 24), invert); - BinarySortableSerDe.writeByte(output, (byte) (v >> 16), invert); - BinarySortableSerDe.writeByte(output, (byte) (v >> 8), invert); - BinarySortableSerDe.writeByte(output, (byte) v, invert); + BinarySortableSerDe.serializeFloat(output, vf, invert); } /* @@ -219,22 +199,7 @@ public void writeDouble(double vd) throws IOException { // This field is not a null. BinarySortableSerDe.writeByte(output, (byte) 1, invert); - long v = Double.doubleToLongBits(vd); - if ((v & (1L << 63)) != 0) { - // negative number, flip all bits - v = ~v; - } else { - // positive number, flip the first bit - v = v ^ (1L << 63); - } - BinarySortableSerDe.writeByte(output, (byte) (v >> 56), invert); - BinarySortableSerDe.writeByte(output, (byte) (v >> 48), invert); - BinarySortableSerDe.writeByte(output, (byte) (v >> 40), invert); - BinarySortableSerDe.writeByte(output, (byte) (v >> 32), invert); - BinarySortableSerDe.writeByte(output, (byte) (v >> 24), invert); - BinarySortableSerDe.writeByte(output, (byte) (v >> 16), invert); - BinarySortableSerDe.writeByte(output, (byte) (v >> 8), invert); - BinarySortableSerDe.writeByte(output, (byte) v, invert); + BinarySortableSerDe.serializeDouble(output, vd, invert); } /* @@ -341,10 +306,7 @@ public void writeTimestamp(Timestamp vt) throws IOException { BinarySortableSerDe.writeByte(output, (byte) 1, invert); tempTimestampWritable.set(vt); - byte[] data = tempTimestampWritable.getBinarySortable(); - for (int i = 0; i < data.length; i++) { - BinarySortableSerDe.writeByte(output, data[i], invert); - } + BinarySortableSerDe.serializeTimestampWritable(output, tempTimestampWritable, invert); } /* @@ -357,8 +319,7 @@ public void writeHiveIntervalYearMonth(HiveIntervalYearMonth viyt) throws IOExce // This field is not a null. BinarySortableSerDe.writeByte(output, (byte) 1, invert); - int totalMonths = viyt.getTotalMonths(); - BinarySortableSerDe.serializeInt(output, totalMonths, invert); + BinarySortableSerDe.serializeHiveIntervalYearMonth(output, viyt, invert); } @Override @@ -381,10 +342,7 @@ public void writeHiveIntervalDayTime(HiveIntervalDayTime vidt) throws IOExceptio // This field is not a null. BinarySortableSerDe.writeByte(output, (byte) 1, invert); - long totalSecs = vidt.getTotalSeconds(); - int nanos = vidt.getNanos(); - BinarySortableSerDe.serializeLong(output, totalSecs, invert); - BinarySortableSerDe.serializeInt(output, nanos, invert); + BinarySortableSerDe.serializeHiveIntervalDayTime(output, vidt, invert); } @Override @@ -410,39 +368,6 @@ public void writeHiveDecimal(HiveDecimal dec) throws IOException { // This field is not a null. BinarySortableSerDe.writeByte(output, (byte) 1, invert); - // decimals are encoded in three pieces: - // sign: 1, 2 or 3 for smaller, equal or larger than 0 respectively - // factor: Number that indicates the amount of digits you have to move - // the decimal point left or right until the resulting number is smaller - // than zero but has something other than 0 as the first digit. - // digits: which is a string of all the digits in the decimal. If the number - // is negative the binary string will be inverted to get the correct ordering. - // Example: 0.00123 - // Sign is 3 (bigger than 0) - // Factor is -2 (move decimal point 2 positions right) - // Digits are: 123 - - // get the sign of the big decimal - int sign = dec.compareTo(HiveDecimal.ZERO); - - // we'll encode the absolute value (sign is separate) - dec = dec.abs(); - - // get the scale factor to turn big decimal into a decimal < 1 - int factor = dec.precision() - dec.scale(); - factor = sign == 1 ? factor : -factor; - - // convert the absolute big decimal to string - dec.scaleByPowerOfTen(Math.abs(dec.scale())); - String digits = dec.unscaledValue().toString(); - - // finally write out the pieces (sign, scale, digits) - BinarySortableSerDe.writeByte(output, (byte) ( sign + 1), invert); - BinarySortableSerDe.writeByte(output, (byte) ((factor >> 24) ^ 0x80), invert); - BinarySortableSerDe.writeByte(output, (byte) ( factor >> 16), invert); - BinarySortableSerDe.writeByte(output, (byte) ( factor >> 8), invert); - BinarySortableSerDe.writeByte(output, (byte) factor, invert); - BinarySortableSerDe.serializeBytes(output, digits.getBytes(BinarySortableSerDe.decimalCharSet), - digits.length(), sign == -1 ? !invert : invert); + BinarySortableSerDe.serializeHiveDecimal(output, dec, invert); } } \ No newline at end of file diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java index 370e857..36f5b09 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestClass.java @@ -96,4 +96,109 @@ public int randomFill(Random r, ExtraTypeInfo extraTypeInfo) { return result; } + public void nonRandomFill(int idx) { + myByte = (Byte) getNonRandValue(nrByte, idx); + myShort = (Short) getNonRandValue(nrShort, idx); + myInt = (Integer) getNonRandValue(nrInt, idx); + myLong = (Long) getNonRandValue(nrLong, idx); + myFloat = (Float) getNonRandValue(nrFloat, idx); + myDouble = (Double) getNonRandValue(nrDouble, idx); + myString = (String) getNonRandValue(nrString, idx); + myHiveChar = new HiveChar(myString, myString.length()); + myHiveVarchar = new HiveVarchar(myString, myString.length()); + myDecimal = (HiveDecimal) getNonRandValue(nrDecimal, idx); + myDate = (Date) getNonRandValue(nrDate, idx); + myIntervalYearMonth = (HiveIntervalYearMonth) getNonRandValue(nrIntervalYearMonth, idx); + myIntervalDayTime = (HiveIntervalDayTime) getNonRandValue(nrIntervalDayTime, idx); + myStruct = null; + myList = null; + } + + public static Object getNonRandValue(Object[] nrArray, int index) { + return nrArray[index % nrArray.length]; + } + + static Object[] nrByte = { + Byte.valueOf((byte) 1) + }; + + static Object[] nrShort = { + Short.valueOf((short) 1) + }; + + static Object[] nrInt = { + Integer.valueOf(1) + }; + + static Object[] nrLong = { + Long.valueOf(1) + }; + + static Object[] nrFloat = { + Float.valueOf(1.0f) + }; + + static Object[] nrDouble = { + Double.valueOf(1.0) + }; + + static Object[] nrDecimal = { + HiveDecimal.create("100"), + HiveDecimal.create("10"), + HiveDecimal.create("1"), + HiveDecimal.create("0"), + HiveDecimal.create("0.1"), + HiveDecimal.create("0.01"), + HiveDecimal.create("0.001"), + HiveDecimal.create("-100"), + HiveDecimal.create("-10"), + HiveDecimal.create("-1"), + HiveDecimal.create("-0.1"), + HiveDecimal.create("-0.01"), + HiveDecimal.create("-0.001"), + HiveDecimal.create("12345678900"), + HiveDecimal.create("1234567890"), + HiveDecimal.create("123456789"), + HiveDecimal.create("12345678.9"), + HiveDecimal.create("1234567.89"), + HiveDecimal.create("123456.789"), + HiveDecimal.create("12345.6789"), + HiveDecimal.create("1234.56789"), + HiveDecimal.create("123.456789"), + HiveDecimal.create("1.23456789"), + HiveDecimal.create("0.123456789"), + HiveDecimal.create("0.0123456789"), + HiveDecimal.create("0.00123456789"), + HiveDecimal.create("0.000123456789"), + HiveDecimal.create("-12345678900"), + HiveDecimal.create("-1234567890"), + HiveDecimal.create("-123456789"), + HiveDecimal.create("-12345678.9"), + HiveDecimal.create("-1234567.89"), + HiveDecimal.create("-123456.789"), + HiveDecimal.create("-12345.6789"), + HiveDecimal.create("-1234.56789"), + HiveDecimal.create("-123.456789"), + HiveDecimal.create("-1.23456789"), + HiveDecimal.create("-0.123456789"), + HiveDecimal.create("-0.0123456789"), + HiveDecimal.create("-0.00123456789"), + HiveDecimal.create("-0.000123456789"), + }; + + static Object[] nrString = { + "abcdefg" + }; + + static Object[] nrDate = { + Date.valueOf("2001-01-01") + }; + + static Object[] nrIntervalYearMonth = { + HiveIntervalYearMonth.valueOf("1-0") + }; + + static Object[] nrIntervalDayTime = { + HiveIntervalDayTime.valueOf("1 0:0:0") + }; } diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestPrimitiveClass.java b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestPrimitiveClass.java index 2e2327b..299a56c 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestPrimitiveClass.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/MyTestPrimitiveClass.java @@ -450,4 +450,24 @@ public StructObjectInspector getRowInspector(PrimitiveTypeInfo[] primitiveTypeIn StandardStructObjectInspector rowOI = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, primitiveObjectInspectorList); return rowOI; } + + public void nonRandomFill(int idx, ExtraTypeInfo extraTypeInfo) { + myByte = (Byte) MyTestClass.getNonRandValue(MyTestClass.nrByte, idx); + myShort = (Short) MyTestClass.getNonRandValue(MyTestClass.nrShort, idx); + myInt = (Integer) MyTestClass.getNonRandValue(MyTestClass.nrInt, idx); + myLong = (Long) MyTestClass.getNonRandValue(MyTestClass.nrLong, idx); + myFloat = (Float) MyTestClass.getNonRandValue(MyTestClass.nrFloat, idx); + myDouble = (Double) MyTestClass.getNonRandValue(MyTestClass.nrDouble, idx); + myString = (String) MyTestClass.getNonRandValue(MyTestClass.nrString, idx); + myHiveChar = new HiveChar(myString, myString.length()); + extraTypeInfo.hiveCharMaxLength = myString.length(); + myHiveVarchar = new HiveVarchar(myString, myString.length()); + extraTypeInfo.hiveVarcharMaxLength = myString.length(); + myDecimal = (HiveDecimal) MyTestClass.getNonRandValue(MyTestClass.nrDecimal, idx); + extraTypeInfo.precision = myDecimal.precision(); + extraTypeInfo.scale = myDecimal.scale(); + myDate = (Date) MyTestClass.getNonRandValue(MyTestClass.nrDate, idx); + myIntervalYearMonth = (HiveIntervalYearMonth) MyTestClass.getNonRandValue(MyTestClass.nrIntervalYearMonth, idx); + myIntervalDayTime = (HiveIntervalDayTime) MyTestClass.getNonRandValue(MyTestClass.nrIntervalDayTime, idx); + } } diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java index 83d3839..4438bdc 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableFast.java @@ -196,7 +196,18 @@ public void testBinarySortableFast() throws Throwable { // Need a map because we sort. Map primitiveTypeInfoMap = new HashMap(); - for (int i = 0; i < num; i++) { + int i; + // First try non-random values + for (i = 0; i < MyTestClass.nrDecimal.length; i++) { + MyTestPrimitiveClass t = new MyTestPrimitiveClass(); + ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo(); + t.nonRandomFill(i, extraTypeInfo); + myTestPrimitiveClasses[i] = t; + PrimitiveTypeInfo[] primitiveTypeInfos = MyTestPrimitiveClass.getPrimitiveTypeInfos(extraTypeInfo); + primitiveTypeInfoMap.put(t, primitiveTypeInfos); + } + + for ( ; i < num; i++) { int randField = r.nextInt(MyTestPrimitiveClass.primitiveCount); MyTestPrimitiveClass t = new MyTestPrimitiveClass(); int field = 0; diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java index b3fb3be..af47e6f 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/binarysortable/TestBinarySortableSerDe.java @@ -147,7 +147,15 @@ public void testBinarySortableSerDe() throws Throwable { Random r = new Random(1234); MyTestClass rows[] = new MyTestClass[num]; - for (int i = 0; i < num; i++) { + int i; + // First try non-random values + for (i = 0; i < MyTestClass.nrDecimal.length; i++) { + MyTestClass t = new MyTestClass(); + t.nonRandomFill(i); + rows[i] = t; + } + + for ( ; i < num; i++) { MyTestClass t = new MyTestClass(); ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo(); t.randomFill(r, extraTypeInfo);