diff --git itests/hive-jmh/src/main/java/org/apache/hive/benchmark/serde/LazySimpleSerDeBench.java itests/hive-jmh/src/main/java/org/apache/hive/benchmark/serde/LazySimpleSerDeBench.java
index 826bf53..a87123e 100644
--- itests/hive-jmh/src/main/java/org/apache/hive/benchmark/serde/LazySimpleSerDeBench.java
+++ itests/hive-jmh/src/main/java/org/apache/hive/benchmark/serde/LazySimpleSerDeBench.java
@@ -15,6 +15,7 @@
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.Random;
import java.util.concurrent.TimeUnit;
@@ -28,6 +29,8 @@
import org.apache.hadoop.hive.serde2.lazy.LazyLong;
import org.apache.hadoop.hive.serde2.lazy.LazyShort;
import org.apache.hadoop.hive.serde2.lazy.LazyTimestamp;
+import org.apache.hadoop.hive.serde2.lazy.fast.ByteString;
+import org.apache.hadoop.hive.serde2.lazy.fast.FloatingDecimal;
import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -62,7 +65,7 @@
@BenchmarkMode(Mode.AverageTime)
@Fork(1)
@State(Scope.Thread)
- @OutputTimeUnit(TimeUnit.NANOSECONDS)
+ @OutputTimeUnit(TimeUnit.MILLISECONDS)
public static abstract class AbstractDeserializer {
public int[] offsets = new int[DEFAULT_DATA_SIZE];
@@ -449,8 +452,33 @@ public void bench() {
@BenchmarkMode(Mode.AverageTime)
@Fork(1)
+ @OutputTimeUnit(TimeUnit.MILLISECONDS)
+ @Warmup(iterations = 4, time = 2, timeUnit = TimeUnit.MILLISECONDS)
+ @Measurement(iterations = 4, time = 2, timeUnit = TimeUnit.MILLISECONDS)
@State(Scope.Thread)
- @OutputTimeUnit(TimeUnit.NANOSECONDS)
+ public static class ParseDouble {
+ byte[] bytes = "1234567890.1234567890".getBytes(StandardCharsets.UTF_8);
+ FloatingDecimal floatingDecimal = new FloatingDecimal();
+
+ @Benchmark
+ public void floatingDecimalBench() {
+ for (int i = 0; i < DEFAULT_ITER_TIME; i++) {
+ floatingDecimal.parseDouble(new ByteString(bytes, 0, bytes.length));
+ }
+ }
+
+ @Benchmark
+ public void doubleBench() {
+ for (int i = 0; i < DEFAULT_ITER_TIME; i++) {
+ Double.parseDouble(new String(bytes, 0, bytes.length, StandardCharsets.UTF_8));
+ }
+ }
+ }
+
+ @BenchmarkMode(Mode.AverageTime)
+ @Fork(1)
+ @State(Scope.Thread)
+ @OutputTimeUnit(TimeUnit.MILLISECONDS)
public static class GoodLazyDate {
final LazyDate obj = new LazyDate(
@@ -533,7 +561,7 @@ public void bench() {
@BenchmarkMode(Mode.AverageTime)
@Fork(1)
@State(Scope.Thread)
- @OutputTimeUnit(TimeUnit.NANOSECONDS)
+ @OutputTimeUnit(TimeUnit.MILLISECONDS)
public static class GoodLazyTimestamp {
final LazyTimestamp obj = new LazyTimestamp(
diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/ByteString.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/ByteString.java
new file mode 100644
index 0000000..b2db9dd
--- /dev/null
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/ByteString.java
@@ -0,0 +1,66 @@
+package org.apache.hadoop.hive.serde2.lazy.fast;
+
+import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
+
+import java.nio.charset.StandardCharsets;
+
+public class ByteString {
+ private byte[] bytes;
+ private int start;
+ private int length;
+
+ public ByteString(byte[] bytes, int start, int length) {
+ set(bytes, start, length);
+ }
+
+ void set(byte[] bytes, int start, int length) {
+ this.bytes = bytes;
+ this.start = start;
+ this.length = length;
+ }
+
+ public ByteString(String string) {
+ byte[] bytes = string.getBytes(StandardCharsets.UTF_8);
+ set(bytes, 0, bytes.length);
+ }
+
+ ByteString trim() {
+ int end = start + length - 1;
+ for (int i = end; i >= start; i--) {
+ if (bytes[i] != ' ') {
+ end = i;
+ break;
+ }
+ }
+ for (int i = start; i < end; i++) {
+ if (bytes[i] != ' ') {
+ start = i;
+ break;
+ }
+ }
+ length = end - start;
+ return this;
+ }
+
+ int length() {
+ return length;
+ }
+
+ byte charAt(int i) {
+ return bytes[i];
+ }
+
+ @Override
+ public String toString() {
+ return new String(bytes, start, length, StandardCharsets.UTF_8);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj.getClass() != ByteString.class) {
+ return false;
+ }
+ ByteString that = (ByteString) obj;
+ return StringExpr.equal(this.bytes, this.start, this.length, that.bytes, that.start, that.length);
+ }
+}
diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/FDBigInteger.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/FDBigInteger.java
new file mode 100644
index 0000000..c30626f
--- /dev/null
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/FDBigInteger.java
@@ -0,0 +1,1467 @@
+package org.apache.hadoop.hive.serde2.lazy.fast;
+
+import java.math.BigInteger;
+import java.util.Arrays;
+
+/**
+ * A simple big integer package specifically for floating point base conversion.
+ */
+public /*@ spec_bigint_math @*/ class FDBigInteger {
+
+ //
+ // This class contains many comments that start with "/*@" mark.
+ // They are behavourial specification in
+ // the Java Modelling Language (JML):
+ // http://www.eecs.ucf.edu/~leavens/JML//index.shtml
+ //
+
+ /*@
+ @ public pure model static \bigint UNSIGNED(int v) {
+ @ return v >= 0 ? v : v + (((\bigint)1) << 32);
+ @ }
+ @
+ @ public pure model static \bigint UNSIGNED(long v) {
+ @ return v >= 0 ? v : v + (((\bigint)1) << 64);
+ @ }
+ @
+ @ public pure model static \bigint AP(int[] data, int len) {
+ @ return (\sum int i; 0 <= 0 && i < len; UNSIGNED(data[i]) << (i*32));
+ @ }
+ @
+ @ public pure model static \bigint pow52(int p5, int p2) {
+ @ ghost \bigint v = 1;
+ @ for (int i = 0; i < p5; i++) v *= 5;
+ @ return v << p2;
+ @ }
+ @
+ @ public pure model static \bigint pow10(int p10) {
+ @ return pow52(p10, p10);
+ @ }
+ @*/
+
+ static final int[] SMALL_5_POW = {
+ 1,
+ 5,
+ 5 * 5,
+ 5 * 5 * 5,
+ 5 * 5 * 5 * 5,
+ 5 * 5 * 5 * 5 * 5,
+ 5 * 5 * 5 * 5 * 5 * 5,
+ 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5
+ };
+
+ static final long[] LONG_5_POW = {
+ 1L,
+ 5L,
+ 5L * 5,
+ 5L * 5 * 5,
+ 5L * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ 5L * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5,
+ };
+
+ // Maximum size of cache of powers of 5 as FDBigIntegers.
+ private static final int MAX_FIVE_POW = 340;
+
+ // Cache of big powers of 5 as FDBigIntegers.
+ private static final FDBigInteger POW_5_CACHE[];
+
+ // Initialize FDBigInteger cache of powers of 5.
+ static {
+ POW_5_CACHE = new FDBigInteger[MAX_FIVE_POW];
+ int i = 0;
+ while (i < SMALL_5_POW.length) {
+ FDBigInteger pow5 = new FDBigInteger(new int[]{SMALL_5_POW[i]}, 0);
+ pow5.makeImmutable();
+ POW_5_CACHE[i] = pow5;
+ i++;
+ }
+ FDBigInteger prev = POW_5_CACHE[i - 1];
+ while (i < MAX_FIVE_POW) {
+ POW_5_CACHE[i] = prev = prev.mult(5);
+ prev.makeImmutable();
+ i++;
+ }
+ }
+
+ // Zero as an FDBigInteger.
+ public static final FDBigInteger ZERO = new FDBigInteger(new int[0], 0);
+
+ // Ensure ZERO is immutable.
+ static {
+ ZERO.makeImmutable();
+ }
+
+ // Constant for casting an int to a long via bitwise AND.
+ private final static long LONG_MASK = 0xffffffffL;
+
+ //@ spec_public non_null;
+ private int data[]; // value: data[0] is least significant
+ //@ spec_public;
+ private int offset; // number of least significant zero padding ints
+ //@ spec_public;
+ private int nWords; // data[nWords-1]!=0, all values above are zero
+ // if nWords==0 -> this FDBigInteger is zero
+ //@ spec_public;
+ private boolean isImmutable = false;
+
+ /*@
+ @ public invariant 0 <= nWords && nWords <= data.length && offset >= 0;
+ @ public invariant nWords == 0 ==> offset == 0;
+ @ public invariant nWords > 0 ==> data[nWords - 1] != 0;
+ @ public invariant (\forall int i; nWords <= i && i < data.length; data[i] == 0);
+ @ public pure model \bigint value() {
+ @ return AP(data, nWords) << (offset*32);
+ @ }
+ @*/
+
+ /**
+ * Constructs an FDBigInteger from data and padding. The
+ * data parameter has the least significant int at
+ * the zeroth index. The offset parameter gives the number of
+ * zero ints to be inferred below the least significant element
+ * of data.
+ *
+ * @param data An array containing all non-zero ints of the value.
+ * @param offset An offset indicating the number of zero ints to pad
+ * below the least significant element of data.
+ */
+ /*@
+ @ requires data != null && offset >= 0;
+ @ ensures this.value() == \old(AP(data, data.length) << (offset*32));
+ @ ensures this.data == \old(data);
+ @*/
+ private FDBigInteger(int[] data, int offset) {
+ this.data = data;
+ this.offset = offset;
+ this.nWords = data.length;
+ trimLeadingZeros();
+ }
+
+ /**
+ * Constructs an FDBigInteger from a starting value and some
+ * decimal digits.
+ *
+ * @param lValue The starting value.
+ * @param digits The decimal digits.
+ * @param kDigits The initial index into digits.
+ * @param nDigits The final index into digits.
+ */
+ /*@
+ @ requires digits != null;
+ @ requires 0 <= kDigits && kDigits <= nDigits && nDigits <= digits.length;
+ @ requires (\forall int i; 0 <= i && i < nDigits; '0' <= digits[i] && digits[i] <= '9');
+ @ ensures this.value() == \old(lValue * pow10(nDigits - kDigits) + (\sum int i; kDigits <= i && i < nDigits; (digits[i] - '0') * pow10(nDigits - i - 1)));
+ @*/
+ public FDBigInteger(long lValue, byte[] digits, int kDigits, int nDigits) {
+ int n = Math.max((nDigits + 8) / 9, 2); // estimate size needed.
+ data = new int[n]; // allocate enough space
+ data[0] = (int) lValue; // starting value
+ data[1] = (int) (lValue >>> 32);
+ offset = 0;
+ nWords = 2;
+ int i = kDigits;
+ int limit = nDigits - 5; // slurp digits 5 at a time.
+ int v;
+ while (i < limit) {
+ int ilim = i + 5;
+ v = (int) digits[i++] - (int) '0';
+ while (i < ilim) {
+ v = 10 * v + (int) digits[i++] - (int) '0';
+ }
+ multAddMe(100000, v); // ... where 100000 is 10^5.
+ }
+ int factor = 1;
+ v = 0;
+ while (i < nDigits) {
+ v = 10 * v + (int) digits[i++] - (int) '0';
+ factor *= 10;
+ }
+ if (factor != 1) {
+ multAddMe(factor, v);
+ }
+ trimLeadingZeros();
+ }
+
+ /**
+ * Returns an FDBigInteger with the numerical value
+ * 5p5 * 2p2.
+ *
+ * @param p5 The exponent of the power-of-five factor.
+ * @param p2 The exponent of the power-of-two factor.
+ * @return 5p5 * 2p2
+ */
+ /*@
+ @ requires p5 >= 0 && p2 >= 0;
+ @ assignable \nothing;
+ @ ensures \result.value() == \old(pow52(p5, p2));
+ @*/
+ public static FDBigInteger valueOfPow52(int p5, int p2) {
+ if (p5 != 0) {
+ if (p2 == 0) {
+ return big5pow(p5);
+ } else if (p5 < SMALL_5_POW.length) {
+ int pow5 = SMALL_5_POW[p5];
+ int wordcount = p2 >> 5;
+ int bitcount = p2 & 0x1f;
+ if (bitcount == 0) {
+ return new FDBigInteger(new int[]{pow5}, wordcount);
+ } else {
+ return new FDBigInteger(new int[]{
+ pow5 << bitcount,
+ pow5 >>> (32 - bitcount)
+ }, wordcount);
+ }
+ } else {
+ return big5pow(p5).leftShift(p2);
+ }
+ } else {
+ return valueOfPow2(p2);
+ }
+ }
+
+ /**
+ * Returns an FDBigInteger with the numerical value
+ * value * 5p5 * 2p2.
+ *
+ * @param value The constant factor.
+ * @param p5 The exponent of the power-of-five factor.
+ * @param p2 The exponent of the power-of-two factor.
+ * @return value * 5p5 * 2p2
+ */
+ /*@
+ @ requires p5 >= 0 && p2 >= 0;
+ @ assignable \nothing;
+ @ ensures \result.value() == \old(UNSIGNED(value) * pow52(p5, p2));
+ @*/
+ public static FDBigInteger valueOfMulPow52(long value, int p5, int p2) {
+ assert p5 >= 0 : p5;
+ assert p2 >= 0 : p2;
+ int v0 = (int) value;
+ int v1 = (int) (value >>> 32);
+ int wordcount = p2 >> 5;
+ int bitcount = p2 & 0x1f;
+ if (p5 != 0) {
+ if (p5 < SMALL_5_POW.length) {
+ long pow5 = SMALL_5_POW[p5] & LONG_MASK;
+ long carry = (v0 & LONG_MASK) * pow5;
+ v0 = (int) carry;
+ carry >>>= 32;
+ carry = (v1 & LONG_MASK) * pow5 + carry;
+ v1 = (int) carry;
+ int v2 = (int) (carry >>> 32);
+ if (bitcount == 0) {
+ return new FDBigInteger(new int[]{v0, v1, v2}, wordcount);
+ } else {
+ return new FDBigInteger(new int[]{
+ v0 << bitcount,
+ (v1 << bitcount) | (v0 >>> (32 - bitcount)),
+ (v2 << bitcount) | (v1 >>> (32 - bitcount)),
+ v2 >>> (32 - bitcount)
+ }, wordcount);
+ }
+ } else {
+ FDBigInteger pow5 = big5pow(p5);
+ int[] r;
+ if (v1 == 0) {
+ r = new int[pow5.nWords + 1 + ((p2 != 0) ? 1 : 0)];
+ mult(pow5.data, pow5.nWords, v0, r);
+ } else {
+ r = new int[pow5.nWords + 2 + ((p2 != 0) ? 1 : 0)];
+ mult(pow5.data, pow5.nWords, v0, v1, r);
+ }
+ return (new FDBigInteger(r, pow5.offset)).leftShift(p2);
+ }
+ } else if (p2 != 0) {
+ if (bitcount == 0) {
+ return new FDBigInteger(new int[]{v0, v1}, wordcount);
+ } else {
+ return new FDBigInteger(new int[]{
+ v0 << bitcount,
+ (v1 << bitcount) | (v0 >>> (32 - bitcount)),
+ v1 >>> (32 - bitcount)
+ }, wordcount);
+ }
+ }
+ return new FDBigInteger(new int[]{v0, v1}, 0);
+ }
+
+ /**
+ * Returns an FDBigInteger with the numerical value
+ * 2p2.
+ *
+ * @param p2 The exponent of 2.
+ * @return 2p2
+ */
+ /*@
+ @ requires p2 >= 0;
+ @ assignable \nothing;
+ @ ensures \result.value() == pow52(0, p2);
+ @*/
+ private static FDBigInteger valueOfPow2(int p2) {
+ int wordcount = p2 >> 5;
+ int bitcount = p2 & 0x1f;
+ return new FDBigInteger(new int[]{1 << bitcount}, wordcount);
+ }
+
+ /**
+ * Removes all leading zeros from this FDBigInteger adjusting
+ * the offset and number of non-zero leading words accordingly.
+ */
+ /*@
+ @ requires data != null;
+ @ requires 0 <= nWords && nWords <= data.length && offset >= 0;
+ @ requires nWords == 0 ==> offset == 0;
+ @ ensures nWords == 0 ==> offset == 0;
+ @ ensures nWords > 0 ==> data[nWords - 1] != 0;
+ @*/
+ private /*@ helper @*/ void trimLeadingZeros() {
+ int i = nWords;
+ if (i > 0 && (data[--i] == 0)) {
+ //for (; i > 0 && data[i - 1] == 0; i--) ;
+ while(i > 0 && data[i - 1] == 0) {
+ i--;
+ }
+ this.nWords = i;
+ if (i == 0) { // all words are zero
+ this.offset = 0;
+ }
+ }
+ }
+
+ /**
+ * Retrieves the normalization bias of the FDBigIntger. The
+ * normalization bias is a left shift such that after it the highest word
+ * of the value will have the 4 highest bits equal to zero:
+ * (highestWord & 0xf0000000) == 0, but the next bit should be 1
+ * (highestWord & 0x08000000) != 0.
+ *
+ * @return The normalization bias.
+ */
+ /*@
+ @ requires this.value() > 0;
+ @*/
+ public /*@ pure @*/ int getNormalizationBias() {
+ if (nWords == 0) {
+ throw new IllegalArgumentException("Zero value cannot be normalized");
+ }
+ int zeros = Integer.numberOfLeadingZeros(data[nWords - 1]);
+ return (zeros < 4) ? 28 + zeros : zeros - 4;
+ }
+
+ // TODO: Why is anticount param needed if it is always 32 - bitcount?
+ /**
+ * Left shifts the contents of one int array into another.
+ *
+ * @param src The source array.
+ * @param idx The initial index of the source array.
+ * @param result The destination array.
+ * @param bitcount The left shift.
+ * @param anticount The left anti-shift, e.g., 32-bitcount.
+ * @param prev The prior source value.
+ */
+ /*@
+ @ requires 0 < bitcount && bitcount < 32 && anticount == 32 - bitcount;
+ @ requires src.length >= idx && result.length > idx;
+ @ assignable result[*];
+ @ ensures AP(result, \old(idx + 1)) == \old((AP(src, idx) + UNSIGNED(prev) << (idx*32)) << bitcount);
+ @*/
+ private static void leftShift(int[] src, int idx, int result[], int bitcount, int anticount, int prev){
+ for (; idx > 0; idx--) {
+ int v = (prev << bitcount);
+ prev = src[idx - 1];
+ v |= (prev >>> anticount);
+ result[idx] = v;
+ }
+ int v = prev << bitcount;
+ result[0] = v;
+ }
+
+ /**
+ * Shifts this FDBigInteger to the left. The shift is performed
+ * in-place unless the FDBigInteger is immutable in which case
+ * a new instance of FDBigInteger is returned.
+ *
+ * @param shift The number of bits to shift left.
+ * @return The shifted FDBigInteger.
+ */
+ /*@
+ @ requires this.value() == 0 || shift == 0;
+ @ assignable \nothing;
+ @ ensures \result == this;
+ @
+ @ also
+ @
+ @ requires this.value() > 0 && shift > 0 && this.isImmutable;
+ @ assignable \nothing;
+ @ ensures \result.value() == \old(this.value() << shift);
+ @
+ @ also
+ @
+ @ requires this.value() > 0 && shift > 0 && this.isImmutable;
+ @ assignable \nothing;
+ @ ensures \result == this;
+ @ ensures \result.value() == \old(this.value() << shift);
+ @*/
+ public FDBigInteger leftShift(int shift) {
+ if (shift == 0 || nWords == 0) {
+ return this;
+ }
+ int wordcount = shift >> 5;
+ int bitcount = shift & 0x1f;
+ if (this.isImmutable) {
+ if (bitcount == 0) {
+ return new FDBigInteger(Arrays.copyOf(data, nWords), offset + wordcount);
+ } else {
+ int anticount = 32 - bitcount;
+ int idx = nWords - 1;
+ int prev = data[idx];
+ int hi = prev >>> anticount;
+ int[] result;
+ if (hi != 0) {
+ result = new int[nWords + 1];
+ result[nWords] = hi;
+ } else {
+ result = new int[nWords];
+ }
+ leftShift(data,idx,result,bitcount,anticount,prev);
+ return new FDBigInteger(result, offset + wordcount);
+ }
+ } else {
+ if (bitcount != 0) {
+ int anticount = 32 - bitcount;
+ if ((data[0] << bitcount) == 0) {
+ int idx = 0;
+ int prev = data[idx];
+ for (; idx < nWords - 1; idx++) {
+ int v = (prev >>> anticount);
+ prev = data[idx + 1];
+ v |= (prev << bitcount);
+ data[idx] = v;
+ }
+ int v = prev >>> anticount;
+ data[idx] = v;
+ if(v==0) {
+ nWords--;
+ }
+ offset++;
+ } else {
+ int idx = nWords - 1;
+ int prev = data[idx];
+ int hi = prev >>> anticount;
+ int[] result = data;
+ int[] src = data;
+ if (hi != 0) {
+ if(nWords == data.length) {
+ data = result = new int[nWords + 1];
+ }
+ result[nWords++] = hi;
+ }
+ leftShift(src,idx,result,bitcount,anticount,prev);
+ }
+ }
+ offset += wordcount;
+ return this;
+ }
+ }
+
+ /**
+ * Returns the number of ints this FDBigInteger represents.
+ *
+ * @return Number of ints required to represent this FDBigInteger.
+ */
+ /*@
+ @ requires this.value() == 0;
+ @ ensures \result == 0;
+ @
+ @ also
+ @
+ @ requires this.value() > 0;
+ @ ensures ((\bigint)1) << (\result - 1) <= this.value() && this.value() <= ((\bigint)1) << \result;
+ @*/
+ private /*@ pure @*/ int size() {
+ return nWords + offset;
+ }
+
+
+ /*@
+ @ requires !this.isImmutable;
+ @ requires this.size() <= S.size();
+ @ requires this.data.length + this.offset >= S.size();
+ @ requires S.value() >= ((\bigint)1) << (S.size()*32 - 4);
+ @ assignable this.nWords, this.offset, this.data, this.data[*];
+ @ ensures \result == \old(this.value() / S.value());
+ @ ensures this.value() == \old(10 * (this.value() % S.value()));
+ @*/
+ public int quoRemIteration(FDBigInteger S) throws IllegalArgumentException {
+ assert !this.isImmutable : "cannot modify immutable value";
+ // ensure that this and S have the same number of
+ // digits. If S is properly normalized and q < 10 then
+ // this must be so.
+ int thSize = this.size();
+ int sSize = S.size();
+ if (thSize < sSize) {
+ // this value is significantly less than S, result of division is zero.
+ // just mult this by 10.
+ int p = multAndCarryBy10(this.data, this.nWords, this.data);
+ if(p!=0) {
+ this.data[nWords++] = p;
+ } else {
+ trimLeadingZeros();
+ }
+ return 0;
+ } else if (thSize > sSize) {
+ throw new IllegalArgumentException("disparate values");
+ }
+ // estimate q the obvious way. We will usually be
+ // right. If not, then we're only off by a little and
+ // will re-add.
+ long q = (this.data[this.nWords - 1] & LONG_MASK) / (S.data[S.nWords - 1] & LONG_MASK);
+ long diff = multDiffMe(q, S);
+ if (diff != 0L) {
+ //@ assert q != 0;
+ //@ assert this.offset == \old(Math.min(this.offset, S.offset));
+ //@ assert this.offset <= S.offset;
+
+ // q is too big.
+ // add S back in until this turns +. This should
+ // not be very many times!
+ long sum = 0L;
+ int tStart = S.offset - this.offset;
+ //@ assert tStart >= 0;
+ int[] sd = S.data;
+ int[] td = this.data;
+ while (sum == 0L) {
+ for (int sIndex = 0, tIndex = tStart; tIndex < this.nWords; sIndex++, tIndex++) {
+ sum += (td[tIndex] & LONG_MASK) + (sd[sIndex] & LONG_MASK);
+ td[tIndex] = (int) sum;
+ sum >>>= 32; // Signed or unsigned, answer is 0 or 1
+ }
+ //
+ // Originally the following line read
+ // "if ( sum !=0 && sum != -1 )"
+ // but that would be wrong, because of the
+ // treatment of the two values as entirely unsigned,
+ // it would be impossible for a carry-out to be interpreted
+ // as -1 -- it would have to be a single-bit carry-out, or +1.
+ //
+ assert sum == 0 || sum == 1 : sum; // carry out of division correction
+ q -= 1;
+ }
+ }
+ // finally, we can multiply this by 10.
+ // it cannot overflow, right, as the high-order word has
+ // at least 4 high-order zeros!
+ int p = multAndCarryBy10(this.data, this.nWords, this.data);
+ assert p == 0 : p; // Carry out of *10
+ trimLeadingZeros();
+ return (int) q;
+ }
+
+ /**
+ * Multiplies this FDBigInteger by 10. The operation will be
+ * performed in place unless the FDBigInteger is immutable in
+ * which case a new FDBigInteger will be returned.
+ *
+ * @return The FDBigInteger multiplied by 10.
+ */
+ /*@
+ @ requires this.value() == 0;
+ @ assignable \nothing;
+ @ ensures \result == this;
+ @
+ @ also
+ @
+ @ requires this.value() > 0 && this.isImmutable;
+ @ assignable \nothing;
+ @ ensures \result.value() == \old(this.value() * 10);
+ @
+ @ also
+ @
+ @ requires this.value() > 0 && !this.isImmutable;
+ @ assignable this.nWords, this.data, this.data[*];
+ @ ensures \result == this;
+ @ ensures \result.value() == \old(this.value() * 10);
+ @*/
+ public FDBigInteger multBy10() {
+ if (nWords == 0) {
+ return this;
+ }
+ if (isImmutable) {
+ int[] res = new int[nWords + 1];
+ res[nWords] = multAndCarryBy10(data, nWords, res);
+ return new FDBigInteger(res, offset);
+ } else {
+ int p = multAndCarryBy10(this.data, this.nWords, this.data);
+ if (p != 0) {
+ if (nWords == data.length) {
+ if (data[0] == 0) {
+ System.arraycopy(data, 1, data, 0, --nWords);
+ offset++;
+ } else {
+ data = Arrays.copyOf(data, data.length + 1);
+ }
+ }
+ data[nWords++] = p;
+ } else {
+ trimLeadingZeros();
+ }
+ return this;
+ }
+ }
+
+ /**
+ * Multiplies this FDBigInteger by
+ * 5p5 * 2p2. The operation will be
+ * performed in place if possible, otherwise a new FDBigInteger
+ * will be returned.
+ *
+ * @param p5 The exponent of the power-of-five factor.
+ * @param p2 The exponent of the power-of-two factor.
+ * @return
+ */
+ /*@
+ @ requires this.value() == 0 || p5 == 0 && p2 == 0;
+ @ assignable \nothing;
+ @ ensures \result == this;
+ @
+ @ also
+ @
+ @ requires this.value() > 0 && (p5 > 0 && p2 >= 0 || p5 == 0 && p2 > 0 && this.isImmutable);
+ @ assignable \nothing;
+ @ ensures \result.value() == \old(this.value() * pow52(p5, p2));
+ @
+ @ also
+ @
+ @ requires this.value() > 0 && p5 == 0 && p2 > 0 && !this.isImmutable;
+ @ assignable this.nWords, this.data, this.data[*];
+ @ ensures \result == this;
+ @ ensures \result.value() == \old(this.value() * pow52(p5, p2));
+ @*/
+ public FDBigInteger multByPow52(int p5, int p2) {
+ if (this.nWords == 0) {
+ return this;
+ }
+ FDBigInteger res = this;
+ if (p5 != 0) {
+ int[] r;
+ int extraSize = (p2 != 0) ? 1 : 0;
+ if (p5 < SMALL_5_POW.length) {
+ r = new int[this.nWords + 1 + extraSize];
+ mult(this.data, this.nWords, SMALL_5_POW[p5], r);
+ res = new FDBigInteger(r, this.offset);
+ } else {
+ FDBigInteger pow5 = big5pow(p5);
+ r = new int[this.nWords + pow5.size() + extraSize];
+ mult(this.data, this.nWords, pow5.data, pow5.nWords, r);
+ res = new FDBigInteger(r, this.offset + pow5.offset);
+ }
+ }
+ return res.leftShift(p2);
+ }
+
+ /**
+ * Multiplies two big integers represented as int arrays.
+ *
+ * @param s1 The first array factor.
+ * @param s1Len The number of elements of s1 to use.
+ * @param s2 The second array factor.
+ * @param s2Len The number of elements of s2 to use.
+ * @param dst The product array.
+ */
+ /*@
+ @ requires s1 != dst && s2 != dst;
+ @ requires s1.length >= s1Len && s2.length >= s2Len && dst.length >= s1Len + s2Len;
+ @ assignable dst[0 .. s1Len + s2Len - 1];
+ @ ensures AP(dst, s1Len + s2Len) == \old(AP(s1, s1Len) * AP(s2, s2Len));
+ @*/
+ private static void mult(int[] s1, int s1Len, int[] s2, int s2Len, int[] dst) {
+ for (int i = 0; i < s1Len; i++) {
+ long v = s1[i] & LONG_MASK;
+ long p = 0L;
+ for (int j = 0; j < s2Len; j++) {
+ p += (dst[i + j] & LONG_MASK) + v * (s2[j] & LONG_MASK);
+ dst[i + j] = (int) p;
+ p >>>= 32;
+ }
+ dst[i + s2Len] = (int) p;
+ }
+ }
+
+ /**
+ * Subtracts the supplied FDBigInteger subtrahend from this
+ * FDBigInteger. Assert that the result is positive.
+ * If the subtrahend is immutable, store the result in this(minuend).
+ * If this(minuend) is immutable a new FDBigInteger is created.
+ *
+ * @param subtrahend The FDBigInteger to be subtracted.
+ * @return This FDBigInteger less the subtrahend.
+ */
+ /*@
+ @ requires this.isImmutable;
+ @ requires this.value() >= subtrahend.value();
+ @ assignable \nothing;
+ @ ensures \result.value() == \old(this.value() - subtrahend.value());
+ @
+ @ also
+ @
+ @ requires !subtrahend.isImmutable;
+ @ requires this.value() >= subtrahend.value();
+ @ assignable this.nWords, this.offset, this.data, this.data[*];
+ @ ensures \result == this;
+ @ ensures \result.value() == \old(this.value() - subtrahend.value());
+ @*/
+ public FDBigInteger leftInplaceSub(FDBigInteger subtrahend) {
+ assert this.size() >= subtrahend.size() : "result should be positive";
+ FDBigInteger minuend;
+ if (this.isImmutable) {
+ minuend = new FDBigInteger(this.data.clone(), this.offset);
+ } else {
+ minuend = this;
+ }
+ int offsetDiff = subtrahend.offset - minuend.offset;
+ int[] sData = subtrahend.data;
+ int[] mData = minuend.data;
+ int subLen = subtrahend.nWords;
+ int minLen = minuend.nWords;
+ if (offsetDiff < 0) {
+ // need to expand minuend
+ int rLen = minLen - offsetDiff;
+ if (rLen < mData.length) {
+ System.arraycopy(mData, 0, mData, -offsetDiff, minLen);
+ Arrays.fill(mData, 0, -offsetDiff, 0);
+ } else {
+ int[] r = new int[rLen];
+ System.arraycopy(mData, 0, r, -offsetDiff, minLen);
+ minuend.data = mData = r;
+ }
+ minuend.offset = subtrahend.offset;
+ minuend.nWords = minLen = rLen;
+ offsetDiff = 0;
+ }
+ long borrow = 0L;
+ int mIndex = offsetDiff;
+ for (int sIndex = 0; sIndex < subLen && mIndex < minLen; sIndex++, mIndex++) {
+ long diff = (mData[mIndex] & LONG_MASK) - (sData[sIndex] & LONG_MASK) + borrow;
+ mData[mIndex] = (int) diff;
+ borrow = diff >> 32; // signed shift
+ }
+ for (; borrow != 0 && mIndex < minLen; mIndex++) {
+ long diff = (mData[mIndex] & LONG_MASK) + borrow;
+ mData[mIndex] = (int) diff;
+ borrow = diff >> 32; // signed shift
+ }
+ assert borrow == 0L : borrow; // borrow out of subtract,
+ // result should be positive
+ minuend.trimLeadingZeros();
+ return minuend;
+ }
+
+ /**
+ * Subtracts the supplied FDBigInteger subtrahend from this
+ * FDBigInteger. Assert that the result is positive.
+ * If the this(minuend) is immutable, store the result in subtrahend.
+ * If subtrahend is immutable a new FDBigInteger is created.
+ *
+ * @param subtrahend The FDBigInteger to be subtracted.
+ * @return This FDBigInteger less the subtrahend.
+ */
+ /*@
+ @ requires subtrahend.isImmutable;
+ @ requires this.value() >= subtrahend.value();
+ @ assignable \nothing;
+ @ ensures \result.value() == \old(this.value() - subtrahend.value());
+ @
+ @ also
+ @
+ @ requires !subtrahend.isImmutable;
+ @ requires this.value() >= subtrahend.value();
+ @ assignable subtrahend.nWords, subtrahend.offset, subtrahend.data, subtrahend.data[*];
+ @ ensures \result == subtrahend;
+ @ ensures \result.value() == \old(this.value() - subtrahend.value());
+ @*/
+ public FDBigInteger rightInplaceSub(FDBigInteger subtrahend) {
+ assert this.size() >= subtrahend.size() : "result should be positive";
+ FDBigInteger minuend = this;
+ if (subtrahend.isImmutable) {
+ subtrahend = new FDBigInteger(subtrahend.data.clone(), subtrahend.offset);
+ }
+ int offsetDiff = minuend.offset - subtrahend.offset;
+ int[] sData = subtrahend.data;
+ int[] mData = minuend.data;
+ int subLen = subtrahend.nWords;
+ int minLen = minuend.nWords;
+ if (offsetDiff < 0) {
+ int rLen = minLen;
+ if (rLen < sData.length) {
+ System.arraycopy(sData, 0, sData, -offsetDiff, subLen);
+ Arrays.fill(sData, 0, -offsetDiff, 0);
+ } else {
+ int[] r = new int[rLen];
+ System.arraycopy(sData, 0, r, -offsetDiff, subLen);
+ subtrahend.data = sData = r;
+ }
+ subtrahend.offset = minuend.offset;
+ subLen -= offsetDiff;
+ offsetDiff = 0;
+ } else {
+ int rLen = minLen + offsetDiff;
+ if (rLen >= sData.length) {
+ subtrahend.data = sData = Arrays.copyOf(sData, rLen);
+ }
+ }
+ //@ assert minuend == this && minuend.value() == \old(this.value());
+ //@ assert mData == minuend.data && minLen == minuend.nWords;
+ //@ assert subtrahend.offset + subtrahend.data.length >= minuend.size();
+ //@ assert sData == subtrahend.data;
+ //@ assert AP(subtrahend.data, subtrahend.data.length) << subtrahend.offset == \old(subtrahend.value());
+ //@ assert subtrahend.offset == Math.min(\old(this.offset), minuend.offset);
+ //@ assert offsetDiff == minuend.offset - subtrahend.offset;
+ //@ assert 0 <= offsetDiff && offsetDiff + minLen <= sData.length;
+ int sIndex = 0;
+ long borrow = 0L;
+ for (; sIndex < offsetDiff; sIndex++) {
+ long diff = 0L - (sData[sIndex] & LONG_MASK) + borrow;
+ sData[sIndex] = (int) diff;
+ borrow = diff >> 32; // signed shift
+ }
+ //@ assert sIndex == offsetDiff;
+ for (int mIndex = 0; mIndex < minLen; sIndex++, mIndex++) {
+ //@ assert sIndex == offsetDiff + mIndex;
+ long diff = (mData[mIndex] & LONG_MASK) - (sData[sIndex] & LONG_MASK) + borrow;
+ sData[sIndex] = (int) diff;
+ borrow = diff >> 32; // signed shift
+ }
+ assert borrow == 0L : borrow; // borrow out of subtract,
+ // result should be positive
+ subtrahend.nWords = sIndex;
+ subtrahend.trimLeadingZeros();
+ return subtrahend;
+
+ }
+
+ /**
+ * Determines whether all elements of an array are zero for all indices less
+ * than a given index.
+ *
+ * @param a The array to be examined.
+ * @param from The index strictly below which elements are to be examined.
+ * @return Zero if all elements in range are zero, 1 otherwise.
+ */
+ /*@
+ @ requires 0 <= from && from <= a.length;
+ @ ensures \result == (AP(a, from) == 0 ? 0 : 1);
+ @*/
+ private /*@ pure @*/ static int checkZeroTail(int[] a, int from) {
+ while (from > 0) {
+ if (a[--from] != 0) {
+ return 1;
+ }
+ }
+ return 0;
+ }
+
+ /**
+ * Compares the parameter with this FDBigInteger. Returns an
+ * integer accordingly as:
+ *
+ * >0: this > other + * 0: this == other + * <0: this < other + *+ * + * @param other The
FDBigInteger to compare.
+ * @return A negative value, zero, or a positive value according to the
+ * result of the comparison.
+ */
+ /*@
+ @ ensures \result == (this.value() < other.value() ? -1 : this.value() > other.value() ? +1 : 0);
+ @*/
+ public /*@ pure @*/ int cmp(FDBigInteger other) {
+ int aSize = nWords + offset;
+ int bSize = other.nWords + other.offset;
+ if (aSize > bSize) {
+ return 1;
+ } else if (aSize < bSize) {
+ return -1;
+ }
+ int aLen = nWords;
+ int bLen = other.nWords;
+ while (aLen > 0 && bLen > 0) {
+ int a = data[--aLen];
+ int b = other.data[--bLen];
+ if (a != b) {
+ return ((a & LONG_MASK) < (b & LONG_MASK)) ? -1 : 1;
+ }
+ }
+ if (aLen > 0) {
+ return checkZeroTail(data, aLen);
+ }
+ if (bLen > 0) {
+ return -checkZeroTail(other.data, bLen);
+ }
+ return 0;
+ }
+
+ /**
+ * Compares this FDBigInteger with
+ * 5p5 * 2p2.
+ * Returns an integer accordingly as:
+ * + * >0: this > other + * 0: this == other + * <0: this < other + *+ * @param p5 The exponent of the power-of-five factor. + * @param p2 The exponent of the power-of-two factor. + * @return A negative value, zero, or a positive value according to the + * result of the comparison. + */ + /*@ + @ requires p5 >= 0 && p2 >= 0; + @ ensures \result == (this.value() < pow52(p5, p2) ? -1 : this.value() > pow52(p5, p2) ? +1 : 0); + @*/ + public /*@ pure @*/ int cmpPow52(int p5, int p2) { + if (p5 == 0) { + int wordcount = p2 >> 5; + int bitcount = p2 & 0x1f; + int size = this.nWords + this.offset; + if (size > wordcount + 1) { + return 1; + } else if (size < wordcount + 1) { + return -1; + } + int a = this.data[this.nWords -1]; + int b = 1 << bitcount; + if (a != b) { + return ( (a & LONG_MASK) < (b & LONG_MASK)) ? -1 : 1; + } + return checkZeroTail(this.data, this.nWords - 1); + } + return this.cmp(big5pow(p5).leftShift(p2)); + } + + /** + * Compares this
FDBigInteger with x + y. Returns a
+ * value according to the comparison as:
+ * + * -1: this < x + y + * 0: this == x + y + * 1: this > x + y + *+ * @param x The first addend of the sum to compare. + * @param y The second addend of the sum to compare. + * @return -1, 0, or 1 according to the result of the comparison. + */ + /*@ + @ ensures \result == (this.value() < x.value() + y.value() ? -1 : this.value() > x.value() + y.value() ? +1 : 0); + @*/ + public /*@ pure @*/ int addAndCmp(FDBigInteger x, FDBigInteger y) { + FDBigInteger big; + FDBigInteger small; + int xSize = x.size(); + int ySize = y.size(); + int bSize; + int sSize; + if (xSize >= ySize) { + big = x; + small = y; + bSize = xSize; + sSize = ySize; + } else { + big = y; + small = x; + bSize = ySize; + sSize = xSize; + } + int thSize = this.size(); + if (bSize == 0) { + return thSize == 0 ? 0 : 1; + } + if (sSize == 0) { + return this.cmp(big); + } + if (bSize > thSize) { + return -1; + } + if (bSize + 1 < thSize) { + return 1; + } + long top = (big.data[big.nWords - 1] & LONG_MASK); + if (sSize == bSize) { + top += (small.data[small.nWords - 1] & LONG_MASK); + } + if ((top >>> 32) == 0) { + if (((top + 1) >>> 32) == 0) { + // good case - no carry extension + if (bSize < thSize) { + return 1; + } + // here sum.nWords == this.nWords + long v = (this.data[this.nWords - 1] & LONG_MASK); + if (v < top) { + return -1; + } + if (v > top + 1) { + return 1; + } + } + } else { // (top>>>32)!=0 guaranteed carry extension + if (bSize + 1 > thSize) { + return -1; + } + // here sum.nWords == this.nWords + top >>>= 32; + long v = (this.data[this.nWords - 1] & LONG_MASK); + if (v < top) { + return -1; + } + if (v > top + 1) { + return 1; + } + } + return this.cmp(big.add(small)); + } + + /** + * Makes this
FDBigInteger immutable.
+ */
+ /*@
+ @ assignable this.isImmutable;
+ @ ensures this.isImmutable;
+ @*/
+ public void makeImmutable() {
+ this.isImmutable = true;
+ }
+
+ /**
+ * Multiplies this FDBigInteger by an integer.
+ *
+ * @param i The factor by which to multiply this FDBigInteger.
+ * @return This FDBigInteger multiplied by an integer.
+ */
+ /*@
+ @ requires this.value() == 0;
+ @ assignable \nothing;
+ @ ensures \result == this;
+ @
+ @ also
+ @
+ @ requires this.value() != 0;
+ @ assignable \nothing;
+ @ ensures \result.value() == \old(this.value() * UNSIGNED(i));
+ @*/
+ private FDBigInteger mult(int i) {
+ if (this.nWords == 0) {
+ return this;
+ }
+ int[] r = new int[nWords + 1];
+ mult(data, nWords, i, r);
+ return new FDBigInteger(r, offset);
+ }
+
+ /**
+ * Multiplies this FDBigInteger by another FDBigInteger.
+ *
+ * @param other The FDBigInteger factor by which to multiply.
+ * @return The product of this and the parameter FDBigIntegers.
+ */
+ /*@
+ @ requires this.value() == 0;
+ @ assignable \nothing;
+ @ ensures \result == this;
+ @
+ @ also
+ @
+ @ requires this.value() != 0 && other.value() == 0;
+ @ assignable \nothing;
+ @ ensures \result == other;
+ @
+ @ also
+ @
+ @ requires this.value() != 0 && other.value() != 0;
+ @ assignable \nothing;
+ @ ensures \result.value() == \old(this.value() * other.value());
+ @*/
+ private FDBigInteger mult(FDBigInteger other) {
+ if (this.nWords == 0) {
+ return this;
+ }
+ if (this.size() == 1) {
+ return other.mult(data[0]);
+ }
+ if (other.nWords == 0) {
+ return other;
+ }
+ if (other.size() == 1) {
+ return this.mult(other.data[0]);
+ }
+ int[] r = new int[nWords + other.nWords];
+ mult(this.data, this.nWords, other.data, other.nWords, r);
+ return new FDBigInteger(r, this.offset + other.offset);
+ }
+
+ /**
+ * Adds another FDBigInteger to this FDBigInteger.
+ *
+ * @param other The FDBigInteger to add.
+ * @return The sum of the FDBigIntegers.
+ */
+ /*@
+ @ assignable \nothing;
+ @ ensures \result.value() == \old(this.value() + other.value());
+ @*/
+ private FDBigInteger add(FDBigInteger other) {
+ FDBigInteger big, small;
+ int bigLen, smallLen;
+ int tSize = this.size();
+ int oSize = other.size();
+ if (tSize >= oSize) {
+ big = this;
+ bigLen = tSize;
+ small = other;
+ smallLen = oSize;
+ } else {
+ big = other;
+ bigLen = oSize;
+ small = this;
+ smallLen = tSize;
+ }
+ int[] r = new int[bigLen + 1];
+ int i = 0;
+ long carry = 0L;
+ for (; i < smallLen; i++) {
+ carry += (i < big.offset ? 0L : (big.data[i - big.offset] & LONG_MASK) )
+ + ((i < small.offset ? 0L : (small.data[i - small.offset] & LONG_MASK)));
+ r[i] = (int) carry;
+ carry >>= 32; // signed shift.
+ }
+ for (; i < bigLen; i++) {
+ carry += (i < big.offset ? 0L : (big.data[i - big.offset] & LONG_MASK) );
+ r[i] = (int) carry;
+ carry >>= 32; // signed shift.
+ }
+ r[bigLen] = (int) carry;
+ return new FDBigInteger(r, 0);
+ }
+
+
+ /**
+ * Multiplies a FDBigInteger by an int and adds another int. The
+ * result is computed in place. This method is intended only to be invoked
+ * from
+ *
+ * FDBigInteger(long lValue, char[] digits, int kDigits, int nDigits)
+ * .
+ *
+ * @param iv The factor by which to multiply this FDBigInteger.
+ * @param addend The value to add to the product of this
+ * FDBigInteger and iv.
+ */
+ /*@
+ @ requires this.value()*UNSIGNED(iv) + UNSIGNED(addend) < ((\bigint)1) << ((this.data.length + this.offset)*32);
+ @ assignable this.data[*];
+ @ ensures this.value() == \old(this.value()*UNSIGNED(iv) + UNSIGNED(addend));
+ @*/
+ private /*@ helper @*/ void multAddMe(int iv, int addend) {
+ long v = iv & LONG_MASK;
+ // unroll 0th iteration, doing addition.
+ long p = v * (data[0] & LONG_MASK) + (addend & LONG_MASK);
+ data[0] = (int) p;
+ p >>>= 32;
+ for (int i = 1; i < nWords; i++) {
+ p += v * (data[i] & LONG_MASK);
+ data[i] = (int) p;
+ p >>>= 32;
+ }
+ if (p != 0L) {
+ data[nWords++] = (int) p; // will fail noisily if illegal!
+ }
+ }
+
+ //
+ // original doc:
+ //
+ // do this -=q*S
+ // returns borrow
+ //
+ /**
+ * Multiplies the parameters and subtracts them from this
+ * FDBigInteger.
+ *
+ * @param q The integer parameter.
+ * @param S The FDBigInteger parameter.
+ * @return this - q*S.
+ */
+ /*@
+ @ ensures nWords == 0 ==> offset == 0;
+ @ ensures nWords > 0 ==> data[nWords - 1] != 0;
+ @*/
+ /*@
+ @ requires 0 < q && q <= (1L << 31);
+ @ requires data != null;
+ @ requires 0 <= nWords && nWords <= data.length && offset >= 0;
+ @ requires !this.isImmutable;
+ @ requires this.size() == S.size();
+ @ requires this != S;
+ @ assignable this.nWords, this.offset, this.data, this.data[*];
+ @ ensures -q <= \result && \result <= 0;
+ @ ensures this.size() == \old(this.size());
+ @ ensures this.value() + (\result << (this.size()*32)) == \old(this.value() - q*S.value());
+ @ ensures this.offset == \old(Math.min(this.offset, S.offset));
+ @ ensures \old(this.offset <= S.offset) ==> this.nWords == \old(this.nWords);
+ @ ensures \old(this.offset <= S.offset) ==> this.offset == \old(this.offset);
+ @ ensures \old(this.offset <= S.offset) ==> this.data == \old(this.data);
+ @
+ @ also
+ @
+ @ requires q == 0;
+ @ assignable \nothing;
+ @ ensures \result == 0;
+ @*/
+ private /*@ helper @*/ long multDiffMe(long q, FDBigInteger S) {
+ long diff = 0L;
+ if (q != 0) {
+ int deltaSize = S.offset - this.offset;
+ if (deltaSize >= 0) {
+ int[] sd = S.data;
+ int[] td = this.data;
+ for (int sIndex = 0, tIndex = deltaSize; sIndex < S.nWords; sIndex++, tIndex++) {
+ diff += (td[tIndex] & LONG_MASK) - q * (sd[sIndex] & LONG_MASK);
+ td[tIndex] = (int) diff;
+ diff >>= 32; // N.B. SIGNED shift.
+ }
+ } else {
+ deltaSize = -deltaSize;
+ int[] rd = new int[nWords + deltaSize];
+ int sIndex = 0;
+ int rIndex = 0;
+ int[] sd = S.data;
+ for (; rIndex < deltaSize && sIndex < S.nWords; sIndex++, rIndex++) {
+ diff -= q * (sd[sIndex] & LONG_MASK);
+ rd[rIndex] = (int) diff;
+ diff >>= 32; // N.B. SIGNED shift.
+ }
+ int tIndex = 0;
+ int[] td = this.data;
+ for (; sIndex < S.nWords; sIndex++, tIndex++, rIndex++) {
+ diff += (td[tIndex] & LONG_MASK) - q * (sd[sIndex] & LONG_MASK);
+ rd[rIndex] = (int) diff;
+ diff >>= 32; // N.B. SIGNED shift.
+ }
+ this.nWords += deltaSize;
+ this.offset -= deltaSize;
+ this.data = rd;
+ }
+ }
+ return diff;
+ }
+
+
+ /**
+ * Multiplies by 10 a big integer represented as an array. The final carry
+ * is returned.
+ *
+ * @param src The array representation of the big integer.
+ * @param srcLen The number of elements of src to use.
+ * @param dst The product array.
+ * @return The final carry of the multiplication.
+ */
+ /*@
+ @ requires src.length >= srcLen && dst.length >= srcLen;
+ @ assignable dst[0 .. srcLen - 1];
+ @ ensures 0 <= \result && \result < 10;
+ @ ensures AP(dst, srcLen) + (\result << (srcLen*32)) == \old(AP(src, srcLen) * 10);
+ @*/
+ private static int multAndCarryBy10(int[] src, int srcLen, int[] dst) {
+ long carry = 0;
+ for (int i = 0; i < srcLen; i++) {
+ long product = (src[i] & LONG_MASK) * 10L + carry;
+ dst[i] = (int) product;
+ carry = product >>> 32;
+ }
+ return (int) carry;
+ }
+
+ /**
+ * Multiplies by a constant value a big integer represented as an array.
+ * The constant factor is an int.
+ *
+ * @param src The array representation of the big integer.
+ * @param srcLen The number of elements of src to use.
+ * @param value The constant factor by which to multiply.
+ * @param dst The product array.
+ */
+ /*@
+ @ requires src.length >= srcLen && dst.length >= srcLen + 1;
+ @ assignable dst[0 .. srcLen];
+ @ ensures AP(dst, srcLen + 1) == \old(AP(src, srcLen) * UNSIGNED(value));
+ @*/
+ private static void mult(int[] src, int srcLen, int value, int[] dst) {
+ long val = value & LONG_MASK;
+ long carry = 0;
+ for (int i = 0; i < srcLen; i++) {
+ long product = (src[i] & LONG_MASK) * val + carry;
+ dst[i] = (int) product;
+ carry = product >>> 32;
+ }
+ dst[srcLen] = (int) carry;
+ }
+
+ /**
+ * Multiplies by a constant value a big integer represented as an array.
+ * The constant factor is a long represent as two ints.
+ *
+ * @param src The array representation of the big integer.
+ * @param srcLen The number of elements of src to use.
+ * @param v0 The lower 32 bits of the long factor.
+ * @param v1 The upper 32 bits of the long factor.
+ * @param dst The product array.
+ */
+ /*@
+ @ requires src != dst;
+ @ requires src.length >= srcLen && dst.length >= srcLen + 2;
+ @ assignable dst[0 .. srcLen + 1];
+ @ ensures AP(dst, srcLen + 2) == \old(AP(src, srcLen) * (UNSIGNED(v0) + (UNSIGNED(v1) << 32)));
+ @*/
+ private static void mult(int[] src, int srcLen, int v0, int v1, int[] dst) {
+ long v = v0 & LONG_MASK;
+ long carry = 0;
+ for (int j = 0; j < srcLen; j++) {
+ long product = v * (src[j] & LONG_MASK) + carry;
+ dst[j] = (int) product;
+ carry = product >>> 32;
+ }
+ dst[srcLen] = (int) carry;
+ v = v1 & LONG_MASK;
+ carry = 0;
+ for (int j = 0; j < srcLen; j++) {
+ long product = (dst[j + 1] & LONG_MASK) + v * (src[j] & LONG_MASK) + carry;
+ dst[j + 1] = (int) product;
+ carry = product >>> 32;
+ }
+ dst[srcLen + 1] = (int) carry;
+ }
+
+ // Fails assertion for negative exponent.
+ /**
+ * Computes 5 raised to a given power.
+ *
+ * @param p The exponent of 5.
+ * @return 5p.
+ */
+ private static FDBigInteger big5pow(int p) {
+ assert p >= 0 : p; // negative power of 5
+ if (p < MAX_FIVE_POW) {
+ return POW_5_CACHE[p];
+ }
+ return big5powRec(p);
+ }
+
+ // slow path
+ /**
+ * Computes 5 raised to a given power.
+ *
+ * @param p The exponent of 5.
+ * @return 5p.
+ */
+ private static FDBigInteger big5powRec(int p) {
+ if (p < MAX_FIVE_POW) {
+ return POW_5_CACHE[p];
+ }
+ // construct the value.
+ // recursively.
+ int q, r;
+ // in order to compute 5^p,
+ // compute its square root, 5^(p/2) and square.
+ // or, let q = p / 2, r = p -q, then
+ // 5^p = 5^(q+r) = 5^q * 5^r
+ q = p >> 1;
+ r = p - q;
+ FDBigInteger bigq = big5powRec(q);
+ if (r < SMALL_5_POW.length) {
+ return bigq.mult(SMALL_5_POW[r]);
+ } else {
+ return bigq.mult(big5powRec(r));
+ }
+ }
+
+ // for debugging ...
+ /**
+ * Converts this FDBigInteger to a hexadecimal string.
+ *
+ * @return The hexadecimal string representation.
+ */
+ public String toHexString(){
+ if(nWords ==0) {
+ return "0";
+ }
+ StringBuilder sb = new StringBuilder((nWords +offset)*8);
+ for(int i= nWords -1; i>=0; i--) {
+ String subStr = Integer.toHexString(data[i]);
+ for(int j = subStr.length(); j<8; j++) {
+ sb.append('0');
+ }
+ sb.append(subStr);
+ }
+ for(int i=offset; i>0; i--) {
+ sb.append("00000000");
+ }
+ return sb.toString();
+ }
+
+ // for debugging ...
+ /**
+ * Converts this FDBigInteger to a BigInteger.
+ *
+ * @return The BigInteger representation.
+ */
+ public BigInteger toBigInteger() {
+ byte[] magnitude = new byte[nWords * 4 + 1];
+ for (int i = 0; i < nWords; i++) {
+ int w = data[i];
+ magnitude[magnitude.length - 4 * i - 1] = (byte) w;
+ magnitude[magnitude.length - 4 * i - 2] = (byte) (w >> 8);
+ magnitude[magnitude.length - 4 * i - 3] = (byte) (w >> 16);
+ magnitude[magnitude.length - 4 * i - 4] = (byte) (w >> 24);
+ }
+ return new BigInteger(magnitude).shiftLeft(offset * 32);
+ }
+
+ // for debugging ...
+ /**
+ * Converts this FDBigInteger to a string.
+ *
+ * @return The string representation.
+ */
+ @Override
+ public String toString(){
+ return toBigInteger().toString();
+ }
+}
\ No newline at end of file
diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/FloatingDecimal.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/FloatingDecimal.java
new file mode 100644
index 0000000..4334f28
--- /dev/null
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/FloatingDecimal.java
@@ -0,0 +1,616 @@
+package org.apache.hadoop.hive.serde2.lazy.fast;
+
+import sun.misc.DoubleConsts;
+
+public class FloatingDecimal{
+ //
+ // Constants of the implementation;
+ // most are IEEE-754 related.
+ // (There are more really boring constants at the end.)
+ //
+ static final int EXP_SHIFT = DoubleConsts.SIGNIFICAND_WIDTH - 1;
+ static final long FRACT_HOB = ( 1L<String to convert.
+ * @return The double precision value.
+ * @throws NumberFormatException If the String does not
+ * represent a properly formatted double precision value.
+ */
+ public double parseDouble(ByteString in) throws NumberFormatException {
+ boolean isNegative = false;
+ boolean signSeen = false;
+ int decExp = 0;
+ byte c;
+ boolean done = false;
+ int nDigits= 0;
+ boolean decSeen = false;
+ int decPt = 0;
+ int nLeadZero = 0;
+ int nTrailZero= 0;
+
+ parseNumber:
+ try{
+ in = in.trim(); // don't fool around with white space.
+ // throws NullPointerException if null
+ int len = in.length();
+ if ( len == 0 ) {
+ throw new NumberFormatException("empty String");
+ }
+ int i = 0;
+ switch (in.charAt(i)){
+ case '-':
+ isNegative = true;
+ //FALLTHROUGH
+ case '+':
+ i++;
+ signSeen = true;
+ }
+ c = in.charAt(i);
+ if(c == 'N') { // Check for NaN
+ if((len-i)==NAN_LENGTH && in.equals(NAN_REP)) {
+ return Double.NaN;
+ }
+ // something went wrong, throw exception
+ break parseNumber;
+ } else if(c == 'I') { // Check for Infinity strings
+ if((len-i)==INFINITY_LENGTH && in.equals(INFINITY_REP)) {
+ return isNegative? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
+ }
+ // something went wrong, throw exception
+ break parseNumber;
+ } else if (c == '0') { // check for hexadecimal floating-point number
+ if (len > i+1 ) {
+ byte ch = in.charAt(i+1);
+ if (ch == 'x' || ch == 'X' ) { // possible hex string
+ throw new IllegalArgumentException();
+ }
+ }
+ } // look for and process decimal floating-point string
+
+ digits = new byte[ len ];
+
+ skipLeadingZerosLoop:
+ while (i < len) {
+ c = in.charAt(i);
+ if (c == '0') {
+ nLeadZero++;
+ } else if (c == '.') {
+ if (decSeen) {
+ // already saw one ., this is the 2nd.
+ throw new NumberFormatException("multiple points");
+ }
+ decPt = i;
+ if (signSeen) {
+ decPt -= 1;
+ }
+ decSeen = true;
+ } else {
+ break skipLeadingZerosLoop;
+ }
+ i++;
+ }
+ digitLoop:
+ while (i < len) {
+ c = in.charAt(i);
+ if (c >= '1' && c <= '9') {
+ digits[nDigits++] = c;
+ nTrailZero = 0;
+ } else if (c == '0') {
+ digits[nDigits++] = c;
+ nTrailZero++;
+ } else if (c == '.') {
+ if (decSeen) {
+ // already saw one ., this is the 2nd.
+ throw new NumberFormatException("multiple points");
+ }
+ decPt = i;
+ if (signSeen) {
+ decPt -= 1;
+ }
+ decSeen = true;
+ } else {
+ break digitLoop;
+ }
+ i++;
+ }
+ nDigits -=nTrailZero;
+ //
+ // At this point, we've scanned all the digits and decimal
+ // point we're going to see. Trim off leading and trailing
+ // zeros, which will just confuse us later, and adjust
+ // our initial decimal exponent accordingly.
+ // To review:
+ // we have seen i total characters.
+ // nLeadZero of them were zeros before any other digits.
+ // nTrailZero of them were zeros after any other digits.
+ // if ( decSeen ), then a . was seen after decPt characters
+ // ( including leading zeros which have been discarded )
+ // nDigits characters were neither lead nor trailing
+ // zeros, nor point
+ //
+ //
+ // special hack: if we saw no non-zero digits, then the
+ // answer is zero!
+ // Unfortunately, we feel honor-bound to keep parsing!
+ //
+ boolean isZero = (nDigits == 0);
+ if ( isZero && nLeadZero == 0 ){
+ // we saw NO DIGITS AT ALL,
+ // not even a crummy 0!
+ // this is not allowed.
+ break parseNumber; // go throw exception
+ }
+ //
+ // Our initial exponent is decPt, adjusted by the number of
+ // discarded zeros. Or, if there was no decPt,
+ // then its just nDigits adjusted by discarded trailing zeros.
+ //
+ if ( decSeen ){
+ decExp = decPt - nLeadZero;
+ } else {
+ decExp = nDigits + nTrailZero;
+ }
+
+ //
+ // Look for 'e' or 'E' and an optionally signed integer.
+ //
+ if ( (i < len) && (((c = in.charAt(i) )=='e') || (c == 'E') ) ){
+ int expSign = 1;
+ int expVal = 0;
+ int reallyBig = Integer.MAX_VALUE / 10;
+ boolean expOverflow = false;
+ switch( in.charAt(++i) ){
+ case '-':
+ expSign = -1;
+ //FALLTHROUGH
+ case '+':
+ i++;
+ }
+ int expAt = i;
+ expLoop:
+ while ( i < len ){
+ if ( expVal >= reallyBig ){
+ // the next character will cause integer
+ // overflow.
+ expOverflow = true;
+ }
+ c = in.charAt(i++);
+ if(c>='0' && c<='9') {
+ expVal = expVal*10 + ( (int)c - (int)'0' );
+ } else {
+ i--; // back up.
+ break expLoop; // stop parsing exponent.
+ }
+ }
+ int expLimit = BIG_DECIMAL_EXPONENT+nDigits+nTrailZero;
+ if ( expOverflow || ( expVal > expLimit ) ){
+ //
+ // The intent here is to end up with
+ // infinity or zero, as appropriate.
+ // The reason for yielding such a small decExponent,
+ // rather than something intuitive such as
+ // expSign*Integer.MAX_VALUE, is that this value
+ // is subject to further manipulation in
+ // doubleValue() and floatValue(), and I don't want
+ // it to be able to cause overflow there!
+ // (The only way we can get into trouble here is for
+ // really outrageous nDigits+nTrailZero, such as 2 billion. )
+ //
+ decExp = expSign*expLimit;
+ } else {
+ // this should not overflow, since we tested
+ // for expVal > (MAX+N), where N >= abs(decExp)
+ decExp = decExp + expSign*expVal;
+ }
+
+ // if we saw something not a digit ( or end of string )
+ // after the [Ee][+-], without seeing any digits at all
+ // this is certainly an error. If we saw some digits,
+ // but then some trailing garbage, that might be ok.
+ // so we just fall through in that case.
+ // HUMBUG
+ if ( i == expAt ) {
+ break parseNumber; // certainly bad
+ }
+ }
+ //
+ // We parsed everything we could.
+ // If there are leftovers, then this is not good input!
+ //
+ if ( i < len &&
+ ((i != len - 1) ||
+ (in.charAt(i) != 'f' &&
+ in.charAt(i) != 'F' &&
+ in.charAt(i) != 'd' &&
+ in.charAt(i) != 'D'))) {
+ break parseNumber; // go throw exception
+ }
+ if(isZero) {
+ return isNegative ? -0.0d : 0.0d;
+ }
+ done = true;
+// return new ASCIIToBinaryBuffer(isNegative, decExp, digits, nDigits);
+ } catch ( StringIndexOutOfBoundsException e ) {
+ ;
+ }
+ if (!done) {
+ throw new NumberFormatException("For input string: \"" + in + "\"");
+ }
+
+ //
+ // doubleValue()
+ //
+
+ int kDigits = Math.min(nDigits, MAX_DECIMAL_DIGITS + 1);
+ //
+ // convert the lead kDigits to a long integer.
+ //
+ // (special performance hack: start to do it using int)
+ int iValue = (int) digits[0] - (int) '0';
+ int iDigits = Math.min(kDigits, INT_DECIMAL_DIGITS);
+ for (int i = 1; i < iDigits; i++) {
+ iValue = iValue * 10 + (int) digits[i] - (int) '0';
+ }
+ long lValue = (long) iValue;
+ for (int i = iDigits; i < kDigits; i++) {
+ lValue = lValue * 10L + (long) ((int) digits[i] - (int) '0');
+ }
+ double dValue = (double) lValue;
+ int exp = decExp - kDigits;
+ //
+ // lValue now contains a long integer with the value of
+ // the first kDigits digits of the number.
+ // dValue contains the (double) of the same.
+ //
+
+ if (nDigits <= MAX_DECIMAL_DIGITS) {
+ //
+ // possibly an easy case.
+ // We know that the digits can be represented
+ // exactly. And if the exponent isn't too outrageous,
+ // the whole thing can be done with one operation,
+ // thus one rounding error.
+ // Note that all our constructors trim all leading and
+ // trailing zeros, so simple values (including zero)
+ // will always end up here
+ //
+ if (exp == 0 || dValue == 0.0) {
+ return (isNegative) ? -dValue : dValue; // small floating integer
+ }
+ else if (exp >= 0) {
+ if (exp <= MAX_SMALL_TEN) {
+ //
+ // Can get the answer with one operation,
+ // thus one roundoff.
+ //
+ double rValue = dValue * SMALL_10_POW[exp];
+ return (isNegative) ? -rValue : rValue;
+ }
+ int slop = MAX_DECIMAL_DIGITS - kDigits;
+ if (exp <= MAX_SMALL_TEN + slop) {
+ //
+ // We can multiply dValue by 10^(slop)
+ // and it is still "small" and exact.
+ // Then we can multiply by 10^(exp-slop)
+ // with one rounding.
+ //
+ dValue *= SMALL_10_POW[slop];
+ double rValue = dValue * SMALL_10_POW[exp - slop];
+ return (isNegative) ? -rValue : rValue;
+ }
+ //
+ // Else we have a hard case with a positive exp.
+ //
+ } else {
+ if (exp >= -MAX_SMALL_TEN) {
+ //
+ // Can get the answer in one division.
+ //
+ double rValue = dValue / SMALL_10_POW[-exp];
+ return (isNegative) ? -rValue : rValue;
+ }
+ //
+ // Else we have a hard case with a negative exp.
+ //
+ }
+ }
+
+ //
+ // Harder cases:
+ // The sum of digits plus exponent is greater than
+ // what we think we can do with one error.
+ //
+ // Start by approximating the right answer by,
+ // naively, scaling by powers of 10.
+ //
+ if (exp > 0) {
+ if (decExp > MAX_DECIMAL_EXPONENT + 1) {
+ //
+ // Lets face it. This is going to be
+ // Infinity. Cut to the chase.
+ //
+ return (isNegative) ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
+ }
+ if ((exp & 15) != 0) {
+ dValue *= SMALL_10_POW[exp & 15];
+ }
+ if ((exp >>= 4) != 0) {
+ int j;
+ for (j = 0; exp > 1; j++, exp >>= 1) {
+ if ((exp & 1) != 0) {
+ dValue *= BIG_10_POW[j];
+ }
+ }
+ //
+ // The reason for the weird exp > 1 condition
+ // in the above loop was so that the last multiply
+ // would get unrolled. We handle it here.
+ // It could overflow.
+ //
+ double t = dValue * BIG_10_POW[j];
+ if (Double.isInfinite(t)) {
+ //
+ // It did overflow.
+ // Look more closely at the result.
+ // If the exponent is just one too large,
+ // then use the maximum finite as our estimate
+ // value. Else call the result infinity
+ // and punt it.
+ // ( I presume this could happen because
+ // rounding forces the result here to be
+ // an ULP or two larger than
+ // Double.MAX_VALUE ).
+ //
+ t = dValue / 2.0;
+ t *= BIG_10_POW[j];
+ if (Double.isInfinite(t)) {
+ return (isNegative) ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
+ }
+ t = Double.MAX_VALUE;
+ }
+ dValue = t;
+ }
+ } else if (exp < 0) {
+ exp = -exp;
+ if (decExp < MIN_DECIMAL_EXPONENT - 1) {
+ //
+ // Lets face it. This is going to be
+ // zero. Cut to the chase.
+ //
+ return (isNegative) ? -0.0 : 0.0;
+ }
+ if ((exp & 15) != 0) {
+ dValue /= SMALL_10_POW[exp & 15];
+ }
+ if ((exp >>= 4) != 0) {
+ int j;
+ for (j = 0; exp > 1; j++, exp >>= 1) {
+ if ((exp & 1) != 0) {
+ dValue *= TINY_10_POW[j];
+ }
+ }
+ //
+ // The reason for the weird exp > 1 condition
+ // in the above loop was so that the last multiply
+ // would get unrolled. We handle it here.
+ // It could underflow.
+ //
+ double t = dValue * TINY_10_POW[j];
+ if (t == 0.0) {
+ //
+ // It did underflow.
+ // Look more closely at the result.
+ // If the exponent is just one too small,
+ // then use the minimum finite as our estimate
+ // value. Else call the result 0.0
+ // and punt it.
+ // ( I presume this could happen because
+ // rounding forces the result here to be
+ // an ULP or two less than
+ // Double.MIN_VALUE ).
+ //
+ t = dValue * 2.0;
+ t *= TINY_10_POW[j];
+ if (t == 0.0) {
+ return (isNegative) ? -0.0 : 0.0;
+ }
+ t = Double.MIN_VALUE;
+ }
+ dValue = t;
+ }
+ }
+
+ //
+ // dValue is now approximately the result.
+ // The hard part is adjusting it, by comparison
+ // with FDBigInteger arithmetic.
+ // Formulate the EXACT big-number result as
+ // bigD0 * 10^exp
+ //
+ if (nDigits > MAX_NDIGITS) {
+ nDigits = MAX_NDIGITS + 1;
+ digits[MAX_NDIGITS] = '1';
+ }
+ FDBigInteger bigD0 = new FDBigInteger(lValue, digits, kDigits, nDigits);
+ exp = decExp - nDigits;
+
+ long ieeeBits = Double.doubleToRawLongBits(dValue); // IEEE-754 bits of double candidate
+ final int B5 = Math.max(0, -exp); // powers of 5 in bigB, value is not modified inside correctionLoop
+ final int D5 = Math.max(0, exp); // powers of 5 in bigD, value is not modified inside correctionLoop
+ bigD0 = bigD0.multByPow52(D5, 0);
+ bigD0.makeImmutable(); // prevent bigD0 modification inside correctionLoop
+ FDBigInteger bigD = null;
+ int prevD2 = 0;
+
+ correctionLoop:
+ while (true) {
+ // here ieeeBits can't be NaN, Infinity or zero
+ int binexp = (int) (ieeeBits >>> EXP_SHIFT);
+ long bigBbits = ieeeBits & DoubleConsts.SIGNIF_BIT_MASK;
+ if (binexp > 0) {
+ bigBbits |= FRACT_HOB;
+ } else { // Normalize denormalized numbers.
+ assert bigBbits != 0L : bigBbits; // doubleToBigInt(0.0)
+ int leadingZeros = Long.numberOfLeadingZeros(bigBbits);
+ int shift = leadingZeros - (63 - EXP_SHIFT);
+ bigBbits <<= shift;
+ binexp = 1 - shift;
+ }
+ binexp -= DoubleConsts.EXP_BIAS;
+ int lowOrderZeros = Long.numberOfTrailingZeros(bigBbits);
+ bigBbits >>>= lowOrderZeros;
+ final int bigIntExp = binexp - EXP_SHIFT + lowOrderZeros;
+ final int bigIntNBits = EXP_SHIFT + 1 - lowOrderZeros;
+
+ //
+ // Scale bigD, bigB appropriately for
+ // big-integer operations.
+ // Naively, we multiply by powers of ten
+ // and powers of two. What we actually do
+ // is keep track of the powers of 5 and
+ // powers of 2 we would use, then factor out
+ // common divisors before doing the work.
+ //
+ int B2 = B5; // powers of 2 in bigB
+ int D2 = D5; // powers of 2 in bigD
+ int Ulp2; // powers of 2 in halfUlp.
+ if (bigIntExp >= 0) {
+ B2 += bigIntExp;
+ } else {
+ D2 -= bigIntExp;
+ }
+ Ulp2 = B2;
+ // shift bigB and bigD left by a number s. t.
+ // halfUlp is still an integer.
+ int hulpbias;
+ if (binexp <= -DoubleConsts.EXP_BIAS) {
+ // This is going to be a denormalized number
+ // (if not actually zero).
+ // half an ULP is at 2^-(DoubleConsts.EXP_BIAS+EXP_SHIFT+1)
+ hulpbias = binexp + lowOrderZeros + DoubleConsts.EXP_BIAS;
+ } else {
+ hulpbias = 1 + lowOrderZeros;
+ }
+ B2 += hulpbias;
+ D2 += hulpbias;
+ // if there are common factors of 2, we might just as well
+ // factor them out, as they add nothing useful.
+ int common2 = Math.min(B2, Math.min(D2, Ulp2));
+ B2 -= common2;
+ D2 -= common2;
+ Ulp2 -= common2;
+ // do multiplications by powers of 5 and 2
+ FDBigInteger bigB = FDBigInteger.valueOfMulPow52(bigBbits, B5, B2);
+ if (bigD == null || prevD2 != D2) {
+ bigD = bigD0.leftShift(D2);
+ prevD2 = D2;
+ }
+ //
+ // to recap:
+ // bigB is the scaled-big-int version of our floating-point
+ // candidate.
+ // bigD is the scaled-big-int version of the exact value
+ // as we understand it.
+ // halfUlp is 1/2 an ulp of bigB, except for special cases
+ // of exact powers of 2
+ //
+ // the plan is to compare bigB with bigD, and if the difference
+ // is less than halfUlp, then we're satisfied. Otherwise,
+ // use the ratio of difference to halfUlp to calculate a fudge
+ // factor to add to the floating value, then go 'round again.
+ //
+ FDBigInteger diff;
+ int cmpResult;
+ boolean overvalue;
+ if ((cmpResult = bigB.cmp(bigD)) > 0) {
+ overvalue = true; // our candidate is too big.
+ diff = bigB.leftInplaceSub(bigD); // bigB is not user further - reuse
+ if ((bigIntNBits == 1) && (bigIntExp > -DoubleConsts.EXP_BIAS + 1)) {
+ // candidate is a normalized exact power of 2 and
+ // is too big (larger than Double.MIN_NORMAL). We will be subtracting.
+ // For our purposes, ulp is the ulp of the
+ // next smaller range.
+ Ulp2 -= 1;
+ if (Ulp2 < 0) {
+ // rats. Cannot de-scale ulp this far.
+ // must scale diff in other direction.
+ Ulp2 = 0;
+ diff = diff.leftShift(1);
+ }
+ }
+ } else if (cmpResult < 0) {
+ overvalue = false; // our candidate is too small.
+ diff = bigD.rightInplaceSub(bigB); // bigB is not user further - reuse
+ } else {
+ // the candidate is exactly right!
+ // this happens with surprising frequency
+ break correctionLoop;
+ }
+ cmpResult = diff.cmpPow52(B5, Ulp2);
+ if ((cmpResult) < 0) {
+ // difference is small.
+ // this is close enough
+ break correctionLoop;
+ } else if (cmpResult == 0) {
+ // difference is exactly half an ULP
+ // round to some other value maybe, then finish
+ if ((ieeeBits & 1) != 0) { // half ties to even
+ ieeeBits += overvalue ? -1 : 1; // nextDown or nextUp
+ }
+ break correctionLoop;
+ } else {
+ // difference is non-trivial.
+ // could scale addend by ratio of difference to
+ // halfUlp here, if we bothered to compute that difference.
+ // Most of the time ( I hope ) it is about 1 anyway.
+ ieeeBits += overvalue ? -1 : 1; // nextDown or nextUp
+ if (ieeeBits == 0 || ieeeBits == DoubleConsts.EXP_BIT_MASK) { // 0.0 or Double.POSITIVE_INFINITY
+ break correctionLoop; // oops. Fell off end of range.
+ }
+ continue; // try again.
+ }
+
+ }
+ if (isNegative) {
+ ieeeBits |= DoubleConsts.SIGN_BIT_MASK;
+ }
+ return Double.longBitsToDouble(ieeeBits);
+ }
+}
\ No newline at end of file
diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
index 264335c..bd3d72e 100644
--- serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
+++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java
@@ -93,6 +93,8 @@
private boolean isEndOfInputReached;
+ private FloatingDecimal floatingDecimal = new FloatingDecimal();
+
public LazySimpleDeserializeRead(TypeInfo[] typeInfos, boolean useExternalBuffer,
byte separator, LazySerDeParameters lazyParams) {
super(typeInfos, useExternalBuffer);
@@ -447,8 +449,7 @@ public boolean readField(int fieldIndex) throws IOException {
return false;
}
currentDouble =
- Double.parseDouble(
- new String(bytes, fieldStart, fieldLength, StandardCharsets.UTF_8));
+ floatingDecimal.parseDouble(new ByteString(bytes, fieldStart, fieldLength));
return true;
case STRING:
case CHAR: