diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToLong.java ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToLong.java new file mode 100644 index 0000000..bde28e8 --- /dev/null +++ ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringToLong.java @@ -0,0 +1,270 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.vector.expressions; + +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; +import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; +import org.apache.hadoop.hive.serde2.lazy.LazyByte; +import org.apache.hadoop.hive.serde2.lazy.LazyInteger; +import org.apache.hadoop.hive.serde2.lazy.LazyLong; +import org.apache.hadoop.hive.serde2.lazy.LazyShort; +import org.apache.hadoop.hive.serde2.lazy.LazyUtils; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; +import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; + +/** + * Cast a string to a long. + * + * If other functions besides cast need to take a string in and produce a long, + * you can subclass this class or convert it to a superclass, and + * implement different "func()" methods for each operation. + */ +public class CastStringToLong extends VectorExpression { + private static final long serialVersionUID = 1L; + int inputColumn; + int outputColumn; + + private transient boolean integerPrimitiveCategoryKnown = false; + protected transient PrimitiveCategory integerPrimitiveCategory; + + public CastStringToLong(int inputColumn, int outputColumn) { + this.inputColumn = inputColumn; + this.outputColumn = outputColumn; + } + + public CastStringToLong() { + super(); + } + + /** + * Convert input string to a long, at position i in the respective vectors. + */ + protected void func(LongColumnVector outV, BytesColumnVector inV, int batchIndex) { + + byte[] bytes = inV.vector[batchIndex]; + final int start = inV.start[batchIndex]; + final int length = inV.length[batchIndex]; + try { + + switch (integerPrimitiveCategory) { + case BOOLEAN: + { + boolean booleanValue; + int i = start; + if (length == 4) { + if ((bytes[i] == 'T' || bytes[i] == 't') && + (bytes[i + 1] == 'R' || bytes[i + 1] == 'r') && + (bytes[i + 2] == 'U' || bytes[i + 2] == 'u') && + (bytes[i + 3] == 'E' || bytes[i + 3] == 'e')) { + booleanValue = true; + } else { + // No boolean value match for 4 char field. + outV.noNulls = false; + outV.isNull[batchIndex] = true; + return; + } + } else if (length == 5) { + if ((bytes[i] == 'F' || bytes[i] == 'f') && + (bytes[i + 1] == 'A' || bytes[i + 1] == 'a') && + (bytes[i + 2] == 'L' || bytes[i + 2] == 'l') && + (bytes[i + 3] == 'S' || bytes[i + 3] == 's') && + (bytes[i + 4] == 'E' || bytes[i + 4] == 'e')) { + booleanValue = false; + } else { + // No boolean value match for 5 char field. + outV.noNulls = false; + outV.isNull[batchIndex] = true; + return; + } + } else if (length == 1) { + byte b = bytes[start]; + if (b == '1' || b == 't' || b == 'T') { + booleanValue = true; + } else if (b == '0' || b == 'f' || b == 'F') { + booleanValue = false; + } else { + // No boolean value match for extended 1 char field. + outV.noNulls = false; + outV.isNull[batchIndex] = true; + return; + } + } else { + // No boolean value match for other lengths. + outV.noNulls = false; + outV.isNull[batchIndex] = true; + return; + } + outV.vector[batchIndex] = (booleanValue ? 1 : 0); + } + break; + case BYTE: + if (!LazyUtils.isNumberMaybe(bytes, start, length)) { + outV.noNulls = false; + outV.isNull[batchIndex] = true; + return; + } + outV.vector[batchIndex] = LazyByte.parseByte(bytes, start, length, 10); + break; + case SHORT: + if (!LazyUtils.isNumberMaybe(bytes, start, length)) { + outV.noNulls = false; + outV.isNull[batchIndex] = true; + return; + } + outV.vector[batchIndex] = LazyShort.parseShort(bytes, start, length, 10); + break; + case INT: + if (!LazyUtils.isNumberMaybe(bytes, start, length)) { + outV.noNulls = false; + outV.isNull[batchIndex] = true; + return; + } + outV.vector[batchIndex] = LazyInteger.parseInt(bytes, start, length, 10); + break; + case LONG: + if (!LazyUtils.isNumberMaybe(bytes, start, length)) { + outV.noNulls = false; + outV.isNull[batchIndex] = true; + return; + } + outV.vector[batchIndex] = LazyLong.parseLong(bytes, start, length, 10); + break; + default: + throw new Error("Unexpected primitive category " + integerPrimitiveCategory); + } + } catch (Exception e) { + + // for any exception in conversion to integer, produce NULL + outV.noNulls = false; + outV.isNull[batchIndex] = true; + } + } + + @Override + public void evaluate(VectorizedRowBatch batch) { + + if (!integerPrimitiveCategoryKnown) { + String typeName = getOutputType().toLowerCase(); + TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeName); + integerPrimitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory(); + integerPrimitiveCategoryKnown = true; + } + + if (childExpressions != null) { + super.evaluateChildren(batch); + } + + BytesColumnVector inV = (BytesColumnVector) batch.cols[inputColumn]; + int[] sel = batch.selected; + int n = batch.size; + LongColumnVector outV = (LongColumnVector) batch.cols[outputColumn]; + + if (n == 0) { + + // Nothing to do + return; + } + + if (inV.noNulls) { + outV.noNulls = true; + if (inV.isRepeating) { + outV.isRepeating = true; + func(outV, inV, 0); + } else if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + func(outV, inV, i); + } + outV.isRepeating = false; + } else { + for(int i = 0; i != n; i++) { + func(outV, inV, i); + } + outV.isRepeating = false; + } + } else { + + // Handle case with nulls. Don't do function if the value is null, + // because the data may be undefined for a null value. + outV.noNulls = false; + if (inV.isRepeating) { + outV.isRepeating = true; + outV.isNull[0] = inV.isNull[0]; + if (!inV.isNull[0]) { + func(outV, inV, 0); + } + } else if (batch.selectedInUse) { + for(int j = 0; j != n; j++) { + int i = sel[j]; + outV.isNull[i] = inV.isNull[i]; + if (!inV.isNull[i]) { + func(outV, inV, i); + } + } + outV.isRepeating = false; + } else { + System.arraycopy(inV.isNull, 0, outV.isNull, 0, n); + for(int i = 0; i != n; i++) { + if (!inV.isNull[i]) { + func(outV, inV, i); + } + } + outV.isRepeating = false; + } + } + } + + @Override + public int getOutputColumn() { + return outputColumn; + } + + public void setOutputColumn(int outputColumn) { + this.outputColumn = outputColumn; + } + + public int getInputColumn() { + return inputColumn; + } + + public void setInputColumn(int inputColumn) { + this.inputColumn = inputColumn; + } + + @Override + public String vectorExpressionParameters() { + return "col " + inputColumn; + } + + @Override + public VectorExpressionDescriptor.Descriptor getDescriptor() { + VectorExpressionDescriptor.Builder b = new VectorExpressionDescriptor.Builder(); + b.setMode(VectorExpressionDescriptor.Mode.PROJECTION) + .setNumArguments(1) + .setArgumentTypes( + VectorExpressionDescriptor.ArgumentType.STRING_FAMILY) + .setInputExpressionTypes( + VectorExpressionDescriptor.InputExpressionType.COLUMN); + return b.build(); + } +} \ No newline at end of file diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java index 0cc0c9e..7cdf2c3 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToBoolean; +import org.apache.hadoop.hive.ql.exec.vector.expressions.CastStringToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastDoubleToBooleanViaDoubleToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastLongToBooleanViaLongToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastDateToBooleanViaLongToLong; @@ -46,7 +47,7 @@ */ @VectorizedExpressions({CastLongToBooleanViaLongToLong.class, CastDateToBooleanViaLongToLong.class, CastTimestampToBoolean.class, - CastDoubleToBooleanViaDoubleToLong.class, CastDecimalToBoolean.class}) + CastDoubleToBooleanViaDoubleToLong.class, CastDecimalToBoolean.class, CastStringToLong.class}) public class UDFToBoolean extends UDF { private final BooleanWritable booleanWritable = new BooleanWritable(); diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java index df2b42f..12f530b 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToLong; +import org.apache.hadoop.hive.ql.exec.vector.expressions.CastStringToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastDoubleToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastTimestampToLong; import org.apache.hadoop.hive.serde2.io.ByteWritable; @@ -42,7 +43,7 @@ * */ @VectorizedExpressions({CastTimestampToLong.class, CastDoubleToLong.class, - CastDecimalToLong.class}) + CastDecimalToLong.class, CastStringToLong.class}) public class UDFToByte extends UDF { private final ByteWritable byteWritable = new ByteWritable(); diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java index 42972c7..461ef86 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastDoubleToLong; +import org.apache.hadoop.hive.ql.exec.vector.expressions.CastStringToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastTimestampToLong; import org.apache.hadoop.hive.ql.io.RecordIdentifier; import org.apache.hadoop.hive.serde2.io.ByteWritable; @@ -43,7 +44,7 @@ * */ @VectorizedExpressions({CastTimestampToLong.class, CastDoubleToLong.class, - CastDecimalToLong.class}) + CastDecimalToLong.class, CastStringToLong.class}) public class UDFToInteger extends UDF { private final IntWritable intWritable = new IntWritable(); diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java index 847b535..11f6a6c 100755 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hive.ql.exec.UDF; import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToLong; +import org.apache.hadoop.hive.ql.exec.vector.expressions.CastStringToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastDoubleToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastTimestampToLong; import org.apache.hadoop.hive.serde2.io.ByteWritable; @@ -42,7 +43,7 @@ * */ @VectorizedExpressions({CastTimestampToLong.class, CastDoubleToLong.class, - CastDecimalToLong.class}) + CastDecimalToLong.class, CastStringToLong.class}) public class UDFToLong extends UDF { private final LongWritable longWritable = new LongWritable(); diff --git ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java index 0ae0c13..5372549 100644 --- ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java +++ ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToDouble; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToLong; +import org.apache.hadoop.hive.ql.exec.vector.expressions.CastStringToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastDoubleToLong; import org.apache.hadoop.hive.ql.exec.vector.expressions.CastTimestampToLong; import org.apache.hadoop.hive.serde2.io.ByteWritable; @@ -43,7 +44,7 @@ * */ @VectorizedExpressions({CastTimestampToLong.class, CastDoubleToLong.class, - CastDecimalToLong.class}) + CastDecimalToLong.class, CastStringToLong.class}) public class UDFToShort extends UDF { ShortWritable shortWritable = new ShortWritable(); diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out index 7cd9820..17b078f 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part_all_primitive.q.out @@ -141,10 +141,10 @@ POSTHOOK: Input: default@part_change_various_various_boolean_to_bigint POSTHOOK: Input: default@part_change_various_various_boolean_to_bigint@part=1 #### A masked pattern was here #### insert_num part c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12 c13 c14 c15 c16 c17 c18 c19 c20 c21 c22 c23 c24 c25 c26 c27 c28 c29 c30 c31 c32 c33 c34 c35 c36 c37 c38 c39 c40 c41 c42 c43 c44 c45 c46 c47 c48 c49 c50 c51 c52 c53 b -101 1 -128 NULL -2147483648 NULL Infinity 1.7976931348623157E308 99999999999999999999.999999999999999999 true 6229-06-28 02:54:28.970117179 NULL NULL -2147483648 NULL Infinity 1.7976931348623157E308 99999999999999999999.999999999999999999 -128 -128 -128 6229-06-28 02:54:28.970117179 NULL -128 -2147483648 NULL Infinity 1.7976931348623157E308 99999999999999999999.999999999999999999 –32768 –32768 –32768 6229-06-28 02:54:28.970117179 NULL -128 NULL NULL Infinity 1.7976931348623157E308 99999999999999999999.999999999999999999 -2147483648 -2147483648 -2147483648 6229-06-28 02:54:28.970117179 NULL -128 NULL -2147483648 Infinity 1.7976931348623157E308 99999999999999999999.999999999999999999 –9223372036854775808 –9223372036854775808 –9223372036854775808 6229-06-28 02:54:28.970117179 original +101 1 -128 NULL -2147483648 NULL Infinity 1.7976931348623157E308 99999999999999999999.999999999999999999 true 6229-06-28 02:54:28.970117179 true NULL -2147483648 NULL Infinity 1.7976931348623157E308 99999999999999999999.999999999999999999 -128 -128 -128 6229-06-28 02:54:28.970117179 true -128 -2147483648 NULL Infinity 1.7976931348623157E308 99999999999999999999.999999999999999999 –32768 –32768 –32768 6229-06-28 02:54:28.970117179 true -128 NULL NULL Infinity 1.7976931348623157E308 99999999999999999999.999999999999999999 -2147483648 -2147483648 -2147483648 6229-06-28 02:54:28.970117179 true -128 NULL -2147483648 Infinity 1.7976931348623157E308 99999999999999999999.999999999999999999 –9223372036854775808 –9223372036854775808 –9223372036854775808 6229-06-28 02:54:28.970117179 original 102 1 127 32767 2147483647 9223372036854775807 -Infinity -1.7976931348623157E308 -99999999999999999999.999999999999999999 false 5966-07-09 03:30:50.597 false 32767 2147483647 9223372036854775807 -Infinity -1.7976931348623157E308 -99999999999999999999.999999999999999999 127 127 127 5966-07-09 03:30:50.597 false 127 2147483647 9223372036854775807 -Infinity -1.7976931348623157E308 -99999999999999999999.999999999999999999 32767 32767 32767 5966-07-09 03:30:50.597 false 127 32767 9223372036854775807 -Infinity -1.7976931348623157E308 -99999999999999999999.999999999999999999 2147483647 2147483647 2147483647 5966-07-09 03:30:50.597 false 127 32767 2147483647 -Infinity -1.7976931348623157E308 -99999999999999999999.999999999999999999 9223372036854775807 9223372036854775807 9223372036854775807 5966-07-09 03:30:50.597 original 103 1 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL original -104 1 23 834 203332 888888857923222 -100.35978 30.774 66475.561431000000000000 true 1978-08-02 06:34:14 NULL 834 203332 888888857923222 -100.35978 30.774 66475.561431000000000000 23 23 23 1978-08-02 06:34:14 NULL 23 203332 888888857923222 -100.35978 30.774 66475.561431000000000000 834 834 834 1978-08-02 06:34:14 NULL 23 834 888888857923222 -100.35978 30.774 66475.561431000000000000 203332 203332 203332 1978-08-02 06:34:14 NULL 23 834 203332 -100.35978 30.774 66475.561431000000000000 888888857923222 888888857923222 888888857923222 1978-08-02 06:34:14 original +104 1 23 834 203332 888888857923222 -100.35978 30.774 66475.561431000000000000 true 1978-08-02 06:34:14 true 834 203332 888888857923222 -100.35978 30.774 66475.561431000000000000 23 23 23 1978-08-02 06:34:14 true 23 203332 888888857923222 -100.35978 30.774 66475.561431000000000000 834 834 834 1978-08-02 06:34:14 true 23 834 888888857923222 -100.35978 30.774 66475.561431000000000000 203332 203332 203332 1978-08-02 06:34:14 true 23 834 203332 -100.35978 30.774 66475.561431000000000000 888888857923222 888888857923222 888888857923222 1978-08-02 06:34:14 original 105 1 -99 -28300 -999992 -222282153733 NULL 46114.28 9250340.750000000000000000 false 1991-01-06 16:20:39.72036854 false -28300 -999992 -222282153733 NULL 46114.28 9250340.750000000000000000 -99 -99 -99 1991-01-06 16:20:39.72036854 false -99 -999992 -222282153733 NULL 46114.28 9250340.750000000000000000 -28300 -28300 -28300 1991-01-06 16:20:39.72036854 false -99 -28300 -222282153733 NULL 46114.28 9250340.750000000000000000 -999992 -999992 -999992 1991-01-06 16:20:39.72036854 false -99 -28300 -999992 NULL 46114.28 9250340.750000000000000000 -222282153733 -222282153733 -222282153733 1991-01-06 16:20:39.72036854 original PREHOOK: query: alter table part_change_various_various_boolean_to_bigint replace columns (insert_num int, c1 BOOLEAN, c2 BOOLEAN, c3 BOOLEAN, c4 BOOLEAN, c5 BOOLEAN, c6 BOOLEAN, c7 BOOLEAN, c8 BOOLEAN, c9 BOOLEAN, @@ -266,7 +266,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: part_change_various_various_boolean_to_bigint - Statistics: Num rows: 10 Data size: 5074 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 10 Data size: 5126 Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55] @@ -321,14 +321,14 @@ POSTHOOK: Input: default@part_change_various_various_boolean_to_bigint POSTHOOK: Input: default@part_change_various_various_boolean_to_bigint@part=1 #### A masked pattern was here #### insert_num part c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12 c13 c14 c15 c16 c17 c18 c19 c20 c21 c22 c23 c24 c25 c26 c27 c28 c29 c30 c31 c32 c33 c34 c35 c36 c37 c38 c39 c40 c41 c42 c43 c44 c45 c46 c47 c48 c49 c50 c51 c52 c53 b -101 1 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL -128 -128 -128 -128 -128 -128 -128 -128 -128 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL -2147483648 -2147483648 -2147483648 -2147483648 -2147483648 -2147483648 -2147483648 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL new -101 1 true NULL true NULL true true true true true NULL NULL 0 NULL -1 -1 NULL -128 -128 -128 84 NULL -128 0 NULL -1 -1 NULL NULL NULL NULL -8620 NULL -128 NULL NULL 2147483647 2147483647 NULL -2147483648 -2147483648 -2147483648 1272503892 NULL -128 NULL -2147483648 9223372036854775807 9223372036854775807 NULL NULL NULL NULL 134416490068 original +101 1 NULL NULL NULL NULL NULL NULL NULL true NULL NULL -128 -128 -128 -128 -128 -128 -128 -128 -128 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL -2147483648 -2147483648 -2147483648 -2147483648 -2147483648 -2147483648 -2147483648 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL new +101 1 true NULL true NULL true true true true true 1 NULL 0 NULL -1 -1 NULL -128 -128 -128 84 1 -128 0 NULL -1 -1 NULL NULL NULL NULL -8620 1 -128 NULL NULL 2147483647 2147483647 NULL -2147483648 -2147483648 -2147483648 1272503892 1 -128 NULL -2147483648 9223372036854775807 9223372036854775807 NULL NULL NULL NULL 134416490068 original 102 1 NULL NULL NULL NULL NULL NULL NULL true NULL NULL 127 127 127 127 127 127 127 127 127 NULL NULL NULL 32767 32767 32767 32767 32767 32767 32767 32767 NULL NULL NULL NULL 2147483647 2147483647 2147483647 2147483647 2147483647 2147483647 2147483647 NULL NULL NULL NULL NULL 9223372036854775807 9223372036854775807 9223372036854775807 9223372036854775807 9223372036854775807 9223372036854775807 NULL new 102 1 true true true true true true true true true 0 -1 -1 -1 0 0 NULL 127 127 127 -38 0 127 -1 -1 0 0 NULL 32767 32767 32767 7898 0 127 32767 -1 -2147483648 -2147483648 NULL 2147483647 2147483647 2147483647 1563893466 0 127 32767 2147483647 -9223372036854775808 -9223372036854775808 NULL 9223372036854775807 9223372036854775807 9223372036854775807 126117945050 original 103 1 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL new 103 1 NULL NULL NULL NULL NULL NULL NULL false NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL original -104 1 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 23 23 23 23 23 23 23 23 23 NULL NULL NULL 834 834 834 834 834 834 834 834 NULL NULL NULL NULL 203332 203332 203332 203332 203332 203332 203332 NULL NULL NULL NULL NULL 888888847499264 888888857923222 888888857923222 888888857923222 888888857923222 888888857923222 NULL new -104 1 true true true true true true true true true NULL 66 68 -106 -100 30 NULL 23 23 23 86 NULL 23 6724 3734 -100 30 NULL 834 834 834 -12970 NULL 23 834 -1868624234 -100 30 66475 203332 203332 203332 270912854 NULL 23 834 203332 -100 30 66475 888888857923222 888888857923222 888888857923222 270912854 original +104 1 NULL NULL NULL NULL NULL NULL NULL true NULL NULL 23 23 23 23 23 23 23 23 23 NULL NULL NULL 834 834 834 834 834 834 834 834 NULL NULL NULL NULL 203332 203332 203332 203332 203332 203332 203332 NULL NULL NULL NULL NULL 888888847499264 888888857923222 888888857923222 888888857923222 888888857923222 888888857923222 NULL new +104 1 true true true true true true true true true 1 66 68 -106 -100 30 NULL 23 23 23 86 1 23 6724 3734 -100 30 NULL 834 834 834 -12970 1 23 834 -1868624234 -100 30 66475 203332 203332 203332 270912854 1 23 834 203332 -100 30 66475 888888857923222 888888857923222 888888857923222 270912854 original 105 1 NULL NULL NULL NULL NULL NULL NULL true NULL NULL -99 -99 -99 -99 -99 -99 -99 -99 -99 NULL NULL NULL -28300 -28300 -28300 -28300 -28300 -28300 -28300 -28300 NULL NULL NULL NULL -999992 -999992 -999992 -999992 -999992 -999992 -999992 NULL NULL NULL NULL NULL -222282153984 -222282153733 -222282153733 -222282153733 -222282153733 -222282153733 NULL new 105 1 true true true true NULL true true true true 0 116 -56 -5 NULL 34 NULL -99 -99 -99 -41 0 -99 -16952 -32517 NULL -19422 NULL -28300 -28300 -28300 -16681 0 -99 -28300 1056145659 NULL 46114 9250340 -999992 -999992 -999992 663207639 0 -99 -28300 -999992 NULL 46114 9250340 -222282153733 -222282153733 -222282153733 663207639 original PREHOOK: query: drop table part_change_various_various_boolean_to_bigint @@ -418,10 +418,10 @@ POSTHOOK: Input: default@part_change_various_various_decimal_to_double POSTHOOK: Input: default@part_change_various_various_decimal_to_double@part=1 #### A masked pattern was here #### insert_num part c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12 c13 c14 c15 c16 c17 c18 c19 c20 c21 c22 c23 c24 c25 c26 c27 c28 c29 c30 c31 c32 c33 b -101 1 NULL -128 NULL -2147483648 NULL Infinity 1.7976931348623157E308 99999999999999999999.999999999999999999 99999999999999999999.9999 99999999999999999999.9999 6229-06-28 02:54:28.970117179 NULL -128 NULL -2147483648 NULL 99999999999999999999.999999999999999999 1.7976931348623157E308 340282347000000000000000000000000000000000 3402823470000000000000000 3402823470000000000000000 6229-06-28 02:54:28.970117179 NULL -128 NULL -2147483648 NULL 99999999999999999999.999999999999999999 Infinity 1.79769313486231570E+308 1.79769313486231570E+308 1.79769313486231570E+308 6229-06-28 02:54:28.970117179 original +101 1 true -128 NULL -2147483648 NULL Infinity 1.7976931348623157E308 99999999999999999999.999999999999999999 99999999999999999999.9999 99999999999999999999.9999 6229-06-28 02:54:28.970117179 true -128 NULL -2147483648 NULL 99999999999999999999.999999999999999999 1.7976931348623157E308 340282347000000000000000000000000000000000 3402823470000000000000000 3402823470000000000000000 6229-06-28 02:54:28.970117179 true -128 NULL -2147483648 NULL 99999999999999999999.999999999999999999 Infinity 1.79769313486231570E+308 1.79769313486231570E+308 1.79769313486231570E+308 6229-06-28 02:54:28.970117179 original 102 1 false 127 32767 2147483647 9223372036854775807 -Infinity -1.7976931348623157E308 -99999999999999999999.999999999999999999 -99999999999999999999.999 -99999999999999999999.999 5966-07-09 03:30:50.597 false 127 32767 2147483647 9223372036854775807 -99999999999999999999.999999999999999999 -1.7976931348623157E308 -340282347000000000000000000000000000000000 -340282347000000000000000 -340282347000000000000000 5966-07-09 03:30:50.597 false 127 32767 2147483647 9223372036854775807 -99999999999999999999.999999999999999999 -Infinity -1.79769313486231570E+308 -1.79769313486231570E+308 -1.79769313486231570E+308 5966-07-09 03:30:50.597 original 103 1 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL original -104 1 NULL 23 834 203332 888888857923222 -100.35978 30.774 66475.561431 66475.561431 66475.561431 1978-08-02 06:34:14 NULL 23 834 203332 888888857923222 66475.561431000000000000 30.774 -100.3597812 -100.3597812 -100.3597812 1978-08-02 06:34:14 NULL 23 834 203332 888888857923222 66475.561431000000000000 -100.35978 30.774 30.774 30.774 1978-08-02 06:34:14 original +104 1 true 23 834 203332 888888857923222 -100.35978 30.774 66475.561431 66475.561431 66475.561431 1978-08-02 06:34:14 true 23 834 203332 888888857923222 66475.561431000000000000 30.774 -100.3597812 -100.3597812 -100.3597812 1978-08-02 06:34:14 true 23 834 203332 888888857923222 66475.561431000000000000 -100.35978 30.774 30.774 30.774 1978-08-02 06:34:14 original 105 1 false -99 -28300 -999992 -222282153733 NULL 46114.28 9250340.75 9250340.75 9250340.75 1991-01-06 16:20:39.72036854 false -99 -28300 -999992 -222282153733 9250340.750000000000000000 46114.28 –32768 –32768 –32768 1991-01-06 16:20:39.72036854 false -99 -28300 -999992 -222282153733 9250340.750000000000000000 NULL 46114.28 46114.28 46114.28 1991-01-06 16:20:39.72036854 original PREHOOK: query: alter table part_change_various_various_decimal_to_double replace columns (insert_num int, c1 DECIMAL(38,18), c2 DECIMAL(38,18), c3 DECIMAL(38,18), c4 DECIMAL(38,18), c5 DECIMAL(38,18), c6 DECIMAL(38,18), c7 DECIMAL(38,18), c8 DECIMAL(38,18), c9 DECIMAL(38,18), c10 DECIMAL(38,18), c11 DECIMAL(38,18), @@ -515,7 +515,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: part_change_various_various_decimal_to_double - Statistics: Num rows: 6 Data size: 2723 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 2735 Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35] @@ -570,10 +570,10 @@ POSTHOOK: Input: default@part_change_various_various_decimal_to_double POSTHOOK: Input: default@part_change_various_various_decimal_to_double@part=1 #### A masked pattern was here #### insert_num part c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12 c13 c14 c15 c16 c17 c18 c19 c20 c21 c22 c23 c24 c25 c26 c27 c28 c29 c30 c31 c32 c33 b -101 1 NULL -128.000000000000000000 NULL -2147483648.000000000000000000 NULL NULL NULL 99999999999999999999.999999999999999999 99999999999999999999.999900000000000000 99999999999999999999.999900000000000000 134416490068.970120000000000000 NULL -128.0 NULL -2.14748365E9 NULL 1.0E20 Infinity Infinity 3.4028236E24 3.4028236E24 1.3441649E11 NULL -128.0 NULL -2.147483648E9 NULL 1.0E20 Infinity 1.7976931348623157E308 1.7976931348623157E308 1.7976931348623157E308 1.3441649006897012E11 original +101 1 1.000000000000000000 -128.000000000000000000 NULL -2147483648.000000000000000000 NULL NULL NULL 99999999999999999999.999999999999999999 99999999999999999999.999900000000000000 99999999999999999999.999900000000000000 134416490068.970120000000000000 1.0 -128.0 NULL -2.14748365E9 NULL 1.0E20 Infinity Infinity 3.4028236E24 3.4028236E24 1.3441649E11 1.0 -128.0 NULL -2.147483648E9 NULL 1.0E20 Infinity 1.7976931348623157E308 1.7976931348623157E308 1.7976931348623157E308 1.3441649006897012E11 original 102 1 0.000000000000000000 127.000000000000000000 32767.000000000000000000 2147483647.000000000000000000 9223372036854775807.000000000000000000 NULL NULL -99999999999999999999.999999999999999999 -99999999999999999999.999000000000000000 -99999999999999999999.999000000000000000 126117945050.597000000000000000 0.0 127.0 32767.0 2.14748365E9 9.223372E18 -1.0E20 -Infinity -Infinity -3.4028233E23 -3.4028233E23 1.26117945E11 0.0 127.0 32767.0 2.147483647E9 9.223372036854776E18 -1.0E20 -Infinity -1.7976931348623157E308 -1.7976931348623157E308 -1.7976931348623157E308 1.26117945050597E11 original 103 1 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL original -104 1 NULL 23.000000000000000000 834.000000000000000000 203332.000000000000000000 888888857923222.000000000000000000 -100.359780000000000000 30.774000000000000000 66475.561431000000000000 66475.561431000000000000 66475.561431000000000000 270912854.000000000000000000 NULL 23.0 834.0 203332.0 8.8888885E14 66475.56 30.774 -100.35978 -100.35978 -100.35978 2.70912864E8 NULL 23.0 834.0 203332.0 8.88888857923222E14 66475.561431 -100.35977935791016 30.774 30.774 30.774 2.70912854E8 original +104 1 1.000000000000000000 23.000000000000000000 834.000000000000000000 203332.000000000000000000 888888857923222.000000000000000000 -100.359780000000000000 30.774000000000000000 66475.561431000000000000 66475.561431000000000000 66475.561431000000000000 270912854.000000000000000000 1.0 23.0 834.0 203332.0 8.8888885E14 66475.56 30.774 -100.35978 -100.35978 -100.35978 2.70912864E8 1.0 23.0 834.0 203332.0 8.88888857923222E14 66475.561431 -100.35977935791016 30.774 30.774 30.774 2.70912854E8 original 105 1 0.000000000000000000 -99.000000000000000000 -28300.000000000000000000 -999992.000000000000000000 -222282153733.000000000000000000 NULL 46114.280000000000000000 9250340.750000000000000000 9250340.750000000000000000 9250340.750000000000000000 663207639.720368500000000000 0.0 -99.0 -28300.0 -999992.0 -2.22282154E11 9250341.0 46114.28 NULL NULL NULL 6.6320762E8 0.0 -99.0 -28300.0 -999992.0 -2.22282153733E11 9250340.75 NULL 46114.28 46114.28 46114.28 6.632076397203685E8 original 111 1 NULL NULL NULL -46114.000000000000000000 -46114.000000000000000000 -46114.285000000000000000 -46114.284799488000000000 -46114.284799488000000000 -46114.284799488000000000 -46114.284799488000000000 NULL NULL NULL NULL NULL NULL -9.0E-8 -9.0E-8 -9.0E-8 -9.0E-8 -9.0E-8 NULL NULL NULL NULL NULL NULL -9.0E-8 -9.000000034120603E-8 -9.0E-8 -9.0E-8 -9.0E-8 NULL new PREHOOK: query: drop table part_change_various_various_decimal_to_double @@ -626,10 +626,10 @@ POSTHOOK: Input: default@part_change_various_various_timestamp POSTHOOK: Input: default@part_change_various_various_timestamp@part=1 #### A masked pattern was here #### insert_num part c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12 b -101 1 NULL -128 NULL -2147483648 NULL Infinity 1.7976931348623157E308 99999999999999999999.999999999999999999 6229-06-28 02:54:28.970117179 6229-06-28 02:54:28.97011 6229-06-28 02:54:28.97011 1950-12-18 original +101 1 true -128 NULL -2147483648 NULL Infinity 1.7976931348623157E308 99999999999999999999.999999999999999999 6229-06-28 02:54:28.970117179 6229-06-28 02:54:28.97011 6229-06-28 02:54:28.97011 1950-12-18 original 102 1 false 127 32767 2147483647 9223372036854775807 -Infinity -1.7976931348623157E308 -99999999999999999999.999999999999999999 5966-07-09 03:30:50.597 5966-07-09 03:30:50.597 5966-07-09 03:30:50.597 2049-12-18 original 103 1 NULL NULL NULL NULL NULL NULL NULL NULL NULL original -104 1 NULL 23 834 203332 888888857923222 -100.35978 30.774 66475.561431000000000000 1978-08-02 06:34:14 1978-08-02 06:34:14 1978-08-02 06:34:14 2021-09-24 original +104 1 true 23 834 203332 888888857923222 -100.35978 30.774 66475.561431000000000000 1978-08-02 06:34:14 1978-08-02 06:34:14 1978-08-02 06:34:14 2021-09-24 original 105 1 false -99 -28300 -999992 -222282153733 NULL 46114.28 9250340.750000000000000000 1991-01-06 16:20:39.72036854 1991-01-06 16:20:39.72036 1991-01-06 16:20:39.72036 2024-11-11 original PREHOOK: query: alter table part_change_various_various_timestamp replace columns (insert_num int, c1 TIMESTAMP, c2 TIMESTAMP, c3 TIMESTAMP, c4 TIMESTAMP, c5 TIMESTAMP, c6 TIMESTAMP, c7 TIMESTAMP, c8 TIMESTAMP, c9 TIMESTAMP, c10 TIMESTAMP, c11 TIMESTAMP, c12 TIMESTAMP, b STRING) PREHOOK: type: ALTERTABLE_REPLACECOLS @@ -686,7 +686,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: part_change_various_various_timestamp - Statistics: Num rows: 6 Data size: 903 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 6 Data size: 907 Basic stats: COMPLETE Column stats: PARTIAL TableScan Vectorization: native: true projectedOutputColumns: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] @@ -741,10 +741,10 @@ POSTHOOK: Input: default@part_change_various_various_timestamp POSTHOOK: Input: default@part_change_various_various_timestamp@part=1 #### A masked pattern was here #### insert_num part c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12 b -101 1 NULL 1969-12-31 15:59:59.872 NULL 1969-12-06 19:28:36.352 NULL NULL NULL NULL 6229-06-28 02:54:28.970117179 6229-06-28 02:54:28.97011 6229-06-28 02:54:28.97011 1950-12-18 00:00:00 original +101 1 1969-12-31 16:00:00.001 1969-12-31 15:59:59.872 NULL 1969-12-06 19:28:36.352 NULL NULL NULL NULL 6229-06-28 02:54:28.970117179 6229-06-28 02:54:28.97011 6229-06-28 02:54:28.97011 1950-12-18 00:00:00 original 102 1 1969-12-31 16:00:00 1969-12-31 16:00:00.127 1969-12-31 16:00:32.767 1970-01-25 12:31:23.647 NULL NULL 1969-12-31 16:00:00 NULL 5966-07-09 03:30:50.597 5966-07-09 03:30:50.597 5966-07-09 03:30:50.597 2049-12-18 00:00:00 original 103 1 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL original -104 1 NULL 1969-12-31 16:00:00.023 1969-12-31 16:00:00.834 1969-12-31 16:03:23.332 NULL 1969-12-31 15:58:19.640220643 1969-12-31 16:00:30.774 1970-01-01 10:27:55.561431 1978-08-02 06:34:14 1978-08-02 06:34:14 1978-08-02 06:34:14 2021-09-24 00:00:00 original +104 1 1969-12-31 16:00:00.001 1969-12-31 16:00:00.023 1969-12-31 16:00:00.834 1969-12-31 16:03:23.332 NULL 1969-12-31 15:58:19.640220643 1969-12-31 16:00:30.774 1970-01-01 10:27:55.561431 1978-08-02 06:34:14 1978-08-02 06:34:14 1978-08-02 06:34:14 2021-09-24 00:00:00 original 105 1 1969-12-31 16:00:00 1969-12-31 15:59:59.901 1969-12-31 15:59:31.7 1969-12-31 15:43:20.008 1962-12-15 22:57:26.267 NULL 1970-01-01 04:48:34.28 1970-04-17 17:32:20.75 1991-01-06 16:20:39.72036854 1991-01-06 16:20:39.72036 1991-01-06 16:20:39.72036 2024-11-11 00:00:00 original 111 1 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL new PREHOOK: query: drop table part_change_various_various_timestamp diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java index 264335c..dddd6a7 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java @@ -376,7 +376,7 @@ public boolean readField(int fieldIndex) throws IOException { if (fieldLength == 4) { if ((bytes[i] == 'T' || bytes[i] == 't') && (bytes[i + 1] == 'R' || bytes[i + 1] == 'r') && - (bytes[i + 2] == 'U' || bytes[i + 1] == 'u') && + (bytes[i + 2] == 'U' || bytes[i + 2] == 'u') && (bytes[i + 3] == 'E' || bytes[i + 3] == 'e')) { currentBoolean = true; } else {