commit 97123f1f5aaca407bb99e47be762b0cd1f2636e4 Author: Janaki Lahorani Date: Fri Mar 16 09:10:11 2018 -0700 HIVE-17843: Read unsigned int values correctly from parquet files Parquet file format supports unsigned int data. But in hive all integer data types are signed. Some of the unsigned integer values cannot be valid in hive. The invalid values will be returned as null. Change-Id: Ia51f2169a918079d20a71c4ac211c6af06a1828b diff --git data/files/data_including_invalid_values.parquet data/files/data_including_invalid_values.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a6e5378621cd8d36a6554065a7b30b0595f7f3ac Binary files /dev/null and data/files/data_including_invalid_values.parquet differ diff --git data/files/data_with_valid_values.parquet data/files/data_with_valid_values.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ec39bed6dcffe37e6ed53c470bcd92019683259c Binary files /dev/null and data/files/data_with_valid_values.parquet differ diff --git data/files/test_uint.parquet data/files/test_uint.parquet new file mode 100644 index 0000000000000000000000000000000000000000..de0ed3cffee3691ffb9bd55228d5afd5bd12a5ea Binary files /dev/null and data/files/test_uint.parquet differ diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java index 6b92b96a584bd88c95baa3788a6d21824d8e150c..53370e43d71079a4c1c8003c2632b0ed4e0ce70b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java @@ -95,121 +95,279 @@ public void addFloat(final float value) { @Override PrimitiveConverter getConverter(final PrimitiveType type, final int index, final ConverterParent parent, TypeInfo hiveTypeInfo) { - if (hiveTypeInfo != null) { - switch (hiveTypeInfo.getTypeName()) { - case serdeConstants.BIGINT_TYPE_NAME: - return new PrimitiveConverter() { - @Override - public void addInt(final int value) { - parent.set(index, new LongWritable((long) value)); - } - }; - case serdeConstants.FLOAT_TYPE_NAME: - return new PrimitiveConverter() { - @Override - public void addInt(final int value) { - parent.set(index, new FloatWritable((float) value)); - } - }; - case serdeConstants.DOUBLE_TYPE_NAME: - return new PrimitiveConverter() { - @Override - public void addInt(final int value) { - parent.set(index, new DoubleWritable((float) value)); - } - }; - case serdeConstants.SMALLINT_TYPE_NAME: - return new PrimitiveConverter() { - @Override - public void addInt(final int value) { - if ((value >= Short.MIN_VALUE) && (value <= Short.MAX_VALUE)) { - parent.set(index, new IntWritable((int)value)); - } else { - parent.set(index, null); + if (OriginalType.UINT_8 == type.getOriginalType() || + OriginalType.UINT_16 == type.getOriginalType() || + OriginalType.UINT_32 == type.getOriginalType() || + OriginalType.UINT_64 == type.getOriginalType()) { + if (hiveTypeInfo != null) { + switch (hiveTypeInfo.getTypeName()) { + case serdeConstants.BIGINT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addInt(final int value) { + if (value >= 0) { + parent.set(index, new LongWritable((long) value)); + } else { + parent.set(index, null); + } } - } - }; - case serdeConstants.TINYINT_TYPE_NAME: - return new PrimitiveConverter() { - @Override - public void addInt(final int value) { - if ((value >= Byte.MIN_VALUE) && (value <= Byte.MAX_VALUE)) { - parent.set(index, new IntWritable((int)value)); - } else { - parent.set(index, null); + }; + case serdeConstants.FLOAT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addInt(final int value) { + if (value >= 0) { + parent.set(index, new FloatWritable((float) value)); + } else { + parent.set(index, null); + } + } + }; + case serdeConstants.DOUBLE_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addInt(final int value) { + if (value >= 0) { + parent.set(index, new DoubleWritable((float) value)); + } else { + parent.set(index, null); + } + } + }; + case serdeConstants.SMALLINT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addInt(final int value) { + if ((value >= 0) && (value <= Short.MAX_VALUE)) { + parent.set(index, new IntWritable((int) value)); + } else { + parent.set(index, null); + } + } + }; + case serdeConstants.TINYINT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addInt(final int value) { + if ((value >= 0) && (value <= Byte.MAX_VALUE)) { + parent.set(index, new IntWritable((int) value)); + } else { + parent.set(index, null); + } } + }; + default: + return new PrimitiveConverter() { + @Override public void addInt(final int value) { + if (value >= 0) { + parent.set(index, new IntWritable(value)); + } else { + parent.set(index, null); + } + } + }; + } + } + return new PrimitiveConverter() { + @Override public void addInt(final int value) { + if (value >= 0) { + parent.set(index, new IntWritable(value)); + } else { + parent.set(index, null); } - }; + } + }; + } else { + if (hiveTypeInfo != null) { + switch (hiveTypeInfo.getTypeName()) { + case serdeConstants.BIGINT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addInt(final int value) { + parent.set(index, new LongWritable((long) value)); + } + }; + case serdeConstants.FLOAT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addInt(final int value) { + parent.set(index, new FloatWritable((float) value)); + } + }; + case serdeConstants.DOUBLE_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addInt(final int value) { + parent.set(index, new DoubleWritable((float) value)); + } + }; + case serdeConstants.SMALLINT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addInt(final int value) { + if ((value >= Short.MIN_VALUE) && (value <= Short.MAX_VALUE)) { + parent.set(index, new IntWritable((int) value)); + } else { + parent.set(index, null); + } + } + }; + case serdeConstants.TINYINT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addInt(final int value) { + if ((value >= Byte.MIN_VALUE) && (value <= Byte.MAX_VALUE)) { + parent.set(index, new IntWritable((int) value)); + } else { + parent.set(index, null); + } + } + }; + default: + return new PrimitiveConverter() { + @Override public void addInt(final int value) { + parent.set(index, new IntWritable(value)); + } + }; + } } + return new PrimitiveConverter() { + @Override public void addInt(final int value) { + parent.set(index, new IntWritable(value)); + } + }; } - return new PrimitiveConverter() { - @Override - public void addInt(final int value) { - parent.set(index, new IntWritable(value)); - } - }; } }, EINT64_CONVERTER(Long.TYPE) { @Override PrimitiveConverter getConverter(final PrimitiveType type, final int index, final ConverterParent parent, TypeInfo hiveTypeInfo) { - if(hiveTypeInfo != null) { - switch(hiveTypeInfo.getTypeName()) { - case serdeConstants.FLOAT_TYPE_NAME: - return new PrimitiveConverter() { - @Override - public void addLong(final long value) { - parent.set(index, new FloatWritable(value)); - } - }; - case serdeConstants.DOUBLE_TYPE_NAME: - return new PrimitiveConverter() { - @Override - public void addLong(final long value) { - parent.set(index, new DoubleWritable(value)); - } - }; - case serdeConstants.INT_TYPE_NAME: - return new PrimitiveConverter() { - @Override - public void addLong(long value) { - if ((value >= Integer.MIN_VALUE) && (value <= Integer.MAX_VALUE)) { - parent.set(index, new IntWritable((int)value)); - } else { - parent.set(index, null); + if (OriginalType.UINT_8 == type.getOriginalType() || + OriginalType.UINT_16 == type.getOriginalType() || + OriginalType.UINT_32 == type.getOriginalType() || + OriginalType.UINT_64 == type.getOriginalType()) { + if (hiveTypeInfo != null) { + switch (hiveTypeInfo.getTypeName()) { + case serdeConstants.FLOAT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addLong(final long value) { + if (value >= 0) { + parent.set(index, new FloatWritable(value)); + } else { + parent.set(index, null); + } } - } - }; - case serdeConstants.SMALLINT_TYPE_NAME: - return new PrimitiveConverter() { - @Override - public void addLong(long value) { - if ((value >= Short.MIN_VALUE) && (value <= Short.MAX_VALUE)) { - parent.set(index, new IntWritable((int)value)); - } else { - parent.set(index, null); + }; + case serdeConstants.DOUBLE_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addLong(final long value) { + if (value >= 0) { + parent.set(index, new DoubleWritable(value)); + } else { + parent.set(index, null); + } } - } - }; - case serdeConstants.TINYINT_TYPE_NAME: - return new PrimitiveConverter() { - @Override - public void addLong(long value) { - if ((value >= Byte.MIN_VALUE) && (value <= Byte.MAX_VALUE)) { - parent.set(index, new IntWritable((int)value)); - } else { - parent.set(index, null); + }; + case serdeConstants.INT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addLong(long value) { + if ((value >= 0) && (value <= Integer.MAX_VALUE)) { + parent.set(index, new IntWritable((int) value)); + } else { + parent.set(index, null); + } + } + }; + case serdeConstants.SMALLINT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addLong(long value) { + if ((value >= 0) && (value <= Short.MAX_VALUE)) { + parent.set(index, new IntWritable((int) value)); + } else { + parent.set(index, null); + } + } + }; + case serdeConstants.TINYINT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addLong(long value) { + if ((value >= 0) && (value <= Byte.MAX_VALUE)) { + parent.set(index, new IntWritable((int) value)); + } else { + parent.set(index, null); + } } + }; + default: + return new PrimitiveConverter() { + @Override public void addLong(final long value) { + if (value >= 0) { + parent.set(index, new LongWritable(value)); + } else { + parent.set(index, null); + } + } + }; + } + } + return new PrimitiveConverter() { + @Override public void addLong(final long value) { + if (value >= 0) { + parent.set(index, new LongWritable(value)); + } else { + parent.set(index, null); } - }; + } + }; + } else { + if (hiveTypeInfo != null) { + switch (hiveTypeInfo.getTypeName()) { + case serdeConstants.FLOAT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addLong(final long value) { + parent.set(index, new FloatWritable(value)); + } + }; + case serdeConstants.DOUBLE_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addLong(final long value) { + parent.set(index, new DoubleWritable(value)); + } + }; + case serdeConstants.INT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addLong(long value) { + if ((value >= Integer.MIN_VALUE) && (value <= Integer.MAX_VALUE)) { + parent.set(index, new IntWritable((int) value)); + } else { + parent.set(index, null); + } + } + }; + case serdeConstants.SMALLINT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addLong(long value) { + if ((value >= Short.MIN_VALUE) && (value <= Short.MAX_VALUE)) { + parent.set(index, new IntWritable((int) value)); + } else { + parent.set(index, null); + } + } + }; + case serdeConstants.TINYINT_TYPE_NAME: + return new PrimitiveConverter() { + @Override public void addLong(long value) { + if ((value >= Byte.MIN_VALUE) && (value <= Byte.MAX_VALUE)) { + parent.set(index, new IntWritable((int) value)); + } else { + parent.set(index, null); + } + } + }; + default: + return new PrimitiveConverter() { + @Override public void addLong(final long value) { + parent.set(index, new LongWritable(value)); + } + }; + } } + return new PrimitiveConverter() { + @Override public void addLong(final long value) { + parent.set(index, new LongWritable(value)); + } + }; } - return new PrimitiveConverter() { - @Override - public void addLong(final long value) { - parent.set(index, new LongWritable(value)); - } - }; } }, EBINARY_CONVERTER(Binary.class) { diff --git ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java index d8009dacd6c100217b92e1be232447685681f54d..17d6e338e6ff136199ed4a3f6bde0f29368f5e2e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java @@ -475,6 +475,80 @@ public boolean isValid(long value) { } } + /** + * The reader who reads unsigned long data. + */ + public static class TypesFromUInt64PageReader extends TypesFromInt64PageReader { + + public TypesFromUInt64PageReader(ValuesReader realReader, int length) { + super(realReader, length); + } + + public TypesFromUInt64PageReader(Dictionary dict, int length) { + super(dict, length); + } + + @Override + public boolean isValid(long value) { + return (value >= 0); + } + } + + /** + * The reader who reads unsigned long data using int type. + */ + public static class Types64UInt2IntPageReader extends TypesFromInt64PageReader { + + public Types64UInt2IntPageReader(ValuesReader realReader, int length) { + super(realReader, length); + } + + public Types64UInt2IntPageReader(Dictionary dict, int length) { + super(dict, length); + } + + @Override + public boolean isValid(long value) { + return ((value <= Integer.MAX_VALUE) && (value >= 0)); + } + } + + /** + * The reader who reads unsigned long data using smallint type. + */ + public static class Types64UInt2SmallintPageReader extends TypesFromInt64PageReader { + public Types64UInt2SmallintPageReader(ValuesReader realReader, int length) { + super(realReader, length); + } + + public Types64UInt2SmallintPageReader(Dictionary dict, int length) { + super(dict, length); + } + + @Override + public boolean isValid(long value) { + return ((value <= Short.MAX_VALUE) && (value >= 0)); + } + } + + /** + * The reader who reads unsigned long data using tinyint type. + */ + public static class Types64UInt2TinyintPageReader extends TypesFromInt64PageReader { + public Types64UInt2TinyintPageReader(ValuesReader realReader, int length) { + super(realReader, length); + } + + public Types64UInt2TinyintPageReader(Dictionary dict, int length) { + super(dict, length); + } + + @Override + public boolean isValid(long value) { + return ((value <= Byte.MAX_VALUE) && (value >= 0)); + } + } + /** * The reader who reads int data using smallint type. */ @@ -511,6 +585,60 @@ public boolean isValid(long value) { } } + /** + * The reader who reads unsigned int data. + */ + public static class TypesFromUInt32PageReader extends TypesFromInt32PageReader { + public TypesFromUInt32PageReader(ValuesReader realReader, int length) { + super(realReader, length); + } + + public TypesFromUInt32PageReader(Dictionary dict, int length) { + super(dict, length); + } + + @Override + public boolean isValid(long value) { + return (value >= 0); + } + } + + /** + * The reader who reads unsigned int data using smallint type. + */ + public static class Types32UInt2SmallintPageReader extends TypesFromInt32PageReader { + public Types32UInt2SmallintPageReader(ValuesReader realReader, int length) { + super(realReader, length); + } + + public Types32UInt2SmallintPageReader(Dictionary dict, int length) { + super(dict, length); + } + + @Override + public boolean isValid(long value) { + return ((value <= Short.MAX_VALUE) && (value >= 0)); + } + } + + /** + * The reader who reads unsigned int data using tinyint type. + */ + public static class Types32UInt2TinyintPageReader extends TypesFromInt32PageReader { + public Types32UInt2TinyintPageReader(ValuesReader realReader, int length) { + super(realReader, length); + } + + public Types32UInt2TinyintPageReader(Dictionary dict, int length) { + super(dict, length); + } + + @Override + public boolean isValid(long value) { + return ((value <= Byte.MAX_VALUE) && (value >= 0)); + } + } + /** * The reader who reads from the underlying float value value. Implementation is in consist with * ETypeConverter EFLOAT_CONVERTER @@ -919,31 +1047,68 @@ private static ParquetDataColumnReader getDataColumnReaderByTypeHelper(boolean i switch (parquetType.getPrimitiveTypeName()) { case INT32: - switch (hiveType.getTypeName()) { - case serdeConstants.SMALLINT_TYPE_NAME: - return isDictionary ? new Types32Int2SmallintPageReader(dictionary, - length) : new Types32Int2SmallintPageReader(valuesReader, length); - case serdeConstants.TINYINT_TYPE_NAME: - return isDictionary ? new Types32Int2TinyintPageReader(dictionary, - length) : new Types32Int2TinyintPageReader(valuesReader, length); - default: - return isDictionary ? new TypesFromInt32PageReader(dictionary, - length) : new TypesFromInt32PageReader(valuesReader, length); + if (OriginalType.UINT_8 == parquetType.getOriginalType() || + OriginalType.UINT_16 == parquetType.getOriginalType() || + OriginalType.UINT_32 == parquetType.getOriginalType() || + OriginalType.UINT_64 == parquetType.getOriginalType()) { + switch (hiveType.getTypeName()) { + case serdeConstants.SMALLINT_TYPE_NAME: + return isDictionary ? new Types32UInt2SmallintPageReader(dictionary, + length) : new Types32UInt2SmallintPageReader(valuesReader, length); + case serdeConstants.TINYINT_TYPE_NAME: + return isDictionary ? new Types32UInt2TinyintPageReader(dictionary, + length) : new Types32UInt2TinyintPageReader(valuesReader, length); + default: + return isDictionary ? new TypesFromUInt32PageReader(dictionary, + length) : new TypesFromUInt32PageReader(valuesReader, length); + } + } else { + switch (hiveType.getTypeName()) { + case serdeConstants.SMALLINT_TYPE_NAME: + return isDictionary ? new Types32Int2SmallintPageReader(dictionary, + length) : new Types32Int2SmallintPageReader(valuesReader, length); + case serdeConstants.TINYINT_TYPE_NAME: + return isDictionary ? new Types32Int2TinyintPageReader(dictionary, + length) : new Types32Int2TinyintPageReader(valuesReader, length); + default: + return isDictionary ? new TypesFromInt32PageReader(dictionary, + length) : new TypesFromInt32PageReader(valuesReader, length); + } } case INT64: - switch (hiveType.getTypeName()) { - case serdeConstants.INT_TYPE_NAME: - return isDictionary ? new Types64Int2IntPageReader(dictionary, - length) : new Types64Int2IntPageReader(valuesReader, length); - case serdeConstants.SMALLINT_TYPE_NAME: - return isDictionary ? new Types64Int2SmallintPageReader(dictionary, - length) : new Types64Int2SmallintPageReader(valuesReader, length); - case serdeConstants.TINYINT_TYPE_NAME: - return isDictionary ? new Types64Int2TinyintPageReader(dictionary, - length) : new Types64Int2TinyintPageReader(valuesReader, length); - default: - return isDictionary ? new TypesFromInt64PageReader(dictionary, - length) : new TypesFromInt64PageReader(valuesReader, length); + if (OriginalType.UINT_8 == parquetType.getOriginalType() || + OriginalType.UINT_16 == parquetType.getOriginalType() || + OriginalType.UINT_32 == parquetType.getOriginalType() || + OriginalType.UINT_64 == parquetType.getOriginalType()) { + switch (hiveType.getTypeName()) { + case serdeConstants.INT_TYPE_NAME: + return isDictionary ? new Types64UInt2IntPageReader(dictionary, + length) : new Types64UInt2IntPageReader(valuesReader, length); + case serdeConstants.SMALLINT_TYPE_NAME: + return isDictionary ? new Types64UInt2SmallintPageReader(dictionary, + length) : new Types64UInt2SmallintPageReader(valuesReader, length); + case serdeConstants.TINYINT_TYPE_NAME: + return isDictionary ? new Types64UInt2TinyintPageReader(dictionary, + length) : new Types64UInt2TinyintPageReader(valuesReader, length); + default: + return isDictionary ? new TypesFromUInt64PageReader(dictionary, + length) : new TypesFromUInt64PageReader(valuesReader, length); + } + } else { + switch (hiveType.getTypeName()) { + case serdeConstants.INT_TYPE_NAME: + return isDictionary ? new Types64Int2IntPageReader(dictionary, + length) : new Types64Int2IntPageReader(valuesReader, length); + case serdeConstants.SMALLINT_TYPE_NAME: + return isDictionary ? new Types64Int2SmallintPageReader(dictionary, + length) : new Types64Int2SmallintPageReader(valuesReader, length); + case serdeConstants.TINYINT_TYPE_NAME: + return isDictionary ? new Types64Int2TinyintPageReader(dictionary, + length) : new Types64Int2TinyintPageReader(valuesReader, length); + default: + return isDictionary ? new TypesFromInt64PageReader(dictionary, + length) : new TypesFromInt64PageReader(valuesReader, length); + } } case FLOAT: return isDictionary ? new TypesFromFloatPageReader(dictionary, length) : new diff --git ql/src/test/queries/clientpositive/read_uint_parquet.q ql/src/test/queries/clientpositive/read_uint_parquet.q new file mode 100644 index 0000000000000000000000000000000000000000..09b0b761f6d66361abda5f4bb91d6ae8af1b17fe --- /dev/null +++ ql/src/test/queries/clientpositive/read_uint_parquet.q @@ -0,0 +1,79 @@ +-- Disable vectorization +SET hive.vectorized.execution.enabled=false; + +create table testbasicint (uint_32_col int) stored as parquet; +load data local inpath '../../data/files/test_uint.parquet' into table testbasicint; +select * from testbasicint; +drop table testbasicint; + +create table testbigintinv +(col_INT32_UINT_8 bigint, + col_INT32_UINT_16 bigint, + col_INT32_UINT_32 bigint, + col_INT64_UINT_64 bigint) stored as parquet; +load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testbigintinv; +select * from testbigintinv; +drop table testbigintinv; + +create table testintinv +(col_INT32_UINT_8 int, + col_INT32_UINT_16 int, + col_INT32_UINT_32 int, + col_INT64_UINT_64 int) stored as parquet; +load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testintinv; +select * from testintinv; +drop table testintinv; + +create table testsmallintinv +(col_INT32_UINT_8 smallint, + col_INT32_UINT_16 smallint, + col_INT32_UINT_32 smallint, + col_INT64_UINT_64 smallint) stored as parquet; +load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testsmallintinv; +select * from testsmallintinv; +drop table testsmallintinv; + +create table testtinyintinv +(col_INT32_UINT_8 tinyint, + col_INT32_UINT_16 tinyint, + col_INT32_UINT_32 tinyint, + col_INT64_UINT_64 tinyint) stored as parquet; +load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testtinyintinv; +select * from testtinyintinv; +drop table testtinyintinv; + +create table testbigintvalid +(col_INT32_UINT_8 bigint, + col_INT32_UINT_16 bigint, + col_INT32_UINT_32 bigint, + col_INT64_UINT_64 bigint) stored as parquet; +load data local inpath '../../data/files/data_with_valid_values.parquet' into table testbigintvalid; +select * from testbigintvalid; +drop table testbigintvalid; + +create table testintvalid +(col_INT32_UINT_8 int, + col_INT32_UINT_16 int, + col_INT32_UINT_32 int, + col_INT64_UINT_64 int) stored as parquet; +load data local inpath '../../data/files/data_with_valid_values.parquet' into table testintvalid; +select * from testintvalid; +drop table testintvalid; + +create table testsmallintvalid +(col_INT32_UINT_8 smallint, + col_INT32_UINT_16 smallint, + col_INT32_UINT_32 smallint, + col_INT64_UINT_64 smallint) stored as parquet; +load data local inpath '../../data/files/data_with_valid_values.parquet' into table testsmallintvalid; +select * from testsmallintvalid; +drop table testsmallintvalid; + +create table testtinyintvalid +(col_INT32_UINT_8 tinyint, + col_INT32_UINT_16 tinyint, + col_INT32_UINT_32 tinyint, + col_INT64_UINT_64 tinyint) stored as parquet; +load data local inpath '../../data/files/data_with_valid_values.parquet' into table testtinyintvalid; +select * from testtinyintvalid; +drop table testtinyintvalid; diff --git ql/src/test/queries/clientpositive/read_uint_parquet_vectorized.q ql/src/test/queries/clientpositive/read_uint_parquet_vectorized.q new file mode 100644 index 0000000000000000000000000000000000000000..eee103f815f8d0b7253398cc9db046218d560bd2 --- /dev/null +++ ql/src/test/queries/clientpositive/read_uint_parquet_vectorized.q @@ -0,0 +1,79 @@ +-- Disable vectorization +SET hive.vectorized.execution.enabled=true; + +create table testbasicint (uint_32_col int) stored as parquet; +load data local inpath '../../data/files/test_uint.parquet' into table testbasicint; +select * from testbasicint; +drop table testbasicint; + +create table testbigintinv +(col_INT32_UINT_8 bigint, + col_INT32_UINT_16 bigint, + col_INT32_UINT_32 bigint, + col_INT64_UINT_64 bigint) stored as parquet; +load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testbigintinv; +select * from testbigintinv; +drop table testbigintinv; + +create table testintinv +(col_INT32_UINT_8 int, + col_INT32_UINT_16 int, + col_INT32_UINT_32 int, + col_INT64_UINT_64 int) stored as parquet; +load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testintinv; +select * from testintinv; +drop table testintinv; + +create table testsmallintinv +(col_INT32_UINT_8 smallint, + col_INT32_UINT_16 smallint, + col_INT32_UINT_32 smallint, + col_INT64_UINT_64 smallint) stored as parquet; +load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testsmallintinv; +select * from testsmallintinv; +drop table testsmallintinv; + +create table testtinyintinv +(col_INT32_UINT_8 tinyint, + col_INT32_UINT_16 tinyint, + col_INT32_UINT_32 tinyint, + col_INT64_UINT_64 tinyint) stored as parquet; +load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testtinyintinv; +select * from testtinyintinv; +drop table testtinyintinv; + +create table testbigintvalid +(col_INT32_UINT_8 bigint, + col_INT32_UINT_16 bigint, + col_INT32_UINT_32 bigint, + col_INT64_UINT_64 bigint) stored as parquet; +load data local inpath '../../data/files/data_with_valid_values.parquet' into table testbigintvalid; +select * from testbigintvalid; +drop table testbigintvalid; + +create table testintvalid +(col_INT32_UINT_8 int, + col_INT32_UINT_16 int, + col_INT32_UINT_32 int, + col_INT64_UINT_64 int) stored as parquet; +load data local inpath '../../data/files/data_with_valid_values.parquet' into table testintvalid; +select * from testintvalid; +drop table testintvalid; + +create table testsmallintvalid +(col_INT32_UINT_8 smallint, + col_INT32_UINT_16 smallint, + col_INT32_UINT_32 smallint, + col_INT64_UINT_64 smallint) stored as parquet; +load data local inpath '../../data/files/data_with_valid_values.parquet' into table testsmallintvalid; +select * from testsmallintvalid; +drop table testsmallintvalid; + +create table testtinyintvalid +(col_INT32_UINT_8 tinyint, + col_INT32_UINT_16 tinyint, + col_INT32_UINT_32 tinyint, + col_INT64_UINT_64 tinyint) stored as parquet; +load data local inpath '../../data/files/data_with_valid_values.parquet' into table testtinyintvalid; +select * from testtinyintvalid; +drop table testtinyintvalid; diff --git ql/src/test/results/clientpositive/read_uint_parquet.q.out ql/src/test/results/clientpositive/read_uint_parquet.q.out new file mode 100644 index 0000000000000000000000000000000000000000..b34f8d4c8a371c37ee8ae8abba489d6ab2e3abfb --- /dev/null +++ ql/src/test/results/clientpositive/read_uint_parquet.q.out @@ -0,0 +1,429 @@ +PREHOOK: query: create table testbasicint (uint_32_col int) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testbasicint +POSTHOOK: query: create table testbasicint (uint_32_col int) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testbasicint +PREHOOK: query: load data local inpath '../../data/files/test_uint.parquet' into table testbasicint +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testbasicint +POSTHOOK: query: load data local inpath '../../data/files/test_uint.parquet' into table testbasicint +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testbasicint +PREHOOK: query: select * from testbasicint +PREHOOK: type: QUERY +PREHOOK: Input: default@testbasicint +#### A masked pattern was here #### +POSTHOOK: query: select * from testbasicint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testbasicint +#### A masked pattern was here #### +0 +1 +2147483647 +NULL +NULL +PREHOOK: query: drop table testbasicint +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testbasicint +PREHOOK: Output: default@testbasicint +POSTHOOK: query: drop table testbasicint +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testbasicint +POSTHOOK: Output: default@testbasicint +PREHOOK: query: create table testbigintinv +(col_INT32_UINT_8 bigint, + col_INT32_UINT_16 bigint, + col_INT32_UINT_32 bigint, + col_INT64_UINT_64 bigint) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testbigintinv +POSTHOOK: query: create table testbigintinv +(col_INT32_UINT_8 bigint, + col_INT32_UINT_16 bigint, + col_INT32_UINT_32 bigint, + col_INT64_UINT_64 bigint) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testbigintinv +PREHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testbigintinv +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testbigintinv +POSTHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testbigintinv +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testbigintinv +PREHOOK: query: select * from testbigintinv +PREHOOK: type: QUERY +PREHOOK: Input: default@testbigintinv +#### A masked pattern was here #### +POSTHOOK: query: select * from testbigintinv +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testbigintinv +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +255 255 255 255 +32767 32767 32767 32767 +65535 65535 65535 65535 +2147483647 2147483647 2147483647 2147483647 +NULL NULL NULL 4294967295 +NULL NULL NULL 9223372036854775807 +NULL NULL NULL NULL +PREHOOK: query: drop table testbigintinv +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testbigintinv +PREHOOK: Output: default@testbigintinv +POSTHOOK: query: drop table testbigintinv +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testbigintinv +POSTHOOK: Output: default@testbigintinv +PREHOOK: query: create table testintinv +(col_INT32_UINT_8 int, + col_INT32_UINT_16 int, + col_INT32_UINT_32 int, + col_INT64_UINT_64 int) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testintinv +POSTHOOK: query: create table testintinv +(col_INT32_UINT_8 int, + col_INT32_UINT_16 int, + col_INT32_UINT_32 int, + col_INT64_UINT_64 int) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testintinv +PREHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testintinv +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testintinv +POSTHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testintinv +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testintinv +PREHOOK: query: select * from testintinv +PREHOOK: type: QUERY +PREHOOK: Input: default@testintinv +#### A masked pattern was here #### +POSTHOOK: query: select * from testintinv +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testintinv +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +255 255 255 255 +32767 32767 32767 32767 +65535 65535 65535 65535 +2147483647 2147483647 2147483647 2147483647 +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +PREHOOK: query: drop table testintinv +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testintinv +PREHOOK: Output: default@testintinv +POSTHOOK: query: drop table testintinv +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testintinv +POSTHOOK: Output: default@testintinv +PREHOOK: query: create table testsmallintinv +(col_INT32_UINT_8 smallint, + col_INT32_UINT_16 smallint, + col_INT32_UINT_32 smallint, + col_INT64_UINT_64 smallint) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testsmallintinv +POSTHOOK: query: create table testsmallintinv +(col_INT32_UINT_8 smallint, + col_INT32_UINT_16 smallint, + col_INT32_UINT_32 smallint, + col_INT64_UINT_64 smallint) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testsmallintinv +PREHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testsmallintinv +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testsmallintinv +POSTHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testsmallintinv +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testsmallintinv +PREHOOK: query: select * from testsmallintinv +PREHOOK: type: QUERY +PREHOOK: Input: default@testsmallintinv +#### A masked pattern was here #### +POSTHOOK: query: select * from testsmallintinv +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testsmallintinv +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +255 255 255 255 +32767 32767 32767 32767 +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +PREHOOK: query: drop table testsmallintinv +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testsmallintinv +PREHOOK: Output: default@testsmallintinv +POSTHOOK: query: drop table testsmallintinv +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testsmallintinv +POSTHOOK: Output: default@testsmallintinv +PREHOOK: query: create table testtinyintinv +(col_INT32_UINT_8 tinyint, + col_INT32_UINT_16 tinyint, + col_INT32_UINT_32 tinyint, + col_INT64_UINT_64 tinyint) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testtinyintinv +POSTHOOK: query: create table testtinyintinv +(col_INT32_UINT_8 tinyint, + col_INT32_UINT_16 tinyint, + col_INT32_UINT_32 tinyint, + col_INT64_UINT_64 tinyint) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testtinyintinv +PREHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testtinyintinv +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testtinyintinv +POSTHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testtinyintinv +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testtinyintinv +PREHOOK: query: select * from testtinyintinv +PREHOOK: type: QUERY +PREHOOK: Input: default@testtinyintinv +#### A masked pattern was here #### +POSTHOOK: query: select * from testtinyintinv +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testtinyintinv +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +PREHOOK: query: drop table testtinyintinv +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testtinyintinv +PREHOOK: Output: default@testtinyintinv +POSTHOOK: query: drop table testtinyintinv +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testtinyintinv +POSTHOOK: Output: default@testtinyintinv +PREHOOK: query: create table testbigintvalid +(col_INT32_UINT_8 bigint, + col_INT32_UINT_16 bigint, + col_INT32_UINT_32 bigint, + col_INT64_UINT_64 bigint) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testbigintvalid +POSTHOOK: query: create table testbigintvalid +(col_INT32_UINT_8 bigint, + col_INT32_UINT_16 bigint, + col_INT32_UINT_32 bigint, + col_INT64_UINT_64 bigint) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testbigintvalid +PREHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testbigintvalid +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testbigintvalid +POSTHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testbigintvalid +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testbigintvalid +PREHOOK: query: select * from testbigintvalid +PREHOOK: type: QUERY +PREHOOK: Input: default@testbigintvalid +#### A masked pattern was here #### +POSTHOOK: query: select * from testbigintvalid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testbigintvalid +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +255 255 255 255 +NULL 32767 32767 32767 +NULL 65535 65535 65535 +NULL NULL 2147483647 2147483647 +NULL NULL NULL 4294967295 +NULL NULL NULL 9223372036854775807 +NULL NULL NULL NULL +PREHOOK: query: drop table testbigintvalid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testbigintvalid +PREHOOK: Output: default@testbigintvalid +POSTHOOK: query: drop table testbigintvalid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testbigintvalid +POSTHOOK: Output: default@testbigintvalid +PREHOOK: query: create table testintvalid +(col_INT32_UINT_8 int, + col_INT32_UINT_16 int, + col_INT32_UINT_32 int, + col_INT64_UINT_64 int) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testintvalid +POSTHOOK: query: create table testintvalid +(col_INT32_UINT_8 int, + col_INT32_UINT_16 int, + col_INT32_UINT_32 int, + col_INT64_UINT_64 int) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testintvalid +PREHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testintvalid +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testintvalid +POSTHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testintvalid +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testintvalid +PREHOOK: query: select * from testintvalid +PREHOOK: type: QUERY +PREHOOK: Input: default@testintvalid +#### A masked pattern was here #### +POSTHOOK: query: select * from testintvalid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testintvalid +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +255 255 255 255 +NULL 32767 32767 32767 +NULL 65535 65535 65535 +NULL NULL 2147483647 2147483647 +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +PREHOOK: query: drop table testintvalid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testintvalid +PREHOOK: Output: default@testintvalid +POSTHOOK: query: drop table testintvalid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testintvalid +POSTHOOK: Output: default@testintvalid +PREHOOK: query: create table testsmallintvalid +(col_INT32_UINT_8 smallint, + col_INT32_UINT_16 smallint, + col_INT32_UINT_32 smallint, + col_INT64_UINT_64 smallint) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testsmallintvalid +POSTHOOK: query: create table testsmallintvalid +(col_INT32_UINT_8 smallint, + col_INT32_UINT_16 smallint, + col_INT32_UINT_32 smallint, + col_INT64_UINT_64 smallint) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testsmallintvalid +PREHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testsmallintvalid +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testsmallintvalid +POSTHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testsmallintvalid +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testsmallintvalid +PREHOOK: query: select * from testsmallintvalid +PREHOOK: type: QUERY +PREHOOK: Input: default@testsmallintvalid +#### A masked pattern was here #### +POSTHOOK: query: select * from testsmallintvalid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testsmallintvalid +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +255 255 255 255 +NULL 32767 32767 32767 +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +PREHOOK: query: drop table testsmallintvalid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testsmallintvalid +PREHOOK: Output: default@testsmallintvalid +POSTHOOK: query: drop table testsmallintvalid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testsmallintvalid +POSTHOOK: Output: default@testsmallintvalid +PREHOOK: query: create table testtinyintvalid +(col_INT32_UINT_8 tinyint, + col_INT32_UINT_16 tinyint, + col_INT32_UINT_32 tinyint, + col_INT64_UINT_64 tinyint) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testtinyintvalid +POSTHOOK: query: create table testtinyintvalid +(col_INT32_UINT_8 tinyint, + col_INT32_UINT_16 tinyint, + col_INT32_UINT_32 tinyint, + col_INT64_UINT_64 tinyint) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testtinyintvalid +PREHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testtinyintvalid +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testtinyintvalid +POSTHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testtinyintvalid +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testtinyintvalid +PREHOOK: query: select * from testtinyintvalid +PREHOOK: type: QUERY +PREHOOK: Input: default@testtinyintvalid +#### A masked pattern was here #### +POSTHOOK: query: select * from testtinyintvalid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testtinyintvalid +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +PREHOOK: query: drop table testtinyintvalid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testtinyintvalid +PREHOOK: Output: default@testtinyintvalid +POSTHOOK: query: drop table testtinyintvalid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testtinyintvalid +POSTHOOK: Output: default@testtinyintvalid diff --git ql/src/test/results/clientpositive/read_uint_parquet_vectorized.q.out ql/src/test/results/clientpositive/read_uint_parquet_vectorized.q.out new file mode 100644 index 0000000000000000000000000000000000000000..b34f8d4c8a371c37ee8ae8abba489d6ab2e3abfb --- /dev/null +++ ql/src/test/results/clientpositive/read_uint_parquet_vectorized.q.out @@ -0,0 +1,429 @@ +PREHOOK: query: create table testbasicint (uint_32_col int) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testbasicint +POSTHOOK: query: create table testbasicint (uint_32_col int) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testbasicint +PREHOOK: query: load data local inpath '../../data/files/test_uint.parquet' into table testbasicint +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testbasicint +POSTHOOK: query: load data local inpath '../../data/files/test_uint.parquet' into table testbasicint +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testbasicint +PREHOOK: query: select * from testbasicint +PREHOOK: type: QUERY +PREHOOK: Input: default@testbasicint +#### A masked pattern was here #### +POSTHOOK: query: select * from testbasicint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testbasicint +#### A masked pattern was here #### +0 +1 +2147483647 +NULL +NULL +PREHOOK: query: drop table testbasicint +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testbasicint +PREHOOK: Output: default@testbasicint +POSTHOOK: query: drop table testbasicint +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testbasicint +POSTHOOK: Output: default@testbasicint +PREHOOK: query: create table testbigintinv +(col_INT32_UINT_8 bigint, + col_INT32_UINT_16 bigint, + col_INT32_UINT_32 bigint, + col_INT64_UINT_64 bigint) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testbigintinv +POSTHOOK: query: create table testbigintinv +(col_INT32_UINT_8 bigint, + col_INT32_UINT_16 bigint, + col_INT32_UINT_32 bigint, + col_INT64_UINT_64 bigint) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testbigintinv +PREHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testbigintinv +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testbigintinv +POSTHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testbigintinv +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testbigintinv +PREHOOK: query: select * from testbigintinv +PREHOOK: type: QUERY +PREHOOK: Input: default@testbigintinv +#### A masked pattern was here #### +POSTHOOK: query: select * from testbigintinv +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testbigintinv +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +255 255 255 255 +32767 32767 32767 32767 +65535 65535 65535 65535 +2147483647 2147483647 2147483647 2147483647 +NULL NULL NULL 4294967295 +NULL NULL NULL 9223372036854775807 +NULL NULL NULL NULL +PREHOOK: query: drop table testbigintinv +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testbigintinv +PREHOOK: Output: default@testbigintinv +POSTHOOK: query: drop table testbigintinv +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testbigintinv +POSTHOOK: Output: default@testbigintinv +PREHOOK: query: create table testintinv +(col_INT32_UINT_8 int, + col_INT32_UINT_16 int, + col_INT32_UINT_32 int, + col_INT64_UINT_64 int) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testintinv +POSTHOOK: query: create table testintinv +(col_INT32_UINT_8 int, + col_INT32_UINT_16 int, + col_INT32_UINT_32 int, + col_INT64_UINT_64 int) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testintinv +PREHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testintinv +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testintinv +POSTHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testintinv +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testintinv +PREHOOK: query: select * from testintinv +PREHOOK: type: QUERY +PREHOOK: Input: default@testintinv +#### A masked pattern was here #### +POSTHOOK: query: select * from testintinv +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testintinv +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +255 255 255 255 +32767 32767 32767 32767 +65535 65535 65535 65535 +2147483647 2147483647 2147483647 2147483647 +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +PREHOOK: query: drop table testintinv +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testintinv +PREHOOK: Output: default@testintinv +POSTHOOK: query: drop table testintinv +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testintinv +POSTHOOK: Output: default@testintinv +PREHOOK: query: create table testsmallintinv +(col_INT32_UINT_8 smallint, + col_INT32_UINT_16 smallint, + col_INT32_UINT_32 smallint, + col_INT64_UINT_64 smallint) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testsmallintinv +POSTHOOK: query: create table testsmallintinv +(col_INT32_UINT_8 smallint, + col_INT32_UINT_16 smallint, + col_INT32_UINT_32 smallint, + col_INT64_UINT_64 smallint) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testsmallintinv +PREHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testsmallintinv +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testsmallintinv +POSTHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testsmallintinv +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testsmallintinv +PREHOOK: query: select * from testsmallintinv +PREHOOK: type: QUERY +PREHOOK: Input: default@testsmallintinv +#### A masked pattern was here #### +POSTHOOK: query: select * from testsmallintinv +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testsmallintinv +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +255 255 255 255 +32767 32767 32767 32767 +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +PREHOOK: query: drop table testsmallintinv +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testsmallintinv +PREHOOK: Output: default@testsmallintinv +POSTHOOK: query: drop table testsmallintinv +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testsmallintinv +POSTHOOK: Output: default@testsmallintinv +PREHOOK: query: create table testtinyintinv +(col_INT32_UINT_8 tinyint, + col_INT32_UINT_16 tinyint, + col_INT32_UINT_32 tinyint, + col_INT64_UINT_64 tinyint) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testtinyintinv +POSTHOOK: query: create table testtinyintinv +(col_INT32_UINT_8 tinyint, + col_INT32_UINT_16 tinyint, + col_INT32_UINT_32 tinyint, + col_INT64_UINT_64 tinyint) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testtinyintinv +PREHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testtinyintinv +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testtinyintinv +POSTHOOK: query: load data local inpath '../../data/files/data_including_invalid_values.parquet' into table testtinyintinv +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testtinyintinv +PREHOOK: query: select * from testtinyintinv +PREHOOK: type: QUERY +PREHOOK: Input: default@testtinyintinv +#### A masked pattern was here #### +POSTHOOK: query: select * from testtinyintinv +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testtinyintinv +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +PREHOOK: query: drop table testtinyintinv +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testtinyintinv +PREHOOK: Output: default@testtinyintinv +POSTHOOK: query: drop table testtinyintinv +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testtinyintinv +POSTHOOK: Output: default@testtinyintinv +PREHOOK: query: create table testbigintvalid +(col_INT32_UINT_8 bigint, + col_INT32_UINT_16 bigint, + col_INT32_UINT_32 bigint, + col_INT64_UINT_64 bigint) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testbigintvalid +POSTHOOK: query: create table testbigintvalid +(col_INT32_UINT_8 bigint, + col_INT32_UINT_16 bigint, + col_INT32_UINT_32 bigint, + col_INT64_UINT_64 bigint) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testbigintvalid +PREHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testbigintvalid +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testbigintvalid +POSTHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testbigintvalid +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testbigintvalid +PREHOOK: query: select * from testbigintvalid +PREHOOK: type: QUERY +PREHOOK: Input: default@testbigintvalid +#### A masked pattern was here #### +POSTHOOK: query: select * from testbigintvalid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testbigintvalid +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +255 255 255 255 +NULL 32767 32767 32767 +NULL 65535 65535 65535 +NULL NULL 2147483647 2147483647 +NULL NULL NULL 4294967295 +NULL NULL NULL 9223372036854775807 +NULL NULL NULL NULL +PREHOOK: query: drop table testbigintvalid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testbigintvalid +PREHOOK: Output: default@testbigintvalid +POSTHOOK: query: drop table testbigintvalid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testbigintvalid +POSTHOOK: Output: default@testbigintvalid +PREHOOK: query: create table testintvalid +(col_INT32_UINT_8 int, + col_INT32_UINT_16 int, + col_INT32_UINT_32 int, + col_INT64_UINT_64 int) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testintvalid +POSTHOOK: query: create table testintvalid +(col_INT32_UINT_8 int, + col_INT32_UINT_16 int, + col_INT32_UINT_32 int, + col_INT64_UINT_64 int) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testintvalid +PREHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testintvalid +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testintvalid +POSTHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testintvalid +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testintvalid +PREHOOK: query: select * from testintvalid +PREHOOK: type: QUERY +PREHOOK: Input: default@testintvalid +#### A masked pattern was here #### +POSTHOOK: query: select * from testintvalid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testintvalid +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +255 255 255 255 +NULL 32767 32767 32767 +NULL 65535 65535 65535 +NULL NULL 2147483647 2147483647 +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +PREHOOK: query: drop table testintvalid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testintvalid +PREHOOK: Output: default@testintvalid +POSTHOOK: query: drop table testintvalid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testintvalid +POSTHOOK: Output: default@testintvalid +PREHOOK: query: create table testsmallintvalid +(col_INT32_UINT_8 smallint, + col_INT32_UINT_16 smallint, + col_INT32_UINT_32 smallint, + col_INT64_UINT_64 smallint) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testsmallintvalid +POSTHOOK: query: create table testsmallintvalid +(col_INT32_UINT_8 smallint, + col_INT32_UINT_16 smallint, + col_INT32_UINT_32 smallint, + col_INT64_UINT_64 smallint) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testsmallintvalid +PREHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testsmallintvalid +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testsmallintvalid +POSTHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testsmallintvalid +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testsmallintvalid +PREHOOK: query: select * from testsmallintvalid +PREHOOK: type: QUERY +PREHOOK: Input: default@testsmallintvalid +#### A masked pattern was here #### +POSTHOOK: query: select * from testsmallintvalid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testsmallintvalid +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +255 255 255 255 +NULL 32767 32767 32767 +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +PREHOOK: query: drop table testsmallintvalid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testsmallintvalid +PREHOOK: Output: default@testsmallintvalid +POSTHOOK: query: drop table testsmallintvalid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testsmallintvalid +POSTHOOK: Output: default@testsmallintvalid +PREHOOK: query: create table testtinyintvalid +(col_INT32_UINT_8 tinyint, + col_INT32_UINT_16 tinyint, + col_INT32_UINT_32 tinyint, + col_INT64_UINT_64 tinyint) stored as parquet +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testtinyintvalid +POSTHOOK: query: create table testtinyintvalid +(col_INT32_UINT_8 tinyint, + col_INT32_UINT_16 tinyint, + col_INT32_UINT_32 tinyint, + col_INT64_UINT_64 tinyint) stored as parquet +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testtinyintvalid +PREHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testtinyintvalid +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@testtinyintvalid +POSTHOOK: query: load data local inpath '../../data/files/data_with_valid_values.parquet' into table testtinyintvalid +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@testtinyintvalid +PREHOOK: query: select * from testtinyintvalid +PREHOOK: type: QUERY +PREHOOK: Input: default@testtinyintvalid +#### A masked pattern was here #### +POSTHOOK: query: select * from testtinyintvalid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testtinyintvalid +#### A masked pattern was here #### +0 0 0 0 +127 127 127 127 +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +NULL NULL NULL NULL +PREHOOK: query: drop table testtinyintvalid +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testtinyintvalid +PREHOOK: Output: default@testtinyintvalid +POSTHOOK: query: drop table testtinyintvalid +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testtinyintvalid +POSTHOOK: Output: default@testtinyintvalid