diff --git ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out index 6632bfb..1fe9a13 100644 --- ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out +++ ql/src/test/results/clientpositive/llap/schema_evol_text_vec_part.q.out @@ -1,23 +1,9 @@ -PREHOOK: query: -- SORT_QUERY_RESULTS --- --- FILE VARIATION: TEXTFILE, Non-Vectorized, MapWork, Partitioned --- NOTE: the use of hive.vectorized.use.vector.serde.deserialize above which enables doing --- vectorized reading of TEXTFILE format files using the vector SERDE methods. --- - -CREATE TABLE schema_evolution_data(insert_num int, boolean1 boolean, tinyint1 tinyint, smallint1 smallint, int1 int, bigint1 bigint, decimal1 decimal(38,18), float1 float, double1 double, string1 string, string2 string, date1 date, timestamp1 timestamp, boolean_str string, tinyint_str string, smallint_str string, int_str string, bigint_str string, decimal_str string, float_str string, double_str string, date_str string, timestamp_str string, filler string) +PREHOOK: query: CREATE TABLE schema_evolution_data(insert_num int, boolean1 boolean, tinyint1 tinyint, smallint1 smallint, int1 int, bigint1 bigint, decimal1 decimal(38,18), float1 float, double1 double, string1 string, string2 string, date1 date, timestamp1 timestamp, boolean_str string, tinyint_str string, smallint_str string, int_str string, bigint_str string, decimal_str string, float_str string, double_str string, date_str string, timestamp_str string, filler string) row format delimited fields terminated by '|' stored as textfile PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@schema_evolution_data -POSTHOOK: query: -- SORT_QUERY_RESULTS --- --- FILE VARIATION: TEXTFILE, Non-Vectorized, MapWork, Partitioned --- NOTE: the use of hive.vectorized.use.vector.serde.deserialize above which enables doing --- vectorized reading of TEXTFILE format files using the vector SERDE methods. --- - -CREATE TABLE schema_evolution_data(insert_num int, boolean1 boolean, tinyint1 tinyint, smallint1 smallint, int1 int, bigint1 bigint, decimal1 decimal(38,18), float1 float, double1 double, string1 string, string2 string, date1 date, timestamp1 timestamp, boolean_str string, tinyint_str string, smallint_str string, int_str string, bigint_str string, decimal_str string, float_str string, double_str string, date_str string, timestamp_str string, filler string) +POSTHOOK: query: CREATE TABLE schema_evolution_data(insert_num int, boolean1 boolean, tinyint1 tinyint, smallint1 smallint, int1 int, bigint1 bigint, decimal1 decimal(38,18), float1 float, double1 double, string1 string, string2 string, date1 date, timestamp1 timestamp, boolean_str string, tinyint_str string, smallint_str string, int_str string, bigint_str string, decimal_str string, float_str string, double_str string, date_str string, timestamp_str string, filler string) row format delimited fields terminated by '|' stored as textfile POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default @@ -30,25 +16,11 @@ POSTHOOK: query: load data local inpath '../../data/files/schema_evolution/schem POSTHOOK: type: LOAD #### A masked pattern was here #### POSTHOOK: Output: default@schema_evolution_data -PREHOOK: query: ------------------------------------------------------------------------------------------ --- SECTION: ALTER TABLE ADD COLUMNS --- --- --- SUBSECTION: ALTER TABLE ADD COLUMNS: INT PERMUTE SELECT --- --- -CREATE TABLE part_add_int_permute_select(insert_num int, a INT, b STRING) PARTITIONED BY(part INT) +PREHOOK: query: CREATE TABLE part_add_int_permute_select(insert_num int, a INT, b STRING) PARTITIONED BY(part INT) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@part_add_int_permute_select -POSTHOOK: query: ------------------------------------------------------------------------------------------ --- SECTION: ALTER TABLE ADD COLUMNS --- --- --- SUBSECTION: ALTER TABLE ADD COLUMNS: INT PERMUTE SELECT --- --- -CREATE TABLE part_add_int_permute_select(insert_num int, a INT, b STRING) PARTITIONED BY(part INT) +POSTHOOK: query: CREATE TABLE part_add_int_permute_select(insert_num int, a INT, b STRING) PARTITIONED BY(part INT) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@part_add_int_permute_select @@ -62,13 +34,11 @@ POSTHOOK: Lineage: part_add_int_permute_select PARTITION(part=1).a EXPRESSION [( POSTHOOK: Lineage: part_add_int_permute_select PARTITION(part=1).b SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ] POSTHOOK: Lineage: part_add_int_permute_select PARTITION(part=1).insert_num EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] _col0 _col1 _col2 -PREHOOK: query: -- Table-Non-Cascade ADD COLUMNS ... -alter table part_add_int_permute_select add columns(c int) +PREHOOK: query: alter table part_add_int_permute_select add columns(c int) PREHOOK: type: ALTERTABLE_ADDCOLS PREHOOK: Input: default@part_add_int_permute_select PREHOOK: Output: default@part_add_int_permute_select -POSTHOOK: query: -- Table-Non-Cascade ADD COLUMNS ... -alter table part_add_int_permute_select add columns(c int) +POSTHOOK: query: alter table part_add_int_permute_select add columns(c int) POSTHOOK: type: ALTERTABLE_ADDCOLS POSTHOOK: Input: default@part_add_int_permute_select POSTHOOK: Output: default@part_add_int_permute_select @@ -103,14 +73,12 @@ Stage-0 TableScan [TS_0] (rows=2 width=16) default@part_add_int_permute_select,part_add_int_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] -PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right -select insert_num,part,a,b from part_add_int_permute_select +PREHOOK: query: select insert_num,part,a,b from part_add_int_permute_select PREHOOK: type: QUERY PREHOOK: Input: default@part_add_int_permute_select PREHOOK: Input: default@part_add_int_permute_select@part=1 #### A masked pattern was here #### -POSTHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right -select insert_num,part,a,b from part_add_int_permute_select +POSTHOOK: query: select insert_num,part,a,b from part_add_int_permute_select POSTHOOK: type: QUERY POSTHOOK: Input: default@part_add_int_permute_select POSTHOOK: Input: default@part_add_int_permute_select@part=1 @@ -152,17 +120,11 @@ POSTHOOK: query: drop table part_add_int_permute_select POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_int_permute_select POSTHOOK: Output: default@part_add_int_permute_select -PREHOOK: query: -- SUBSECTION: ALTER TABLE ADD COLUMNS: INT, STRING, PERMUTE SELECT --- --- -CREATE TABLE part_add_int_string_permute_select(insert_num int, a INT, b STRING) PARTITIONED BY(part INT) +PREHOOK: query: CREATE TABLE part_add_int_string_permute_select(insert_num int, a INT, b STRING) PARTITIONED BY(part INT) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@part_add_int_string_permute_select -POSTHOOK: query: -- SUBSECTION: ALTER TABLE ADD COLUMNS: INT, STRING, PERMUTE SELECT --- --- -CREATE TABLE part_add_int_string_permute_select(insert_num int, a INT, b STRING) PARTITIONED BY(part INT) +POSTHOOK: query: CREATE TABLE part_add_int_string_permute_select(insert_num int, a INT, b STRING) PARTITIONED BY(part INT) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@part_add_int_string_permute_select @@ -176,13 +138,11 @@ POSTHOOK: Lineage: part_add_int_string_permute_select PARTITION(part=1).a EXPRES POSTHOOK: Lineage: part_add_int_string_permute_select PARTITION(part=1).b SIMPLE [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col3, type:string, comment:), ] POSTHOOK: Lineage: part_add_int_string_permute_select PARTITION(part=1).insert_num EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] _col0 _col1 _col2 -PREHOOK: query: -- Table-Non-Cascade ADD COLUMNS ... -alter table part_add_int_string_permute_select add columns(c int, d string) +PREHOOK: query: alter table part_add_int_string_permute_select add columns(c int, d string) PREHOOK: type: ALTERTABLE_ADDCOLS PREHOOK: Input: default@part_add_int_string_permute_select PREHOOK: Output: default@part_add_int_string_permute_select -POSTHOOK: query: -- Table-Non-Cascade ADD COLUMNS ... -alter table part_add_int_string_permute_select add columns(c int, d string) +POSTHOOK: query: alter table part_add_int_string_permute_select add columns(c int, d string) POSTHOOK: type: ALTERTABLE_ADDCOLS POSTHOOK: Input: default@part_add_int_string_permute_select POSTHOOK: Output: default@part_add_int_string_permute_select @@ -218,14 +178,12 @@ Stage-0 TableScan [TS_0] (rows=2 width=19) default@part_add_int_string_permute_select,part_add_int_string_permute_select,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","a","b"] -PREHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right -select insert_num,part,a,b from part_add_int_string_permute_select +PREHOOK: query: select insert_num,part,a,b from part_add_int_string_permute_select PREHOOK: type: QUERY PREHOOK: Input: default@part_add_int_string_permute_select PREHOOK: Input: default@part_add_int_string_permute_select@part=1 #### A masked pattern was here #### -POSTHOOK: query: -- SELECT permutation columns to make sure NULL defaulting works right -select insert_num,part,a,b from part_add_int_string_permute_select +POSTHOOK: query: select insert_num,part,a,b from part_add_int_string_permute_select POSTHOOK: type: QUERY POSTHOOK: Input: default@part_add_int_string_permute_select POSTHOOK: Input: default@part_add_int_string_permute_select@part=1 @@ -319,25 +277,11 @@ POSTHOOK: query: drop table part_add_int_string_permute_select POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_add_int_string_permute_select POSTHOOK: Output: default@part_add_int_string_permute_select -PREHOOK: query: ------------------------------------------------------------------------------------------ --- SECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> DOUBLE --- --- --- --- SUBSECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> DOUBLE: (STRING, CHAR, VARCHAR) --- -CREATE TABLE part_change_string_group_double(insert_num int, c1 STRING, c2 CHAR(50), c3 VARCHAR(50), b STRING) PARTITIONED BY(part INT) +PREHOOK: query: CREATE TABLE part_change_string_group_double(insert_num int, c1 STRING, c2 CHAR(50), c3 VARCHAR(50), b STRING) PARTITIONED BY(part INT) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@part_change_string_group_double -POSTHOOK: query: ------------------------------------------------------------------------------------------ --- SECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> DOUBLE --- --- --- --- SUBSECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> DOUBLE: (STRING, CHAR, VARCHAR) --- -CREATE TABLE part_change_string_group_double(insert_num int, c1 STRING, c2 CHAR(50), c3 VARCHAR(50), b STRING) PARTITIONED BY(part INT) +POSTHOOK: query: CREATE TABLE part_change_string_group_double(insert_num int, c1 STRING, c2 CHAR(50), c3 VARCHAR(50), b STRING) PARTITIONED BY(part INT) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@part_change_string_group_double @@ -355,13 +299,11 @@ POSTHOOK: Lineage: part_change_string_group_double PARTITION(part=1).c2 EXPRESSI POSTHOOK: Lineage: part_change_string_group_double PARTITION(part=1).c3 EXPRESSION [(schema_evolution_data)schema_evolution_data.FieldSchema(name:double_str, type:string, comment:null), ] POSTHOOK: Lineage: part_change_string_group_double PARTITION(part=1).insert_num SIMPLE [(schema_evolution_data)schema_evolution_data.FieldSchema(name:insert_num, type:int, comment:null), ] _col0 _col1 _col2 _col3 _col4 -PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... -alter table part_change_string_group_double replace columns (insert_num int, c1 DOUBLE, c2 DOUBLE, c3 DOUBLE, b STRING) +PREHOOK: query: alter table part_change_string_group_double replace columns (insert_num int, c1 DOUBLE, c2 DOUBLE, c3 DOUBLE, b STRING) PREHOOK: type: ALTERTABLE_REPLACECOLS PREHOOK: Input: default@part_change_string_group_double PREHOOK: Output: default@part_change_string_group_double -POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... -alter table part_change_string_group_double replace columns (insert_num int, c1 DOUBLE, c2 DOUBLE, c3 DOUBLE, b STRING) +POSTHOOK: query: alter table part_change_string_group_double replace columns (insert_num int, c1 DOUBLE, c2 DOUBLE, c3 DOUBLE, b STRING) POSTHOOK: type: ALTERTABLE_REPLACECOLS POSTHOOK: Input: default@part_change_string_group_double POSTHOOK: Output: default@part_change_string_group_double @@ -396,7 +338,7 @@ Stage-0 File Output Operator [FS_4] Select Operator [SEL_3] (rows=5 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5"] - TableScan [TS_0] (rows=5 width=56) + TableScan [TS_0] (rows=5 width=94) default@part_change_string_group_double,part_change_string_group_double,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","b"] PREHOOK: query: select insert_num,part,c1,c2,c3,b from part_change_string_group_double @@ -423,25 +365,11 @@ POSTHOOK: query: drop table part_change_string_group_double POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_change_string_group_double POSTHOOK: Output: default@part_change_string_group_double -PREHOOK: query: ------------------------------------------------------------------------------------------ --- SECTION: ALTER TABLE CHANGE COLUMNS for DATE_GROUP -> STRING_GROUP --- --- --- --- SUBSECTION: ALTER TABLE CHANGE COLUMNS for DATE_GROUP -> STRING_GROUP: DATE,TIMESTAMP, (STRING, CHAR, CHAR trunc, VARCHAR, VARCHAR trunc) --- -CREATE TABLE part_change_date_group_string_group_date_timestamp(insert_num int, c1 DATE, c2 DATE, c3 DATE, c4 DATE, c5 DATE, c6 TIMESTAMP, c7 TIMESTAMP, c8 TIMESTAMP, c9 TIMESTAMP, c10 TIMESTAMP, b STRING) PARTITIONED BY(part INT) +PREHOOK: query: CREATE TABLE part_change_date_group_string_group_date_timestamp(insert_num int, c1 DATE, c2 DATE, c3 DATE, c4 DATE, c5 DATE, c6 TIMESTAMP, c7 TIMESTAMP, c8 TIMESTAMP, c9 TIMESTAMP, c10 TIMESTAMP, b STRING) PARTITIONED BY(part INT) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@part_change_date_group_string_group_date_timestamp -POSTHOOK: query: ------------------------------------------------------------------------------------------ --- SECTION: ALTER TABLE CHANGE COLUMNS for DATE_GROUP -> STRING_GROUP --- --- --- --- SUBSECTION: ALTER TABLE CHANGE COLUMNS for DATE_GROUP -> STRING_GROUP: DATE,TIMESTAMP, (STRING, CHAR, CHAR trunc, VARCHAR, VARCHAR trunc) --- -CREATE TABLE part_change_date_group_string_group_date_timestamp(insert_num int, c1 DATE, c2 DATE, c3 DATE, c4 DATE, c5 DATE, c6 TIMESTAMP, c7 TIMESTAMP, c8 TIMESTAMP, c9 TIMESTAMP, c10 TIMESTAMP, b STRING) PARTITIONED BY(part INT) +POSTHOOK: query: CREATE TABLE part_change_date_group_string_group_date_timestamp(insert_num int, c1 DATE, c2 DATE, c3 DATE, c4 DATE, c5 DATE, c6 TIMESTAMP, c7 TIMESTAMP, c8 TIMESTAMP, c9 TIMESTAMP, c10 TIMESTAMP, b STRING) PARTITIONED BY(part INT) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default POSTHOOK: Output: default@part_change_date_group_string_group_date_timestamp @@ -466,13 +394,11 @@ POSTHOOK: Lineage: part_change_date_group_string_group_date_timestamp PARTITION( POSTHOOK: Lineage: part_change_date_group_string_group_date_timestamp PARTITION(part=1).c9 SIMPLE [(schema_evolution_data)schema_evolution_data.FieldSchema(name:timestamp1, type:timestamp, comment:null), ] POSTHOOK: Lineage: part_change_date_group_string_group_date_timestamp PARTITION(part=1).insert_num SIMPLE [(schema_evolution_data)schema_evolution_data.FieldSchema(name:insert_num, type:int, comment:null), ] insert_num date1 date1 date1 date1 date1 timestamp1 timestamp1 timestamp1 timestamp1 timestamp1 _c11 -PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... -alter table part_change_date_group_string_group_date_timestamp replace columns(insert_num int, c1 STRING, c2 CHAR(50), c3 CHAR(15), c4 VARCHAR(50), c5 VARCHAR(15), c6 STRING, c7 CHAR(50), c8 CHAR(15), c9 VARCHAR(50), c10 VARCHAR(15), b STRING) +PREHOOK: query: alter table part_change_date_group_string_group_date_timestamp replace columns(insert_num int, c1 STRING, c2 CHAR(50), c3 CHAR(15), c4 VARCHAR(50), c5 VARCHAR(15), c6 STRING, c7 CHAR(50), c8 CHAR(15), c9 VARCHAR(50), c10 VARCHAR(15), b STRING) PREHOOK: type: ALTERTABLE_REPLACECOLS PREHOOK: Input: default@part_change_date_group_string_group_date_timestamp PREHOOK: Output: default@part_change_date_group_string_group_date_timestamp -POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... -alter table part_change_date_group_string_group_date_timestamp replace columns(insert_num int, c1 STRING, c2 CHAR(50), c3 CHAR(15), c4 VARCHAR(50), c5 VARCHAR(15), c6 STRING, c7 CHAR(50), c8 CHAR(15), c9 VARCHAR(50), c10 VARCHAR(15), b STRING) +POSTHOOK: query: alter table part_change_date_group_string_group_date_timestamp replace columns(insert_num int, c1 STRING, c2 CHAR(50), c3 CHAR(15), c4 VARCHAR(50), c5 VARCHAR(15), c6 STRING, c7 CHAR(50), c8 CHAR(15), c9 VARCHAR(50), c10 VARCHAR(15), b STRING) POSTHOOK: type: ALTERTABLE_REPLACECOLS POSTHOOK: Input: default@part_change_date_group_string_group_date_timestamp POSTHOOK: Output: default@part_change_date_group_string_group_date_timestamp @@ -512,7 +438,7 @@ Stage-0 File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=154) + TableScan [TS_0] (rows=6 width=172) default@part_change_date_group_string_group_date_timestamp,part_change_date_group_string_group_date_timestamp,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_date_group_string_group_date_timestamp @@ -540,18 +466,7 @@ POSTHOOK: query: drop table part_change_date_group_string_group_date_timestamp POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_change_date_group_string_group_date_timestamp POSTHOOK: Output: default@part_change_date_group_string_group_date_timestamp -PREHOOK: query: ------------------------------------------------------------------------------------------ --- SECTION: ALTER TABLE CHANGE COLUMNS for NUMERIC_GROUP -> STRING_GROUP --- --- --- --- SUBSECTION: ALTER TABLE CHANGE COLUMNS for NUMERIC_GROUP -> STRING_GROUP: --- (TINYINT, SMALLINT, INT, BIGINT), STRING and --- (TINYINT, SMALLINT, INT, BIGINT), CHAR and CHAR trunc and --- (TINYINT, SMALLINT, INT, BIGINT), VARCHAR and VARCHAR trunc --- --- -CREATE TABLE part_change_numeric_group_string_group_multi_ints_string_group(insert_num int, +PREHOOK: query: CREATE TABLE part_change_numeric_group_string_group_multi_ints_string_group(insert_num int, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 tinyint, c6 smallint, c7 int, c8 bigint, c9 tinyint, c10 smallint, c11 int, c12 bigint, c13 tinyint, c14 smallint, c15 int, c16 bigint, c17 tinyint, c18 smallint, c19 int, c20 bigint, @@ -559,18 +474,7 @@ CREATE TABLE part_change_numeric_group_string_group_multi_ints_string_group(inse PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@part_change_numeric_group_string_group_multi_ints_string_group -POSTHOOK: query: ------------------------------------------------------------------------------------------ --- SECTION: ALTER TABLE CHANGE COLUMNS for NUMERIC_GROUP -> STRING_GROUP --- --- --- --- SUBSECTION: ALTER TABLE CHANGE COLUMNS for NUMERIC_GROUP -> STRING_GROUP: --- (TINYINT, SMALLINT, INT, BIGINT), STRING and --- (TINYINT, SMALLINT, INT, BIGINT), CHAR and CHAR trunc and --- (TINYINT, SMALLINT, INT, BIGINT), VARCHAR and VARCHAR trunc --- --- -CREATE TABLE part_change_numeric_group_string_group_multi_ints_string_group(insert_num int, +POSTHOOK: query: CREATE TABLE part_change_numeric_group_string_group_multi_ints_string_group(insert_num int, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 tinyint, c6 smallint, c7 int, c8 bigint, c9 tinyint, c10 smallint, c11 int, c12 bigint, c13 tinyint, c14 smallint, c15 int, c16 bigint, c17 tinyint, c18 smallint, c19 int, c20 bigint, @@ -633,8 +537,7 @@ insert_num part c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12 c13 c14 c15 c16 c17 c18 c 103 1 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL original 104 1 23 834 203332 888888857923222 23 834 203332 888888857923222 23 834 203332 888888857923222 23 834 203332 888888857923222 23 834 203332 888888857923222 original 105 1 -99 -28300 -999992 -222282153733 -99 -28300 -999992 -222282153733 -99 -28300 -999992 -222282153733 -99 -28300 -999992 -222282153733 -99 -28300 -999992 -222282153733 original -PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... -alter table part_change_numeric_group_string_group_multi_ints_string_group replace columns (insert_num int, +PREHOOK: query: alter table part_change_numeric_group_string_group_multi_ints_string_group replace columns (insert_num int, c1 STRING, c2 STRING, c3 STRING, c4 STRING, c5 CHAR(50), c6 CHAR(50), c7 CHAR(50), c8 CHAR(50), c9 CHAR(5), c10 CHAR(5), c11 CHAR(5), c12 CHAR(5), c13 VARCHAR(50), c14 VARCHAR(50), c15 VARCHAR(50), c16 VARCHAR(50), c17 VARCHAR(5), c18 VARCHAR(5), c19 VARCHAR(5), c20 VARCHAR(5), @@ -642,8 +545,7 @@ alter table part_change_numeric_group_string_group_multi_ints_string_group repla PREHOOK: type: ALTERTABLE_REPLACECOLS PREHOOK: Input: default@part_change_numeric_group_string_group_multi_ints_string_group PREHOOK: Output: default@part_change_numeric_group_string_group_multi_ints_string_group -POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... -alter table part_change_numeric_group_string_group_multi_ints_string_group replace columns (insert_num int, +POSTHOOK: query: alter table part_change_numeric_group_string_group_multi_ints_string_group replace columns (insert_num int, c1 STRING, c2 STRING, c3 STRING, c4 STRING, c5 CHAR(50), c6 CHAR(50), c7 CHAR(50), c8 CHAR(50), c9 CHAR(5), c10 CHAR(5), c11 CHAR(5), c12 CHAR(5), c13 VARCHAR(50), c14 VARCHAR(50), c15 VARCHAR(50), c16 VARCHAR(50), c17 VARCHAR(5), c18 VARCHAR(5), c19 VARCHAR(5), c20 VARCHAR(5), @@ -705,7 +607,7 @@ Stage-0 File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22"] - TableScan [TS_0] (rows=6 width=153) + TableScan [TS_0] (rows=6 width=182) default@part_change_numeric_group_string_group_multi_ints_string_group,part_change_numeric_group_string_group_multi_ints_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","c16","c17","c18","c19","c20","b"] PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,b from part_change_numeric_group_string_group_multi_ints_string_group @@ -733,14 +635,7 @@ POSTHOOK: query: drop table part_change_numeric_group_string_group_multi_ints_st POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_change_numeric_group_string_group_multi_ints_string_group POSTHOOK: Output: default@part_change_numeric_group_string_group_multi_ints_string_group -PREHOOK: query: -- --- SUBSECTION: ALTER TABLE CHANGE COLUMNS for NUMERIC_GROUP -> STRING_GROUP: --- (DECIMAL, FLOAT, DOUBLE), STRING and --- (DECIMAL, FLOAT, DOUBLE), CHAR and CHAR trunc and --- (DECIMAL, FLOAT, DOUBLE), VARCHAR and VARCHAR trunc --- --- -CREATE TABLE part_change_numeric_group_string_group_floating_string_group(insert_num int, +PREHOOK: query: CREATE TABLE part_change_numeric_group_string_group_floating_string_group(insert_num int, c1 decimal(38,18), c2 float, c3 double, c4 decimal(38,18), c5 float, c6 double, c7 decimal(38,18), c8 float, c9 double, c10 decimal(38,18), c11 float, c12 double, c13 decimal(38,18), c14 float, c15 double, @@ -748,14 +643,7 @@ CREATE TABLE part_change_numeric_group_string_group_floating_string_group(insert PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@part_change_numeric_group_string_group_floating_string_group -POSTHOOK: query: -- --- SUBSECTION: ALTER TABLE CHANGE COLUMNS for NUMERIC_GROUP -> STRING_GROUP: --- (DECIMAL, FLOAT, DOUBLE), STRING and --- (DECIMAL, FLOAT, DOUBLE), CHAR and CHAR trunc and --- (DECIMAL, FLOAT, DOUBLE), VARCHAR and VARCHAR trunc --- --- -CREATE TABLE part_change_numeric_group_string_group_floating_string_group(insert_num int, +POSTHOOK: query: CREATE TABLE part_change_numeric_group_string_group_floating_string_group(insert_num int, c1 decimal(38,18), c2 float, c3 double, c4 decimal(38,18), c5 float, c6 double, c7 decimal(38,18), c8 float, c9 double, c10 decimal(38,18), c11 float, c12 double, c13 decimal(38,18), c14 float, c15 double, @@ -813,8 +701,7 @@ insert_num part c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12 c13 c14 c15 b 103 1 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL original 104 1 66475.561431000000000000 -100.35978 30.774 66475.561431000000000000 -100.35978 30.774 66475.561431000000000000 -100.35978 30.774 66475.561431000000000000 -100.35978 30.774 66475.561431000000000000 -100.35978 30.774 original 105 1 9250340.750000000000000000 NULL 46114.28 9250340.750000000000000000 NULL 46114.28 9250340.750000000000000000 NULL 46114.28 9250340.750000000000000000 NULL 46114.28 9250340.750000000000000000 NULL 46114.28 original -PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... -alter table part_change_numeric_group_string_group_floating_string_group replace columns (insert_num int, +PREHOOK: query: alter table part_change_numeric_group_string_group_floating_string_group replace columns (insert_num int, c1 STRING, c2 STRING, c3 STRING, c4 CHAR(50), c5 CHAR(50), c6 CHAR(50), c7 CHAR(7), c8 CHAR(7), c9 CHAR(7), c10 VARCHAR(50), c11 VARCHAR(50), c12 VARCHAR(50), c13 VARCHAR(7), c14 VARCHAR(7), c15 VARCHAR(7), @@ -822,8 +709,7 @@ alter table part_change_numeric_group_string_group_floating_string_group replace PREHOOK: type: ALTERTABLE_REPLACECOLS PREHOOK: Input: default@part_change_numeric_group_string_group_floating_string_group PREHOOK: Output: default@part_change_numeric_group_string_group_floating_string_group -POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... -alter table part_change_numeric_group_string_group_floating_string_group replace columns (insert_num int, +POSTHOOK: query: alter table part_change_numeric_group_string_group_floating_string_group replace columns (insert_num int, c1 STRING, c2 STRING, c3 STRING, c4 CHAR(50), c5 CHAR(50), c6 CHAR(50), c7 CHAR(7), c8 CHAR(7), c9 CHAR(7), c10 VARCHAR(50), c11 VARCHAR(50), c12 VARCHAR(50), c13 VARCHAR(7), c14 VARCHAR(7), c15 VARCHAR(7), @@ -880,7 +766,7 @@ Stage-0 File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"] - TableScan [TS_0] (rows=6 width=231) + TableScan [TS_0] (rows=6 width=253) default@part_change_numeric_group_string_group_floating_string_group,part_change_numeric_group_string_group_floating_string_group,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","c11","c12","c13","c14","c15","b"] PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,b from part_change_numeric_group_string_group_floating_string_group @@ -908,30 +794,14 @@ POSTHOOK: query: drop table part_change_numeric_group_string_group_floating_stri POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_change_numeric_group_string_group_floating_string_group POSTHOOK: Output: default@part_change_numeric_group_string_group_floating_string_group -PREHOOK: query: ------------------------------------------------------------------------------------------ --- SECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> STRING_GROUP --- --- --- --- SUBSECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> STRING_GROUP: STRING, (CHAR, CHAR trunc, VARCHAR, VARCHAR trunc) and --- CHAR, (VARCHAR, VARCHAR trunc, STRING) and VARCHAR, (CHAR, CHAR trunc, STRING) --- -CREATE TABLE part_change_string_group_string_group_string(insert_num int, +PREHOOK: query: CREATE TABLE part_change_string_group_string_group_string(insert_num int, c1 string, c2 string, c3 string, c4 string, c5 CHAR(50), c6 CHAR(50), c7 CHAR(50), c8 VARCHAR(50), c9 VARCHAR(50), c10 VARCHAR(50), b STRING) PARTITIONED BY(part INT) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@part_change_string_group_string_group_string -POSTHOOK: query: ------------------------------------------------------------------------------------------ --- SECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> STRING_GROUP --- --- --- --- SUBSECTION: ALTER TABLE CHANGE COLUMNS for STRING_GROUP -> STRING_GROUP: STRING, (CHAR, CHAR trunc, VARCHAR, VARCHAR trunc) and --- CHAR, (VARCHAR, VARCHAR trunc, STRING) and VARCHAR, (CHAR, CHAR trunc, STRING) --- -CREATE TABLE part_change_string_group_string_group_string(insert_num int, +POSTHOOK: query: CREATE TABLE part_change_string_group_string_group_string(insert_num int, c1 string, c2 string, c3 string, c4 string, c5 CHAR(50), c6 CHAR(50), c7 CHAR(50), c8 VARCHAR(50), c9 VARCHAR(50), c10 VARCHAR(50), b STRING) PARTITIONED BY(part INT) @@ -983,16 +853,14 @@ insert_num part c1 c2 c3 c4 b 103 1 original 104 1 I cooked I cooked I cooked I cooked original 105 1 200 200 200 200 original -PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... -alter table part_change_string_group_string_group_string replace columns (insert_num int, +PREHOOK: query: alter table part_change_string_group_string_group_string replace columns (insert_num int, c1 CHAR(50), c2 CHAR(9), c3 VARCHAR(50), c4 CHAR(9), c5 VARCHAR(50), c6 VARCHAR(9), c7 STRING, c8 CHAR(50), c9 CHAR(9), c10 STRING, b STRING) PREHOOK: type: ALTERTABLE_REPLACECOLS PREHOOK: Input: default@part_change_string_group_string_group_string PREHOOK: Output: default@part_change_string_group_string_group_string -POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... -alter table part_change_string_group_string_group_string replace columns (insert_num int, +POSTHOOK: query: alter table part_change_string_group_string_group_string replace columns (insert_num int, c1 CHAR(50), c2 CHAR(9), c3 VARCHAR(50), c4 CHAR(9), c5 VARCHAR(50), c6 VARCHAR(9), c7 STRING, c8 CHAR(50), c9 CHAR(9), c10 STRING, b STRING) @@ -1043,7 +911,7 @@ Stage-0 File Output Operator [FS_4] Select Operator [SEL_3] (rows=6 width=4) Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"] - TableScan [TS_0] (rows=6 width=70) + TableScan [TS_0] (rows=6 width=200) default@part_change_string_group_string_group_string,part_change_string_group_string_group_string,Tbl:COMPLETE,Col:PARTIAL,Output:["insert_num","c1","c2","c3","c4","c5","c6","c7","c8","c9","c10","b"] PREHOOK: query: select insert_num,part,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,b from part_change_string_group_string_group_string @@ -1071,18 +939,7 @@ POSTHOOK: query: drop table part_change_string_group_string_group_string POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_change_string_group_string_group_string POSTHOOK: Output: default@part_change_string_group_string_group_string -PREHOOK: query: ------------------------------------------------------------------------------------------ --- SECTION: ALTER TABLE CHANGE COLUMNS for "lower" type to "higher" NUMERIC_GROUP --- --- --- --- SUBSECTION: ALTER TABLE CHANGE COLUMNS for "lower" type to "higher" NUMERIC_GROUP: --- TINYINT, (SMALLINT, INT, BIGINT, DECIMAL, FLOAT, DOUBLE) and --- SMALLINT, (INT, BIGINT, DECIMAL, FLOAT, DOUBLE) and --- INT, (BIGINT, DECIMAL, FLOAT, DOUBLE) and --- BIGINT, (DECIMAL, FLOAT, DOUBLE) --- -CREATE TABLE part_change_lower_to_higher_numeric_group_tinyint_to_bigint(insert_num int, +PREHOOK: query: CREATE TABLE part_change_lower_to_higher_numeric_group_tinyint_to_bigint(insert_num int, c1 tinyint, c2 tinyint, c3 tinyint, c4 tinyint, c5 tinyint, c6 tinyint, c7 smallint, c8 smallint, c9 smallint, c10 smallint, c11 smallint, c12 int, c13 int, c14 int, c15 int, @@ -1091,18 +948,7 @@ CREATE TABLE part_change_lower_to_higher_numeric_group_tinyint_to_bigint(insert_ PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint -POSTHOOK: query: ------------------------------------------------------------------------------------------ --- SECTION: ALTER TABLE CHANGE COLUMNS for "lower" type to "higher" NUMERIC_GROUP --- --- --- --- SUBSECTION: ALTER TABLE CHANGE COLUMNS for "lower" type to "higher" NUMERIC_GROUP: --- TINYINT, (SMALLINT, INT, BIGINT, DECIMAL, FLOAT, DOUBLE) and --- SMALLINT, (INT, BIGINT, DECIMAL, FLOAT, DOUBLE) and --- INT, (BIGINT, DECIMAL, FLOAT, DOUBLE) and --- BIGINT, (DECIMAL, FLOAT, DOUBLE) --- -CREATE TABLE part_change_lower_to_higher_numeric_group_tinyint_to_bigint(insert_num int, +POSTHOOK: query: CREATE TABLE part_change_lower_to_higher_numeric_group_tinyint_to_bigint(insert_num int, c1 tinyint, c2 tinyint, c3 tinyint, c4 tinyint, c5 tinyint, c6 tinyint, c7 smallint, c8 smallint, c9 smallint, c10 smallint, c11 smallint, c12 int, c13 int, c14 int, c15 int, @@ -1166,8 +1012,7 @@ insert_num part c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12 c13 c14 c15 c16 c17 c18 b 103 1 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL original 104 1 23 23 23 23 23 23 834 834 834 834 834 203332 203332 203332 203332 888888857923222 888888857923222 888888857923222 original 105 1 -99 -99 -99 -99 -99 -99 -28300 -28300 -28300 -28300 -28300 -999992 -999992 -999992 -999992 -222282153733 -222282153733 -222282153733 original -PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... -alter table part_change_lower_to_higher_numeric_group_tinyint_to_bigint replace columns (insert_num int, +PREHOOK: query: alter table part_change_lower_to_higher_numeric_group_tinyint_to_bigint replace columns (insert_num int, c1 SMALLINT, c2 INT, c3 BIGINT, c4 decimal(38,18), c5 FLOAT, c6 DOUBLE, c7 INT, c8 BIGINT, c9 decimal(38,18), c10 FLOAT, c11 DOUBLE, c12 BIGINT, c13 decimal(38,18), c14 FLOAT, c15 DOUBLE, @@ -1176,8 +1021,7 @@ alter table part_change_lower_to_higher_numeric_group_tinyint_to_bigint replace PREHOOK: type: ALTERTABLE_REPLACECOLS PREHOOK: Input: default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint PREHOOK: Output: default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint -POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... -alter table part_change_lower_to_higher_numeric_group_tinyint_to_bigint replace columns (insert_num int, +POSTHOOK: query: alter table part_change_lower_to_higher_numeric_group_tinyint_to_bigint replace columns (insert_num int, c1 SMALLINT, c2 INT, c3 BIGINT, c4 decimal(38,18), c5 FLOAT, c6 DOUBLE, c7 INT, c8 BIGINT, c9 decimal(38,18), c10 FLOAT, c11 DOUBLE, c12 BIGINT, c13 decimal(38,18), c14 FLOAT, c15 DOUBLE, @@ -1268,24 +1112,14 @@ POSTHOOK: query: drop table part_change_lower_to_higher_numeric_group_tinyint_to POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint POSTHOOK: Output: default@part_change_lower_to_higher_numeric_group_tinyint_to_bigint -PREHOOK: query: -- --- SUBSECTION: ALTER TABLE CHANGE COLUMNS for "lower" type to "higher" NUMERIC_GROUP: --- DECIMAL, (FLOAT, DOUBLE) and --- FLOAT, (DOUBLE) --- -CREATE TABLE part_change_lower_to_higher_numeric_group_decimal_to_float(insert_num int, +PREHOOK: query: CREATE TABLE part_change_lower_to_higher_numeric_group_decimal_to_float(insert_num int, c1 decimal(38,18), c2 decimal(38,18), c3 float, b STRING) PARTITIONED BY(part INT) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default PREHOOK: Output: default@part_change_lower_to_higher_numeric_group_decimal_to_float -POSTHOOK: query: -- --- SUBSECTION: ALTER TABLE CHANGE COLUMNS for "lower" type to "higher" NUMERIC_GROUP: --- DECIMAL, (FLOAT, DOUBLE) and --- FLOAT, (DOUBLE) --- -CREATE TABLE part_change_lower_to_higher_numeric_group_decimal_to_float(insert_num int, +POSTHOOK: query: CREATE TABLE part_change_lower_to_higher_numeric_group_decimal_to_float(insert_num int, c1 decimal(38,18), c2 decimal(38,18), c3 float, b STRING) PARTITIONED BY(part INT) @@ -1328,13 +1162,11 @@ insert_num part c1 c2 c3 b 103 1 NULL NULL NULL original 104 1 66475.561431000000000000 66475.561431000000000000 -100.35978 original 105 1 9250340.750000000000000000 9250340.750000000000000000 NULL original -PREHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... -alter table part_change_lower_to_higher_numeric_group_decimal_to_float replace columns (insert_num int, c1 float, c2 double, c3 DOUBLE, b STRING) +PREHOOK: query: alter table part_change_lower_to_higher_numeric_group_decimal_to_float replace columns (insert_num int, c1 float, c2 double, c3 DOUBLE, b STRING) PREHOOK: type: ALTERTABLE_REPLACECOLS PREHOOK: Input: default@part_change_lower_to_higher_numeric_group_decimal_to_float PREHOOK: Output: default@part_change_lower_to_higher_numeric_group_decimal_to_float -POSTHOOK: query: -- Table-Non-Cascade CHANGE COLUMNS ... -alter table part_change_lower_to_higher_numeric_group_decimal_to_float replace columns (insert_num int, c1 float, c2 double, c3 DOUBLE, b STRING) +POSTHOOK: query: alter table part_change_lower_to_higher_numeric_group_decimal_to_float replace columns (insert_num int, c1 float, c2 double, c3 DOUBLE, b STRING) POSTHOOK: type: ALTERTABLE_REPLACECOLS POSTHOOK: Input: default@part_change_lower_to_higher_numeric_group_decimal_to_float POSTHOOK: Output: default@part_change_lower_to_higher_numeric_group_decimal_to_float diff --git serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java index a597fd7..264335c 100644 --- serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java +++ serde/src/java/org/apache/hadoop/hive/serde2/lazy/fast/LazySimpleDeserializeRead.java @@ -583,6 +583,10 @@ public boolean readField(int fieldIndex) throws IOException { // U+FFFD will throw this as well logExceptionMessage(bytes, fieldStart, fieldLength, primitiveCategories[fieldIndex]); return false; + } catch (IllegalArgumentException iae) { + // E.g. can be thrown by Date.valueOf + logExceptionMessage(bytes, fieldStart, fieldLength, primitiveCategories[fieldIndex]); + return false; } }