Index: ql/src/test/results/clientnegative/alter_partition_coltype_invalidtype_conversion.q.out =================================================================== --- ql/src/test/results/clientnegative/alter_partition_coltype_invalidtype_conversion.q.out (revision 0) +++ ql/src/test/results/clientnegative/alter_partition_coltype_invalidtype_conversion.q.out (revision 0) @@ -0,0 +1,42 @@ +PREHOOK: query: -- create testing table +create table alter_coltype(key string, value string) partitioned by (dt string, ts string) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +POSTHOOK: query: -- create testing table +create table alter_coltype(key string, value string) partitioned by (dt string, ts string) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@alter_coltype +PREHOOK: query: -- insert and create a partition +insert overwrite table alter_coltype partition(dt='100x', ts='6:30pm') select * from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +PREHOOK: Output: default@alter_coltype@dt=100x/ts=6%3A30pm +POSTHOOK: query: -- insert and create a partition +insert overwrite table alter_coltype partition(dt='100x', ts='6:30pm') select * from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@alter_coltype@dt=100x/ts=6%3A30pm +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: desc alter_coltype +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc alter_coltype +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +key string +value string +dt string +ts string + +# Partition Information +# col_name data_type comment + +dt string +ts string +PREHOOK: query: -- alter partition key column data type for dt column to a wrong type +alter table alter_coltype partition column (dt int) +PREHOOK: type: ALTERTABLE_PARTCOLTYPE +PREHOOK: Input: default@alter_coltype +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Exception while checking type conversion of existing partition values to FieldSchema(name:dt, type:int, comment:null) : Exception while converting string to int for value : 100x Index: ql/src/test/results/clientpositive/alter_partition_coltype.q.out =================================================================== --- ql/src/test/results/clientpositive/alter_partition_coltype.q.out (revision 1584206) +++ ql/src/test/results/clientpositive/alter_partition_coltype.q.out (working copy) @@ -8,23 +8,23 @@ POSTHOOK: Output: database:default POSTHOOK: Output: default@alter_coltype PREHOOK: query: -- insert and create a partition. -insert overwrite table alter_coltype partition(dt='100x', ts='6:30pm') select * from src1 +insert overwrite table alter_coltype partition(dt='100', ts='6.30') select * from src1 PREHOOK: type: QUERY PREHOOK: Input: default@src1 -PREHOOK: Output: default@alter_coltype@dt=100x/ts=6%3A30pm +PREHOOK: Output: default@alter_coltype@dt=100/ts=6.30 POSTHOOK: query: -- insert and create a partition. -insert overwrite table alter_coltype partition(dt='100x', ts='6:30pm') select * from src1 +insert overwrite table alter_coltype partition(dt='100', ts='6.30') select * from src1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src1 -POSTHOOK: Output: default@alter_coltype@dt=100x/ts=6%3A30pm -POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Output: default@alter_coltype@dt=100/ts=6.30 +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: desc alter_coltype PREHOOK: type: DESCTABLE POSTHOOK: query: desc alter_coltype POSTHOOK: type: DESCTABLE -POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] key string value string dt string @@ -36,115 +36,986 @@ dt string ts string PREHOOK: query: -- select with paritition predicate. -select count(*) from alter_coltype where dt = '100x' +select count(*) from alter_coltype where dt = '100' PREHOOK: type: QUERY PREHOOK: Input: default@alter_coltype -PREHOOK: Input: default@alter_coltype@dt=100x/ts=6%3A30pm +PREHOOK: Input: default@alter_coltype@dt=100/ts=6.30 #### A masked pattern was here #### POSTHOOK: query: -- select with paritition predicate. -select count(*) from alter_coltype where dt = '100x' +select count(*) from alter_coltype where dt = '100' POSTHOOK: type: QUERY POSTHOOK: Input: default@alter_coltype -POSTHOOK: Input: default@alter_coltype@dt=100x/ts=6%3A30pm +POSTHOOK: Input: default@alter_coltype@dt=100/ts=6.30 #### A masked pattern was here #### -POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] 25 PREHOOK: query: -- alter partition key column data type for dt column. --- alter table alter_coltype partition column (dt int); +alter table alter_coltype partition column (dt int) +PREHOOK: type: ALTERTABLE_PARTCOLTYPE +PREHOOK: Input: default@alter_coltype +POSTHOOK: query: -- alter partition key column data type for dt column. +alter table alter_coltype partition column (dt int) +POSTHOOK: type: ALTERTABLE_PARTCOLTYPE +POSTHOOK: Input: default@alter_coltype +POSTHOOK: Output: default@alter_coltype +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- load a new partition using new data type. +insert overwrite table alter_coltype partition(dt=100, ts='3.0') select * from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +PREHOOK: Output: default@alter_coltype@dt=100/ts=3.0 +POSTHOOK: query: -- load a new partition using new data type. +insert overwrite table alter_coltype partition(dt=100, ts='3.0') select * from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@alter_coltype@dt=100/ts=3.0 +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- make sure the partition predicate still works. +select count(*) from alter_coltype where dt = '100' +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_coltype +PREHOOK: Input: default@alter_coltype@dt=100/ts=3.0 +PREHOOK: Input: default@alter_coltype@dt=100/ts=6.30 +#### A masked pattern was here #### +POSTHOOK: query: -- make sure the partition predicate still works. +select count(*) from alter_coltype where dt = '100' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_coltype +POSTHOOK: Input: default@alter_coltype@dt=100/ts=3.0 +POSTHOOK: Input: default@alter_coltype@dt=100/ts=6.30 +#### A masked pattern was here #### +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +50 +PREHOOK: query: explain extended select count(*) from alter_coltype where dt = '100' +PREHOOK: type: QUERY +POSTHOOK: query: explain extended select count(*) from alter_coltype where dt = '100' +POSTHOOK: type: QUERY +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + alter_coltype + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_FUNCTIONSTAR + count + TOK_WHERE + = + TOK_TABLE_OR_COL + dt + '100' --- load a new partition using new data type. --- insert overwrite table alter_coltype partition(dt=10, ts='3.0') select * from src1; --- make sure the partition predicate still works. --- select count(*) from alter_coltype where dt = '100x'; --- explain extended select count(*) from alter_coltype where dt = '100x'; +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage --- select count(*) from alter_coltype where dt = '100'; +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alter_coltype + Statistics: Num rows: 50 Data size: 382 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + Statistics: Num rows: 50 Data size: 382 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col0 (type: bigint) + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ts=3.0 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + dt 100 + ts 3.0 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.alter_coltype + numFiles 1 + numRows 25 + partition_columns dt/ts + partition_columns.types int:string + rawDataSize 191 + serialization.ddl struct alter_coltype { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.alter_coltype + partition_columns dt/ts + partition_columns.types int:string + serialization.ddl struct alter_coltype { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.alter_coltype + name: default.alter_coltype +#### A masked pattern was here #### + Partition + base file name: ts=6.30 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + dt 100 + ts 6.30 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.alter_coltype + numFiles 1 + numRows 25 + partition_columns dt/ts + partition_columns.types int:string + rawDataSize 191 + serialization.ddl struct alter_coltype { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.alter_coltype + partition_columns dt/ts + partition_columns.types int:string + serialization.ddl struct alter_coltype { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.alter_coltype + name: default.alter_coltype + Truncated Path -> Alias: + /alter_coltype/dt=100/ts=3.0 [alter_coltype] + /alter_coltype/dt=100/ts=6.30 [alter_coltype] + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false --- alter partition key column data type for ts column. --- alter table alter_coltype partition column (ts double); + Stage: Stage-0 + Fetch Operator + limit: -1 --- alter table alter_coltype partition column (dt string); +PREHOOK: query: -- alter partition key column data type for ts column. +alter table alter_coltype partition column (ts double) +PREHOOK: type: ALTERTABLE_PARTCOLTYPE +PREHOOK: Input: default@alter_coltype +POSTHOOK: query: -- alter partition key column data type for ts column. +alter table alter_coltype partition column (ts double) +POSTHOOK: type: ALTERTABLE_PARTCOLTYPE +POSTHOOK: Input: default@alter_coltype +POSTHOOK: Output: default@alter_coltype +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: alter table alter_coltype partition column (dt string) +PREHOOK: type: ALTERTABLE_PARTCOLTYPE +PREHOOK: Input: default@alter_coltype +POSTHOOK: query: alter table alter_coltype partition column (dt string) +POSTHOOK: type: ALTERTABLE_PARTCOLTYPE +POSTHOOK: Input: default@alter_coltype +POSTHOOK: Output: default@alter_coltype +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- load a new partition using new data type. +insert overwrite table alter_coltype partition(dt='100', ts=3.0) select * from src1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src1 +PREHOOK: Output: default@alter_coltype@dt=100/ts=3.0 +POSTHOOK: query: -- load a new partition using new data type. +insert overwrite table alter_coltype partition(dt='100', ts=3.0) select * from src1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src1 +POSTHOOK: Output: default@alter_coltype@dt=100/ts=3.0 +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +PREHOOK: query: -- validate partition key column predicate can still work. +select count(*) from alter_coltype where ts = '6.30' +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_coltype +PREHOOK: Input: default@alter_coltype@dt=100/ts=6.30 +#### A masked pattern was here #### +POSTHOOK: query: -- validate partition key column predicate can still work. +select count(*) from alter_coltype where ts = '6.30' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_coltype +POSTHOOK: Input: default@alter_coltype@dt=100/ts=6.30 +#### A masked pattern was here #### +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +25 +PREHOOK: query: explain extended select count(*) from alter_coltype where ts = '6.30' +PREHOOK: type: QUERY +POSTHOOK: query: explain extended select count(*) from alter_coltype where ts = '6.30' +POSTHOOK: type: QUERY +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + alter_coltype + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_FUNCTIONSTAR + count + TOK_WHERE + = + TOK_TABLE_OR_COL + ts + '6.30' --- load a new partition using new data type. --- insert overwrite table alter_coltype partition(dt='100x', ts=3.0) select * from src1; --- validate partition key column predicate can still work. --- select count(*) from alter_coltype where ts = '6:30pm'; --- explain extended select count(*) from alter_coltype where ts = '6:30pm'; +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage --- validate partition key column predicate on two different partition column data type --- can still work. --- select count(*) from alter_coltype where ts = 3.0 and dt=10; --- explain extended select count(*) from alter_coltype where ts = 3.0 and dt=10; +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alter_coltype + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col0 (type: bigint) + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ts=6.30 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + dt 100 + ts 6.30 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.alter_coltype + numFiles 1 + numRows 25 + partition_columns dt/ts + partition_columns.types string:double + rawDataSize 191 + serialization.ddl struct alter_coltype { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.alter_coltype + partition_columns dt/ts + partition_columns.types string:double + serialization.ddl struct alter_coltype { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.alter_coltype + name: default.alter_coltype + Truncated Path -> Alias: + /alter_coltype/dt=100/ts=6.30 [alter_coltype] + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false --- query where multiple partition values (of different datatypes) are being selected --- select key, value, dt, ts from alter_coltype where dt is not null; --- explain extended select key, value, dt, ts from alter_coltype where dt is not null; + Stage: Stage-0 + Fetch Operator + limit: -1 --- select count(*) from alter_coltype where ts = 3.0; - --- make sure the partition predicate still works. --- select count(*) from alter_coltype where dt = '100x' or dt = '10'; --- explain extended select count(*) from alter_coltype where dt = '100x' or dt = '10'; - --- desc alter_coltype; --- desc alter_coltype partition (dt='100x', ts='6:30pm'); --- desc alter_coltype partition (dt='100x', ts=3.0); --- desc alter_coltype partition (dt=10, ts=3.0); - -drop table alter_coltype -PREHOOK: type: DROPTABLE +PREHOOK: query: -- validate partition key column predicate on two different partition column data type +-- can still work. +select count(*) from alter_coltype where ts = 3.0 and dt=100 +PREHOOK: type: QUERY PREHOOK: Input: default@alter_coltype -PREHOOK: Output: default@alter_coltype -POSTHOOK: query: -- alter partition key column data type for dt column. --- alter table alter_coltype partition column (dt int); +PREHOOK: Input: default@alter_coltype@dt=100/ts=3.0 +#### A masked pattern was here #### +POSTHOOK: query: -- validate partition key column predicate on two different partition column data type +-- can still work. +select count(*) from alter_coltype where ts = 3.0 and dt=100 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_coltype +POSTHOOK: Input: default@alter_coltype@dt=100/ts=3.0 +#### A masked pattern was here #### +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +25 +PREHOOK: query: explain extended select count(*) from alter_coltype where ts = 3.0 and dt=100 +PREHOOK: type: QUERY +POSTHOOK: query: explain extended select count(*) from alter_coltype where ts = 3.0 and dt=100 +POSTHOOK: type: QUERY +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + alter_coltype + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_FUNCTIONSTAR + count + TOK_WHERE + and + = + TOK_TABLE_OR_COL + ts + 3.0 + = + TOK_TABLE_OR_COL + dt + 100 --- load a new partition using new data type. --- insert overwrite table alter_coltype partition(dt=10, ts='3.0') select * from src1; --- make sure the partition predicate still works. --- select count(*) from alter_coltype where dt = '100x'; --- explain extended select count(*) from alter_coltype where dt = '100x'; +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage --- select count(*) from alter_coltype where dt = '100'; +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alter_coltype + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + GatherStats: false + Select Operator + Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + tag: -1 + value expressions: _col0 (type: bigint) + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ts=3.0 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + dt 100 + ts 3.0 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.alter_coltype + numFiles 1 + numRows 25 + partition_columns dt/ts + partition_columns.types string:double + rawDataSize 191 + serialization.ddl struct alter_coltype { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.alter_coltype + partition_columns dt/ts + partition_columns.types string:double + serialization.ddl struct alter_coltype { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.alter_coltype + name: default.alter_coltype + Truncated Path -> Alias: + /alter_coltype/dt=100/ts=3.0 [alter_coltype] + Needs Tagging: false + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col0 (type: bigint) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0 + columns.types bigint + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false --- alter partition key column data type for ts column. --- alter table alter_coltype partition column (ts double); + Stage: Stage-0 + Fetch Operator + limit: -1 --- alter table alter_coltype partition column (dt string); +PREHOOK: query: -- query where multiple partition values (of different datatypes) are being selected +select key, value, dt, ts from alter_coltype where dt is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_coltype +PREHOOK: Input: default@alter_coltype@dt=100/ts=3.0 +PREHOOK: Input: default@alter_coltype@dt=100/ts=6.30 +#### A masked pattern was here #### +POSTHOOK: query: -- query where multiple partition values (of different datatypes) are being selected +select key, value, dt, ts from alter_coltype where dt is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_coltype +POSTHOOK: Input: default@alter_coltype@dt=100/ts=3.0 +POSTHOOK: Input: default@alter_coltype@dt=100/ts=6.30 +#### A masked pattern was here #### +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +238 val_238 100 3.0 + 100 3.0 +311 val_311 100 3.0 + val_27 100 3.0 + val_165 100 3.0 + val_409 100 3.0 +255 val_255 100 3.0 +278 val_278 100 3.0 +98 val_98 100 3.0 + val_484 100 3.0 + val_265 100 3.0 + val_193 100 3.0 +401 val_401 100 3.0 +150 val_150 100 3.0 +273 val_273 100 3.0 +224 100 3.0 +369 100 3.0 +66 val_66 100 3.0 +128 100 3.0 +213 val_213 100 3.0 +146 val_146 100 3.0 +406 val_406 100 3.0 + 100 3.0 + 100 3.0 + 100 3.0 +238 val_238 100 6.3 + 100 6.3 +311 val_311 100 6.3 + val_27 100 6.3 + val_165 100 6.3 + val_409 100 6.3 +255 val_255 100 6.3 +278 val_278 100 6.3 +98 val_98 100 6.3 + val_484 100 6.3 + val_265 100 6.3 + val_193 100 6.3 +401 val_401 100 6.3 +150 val_150 100 6.3 +273 val_273 100 6.3 +224 100 6.3 +369 100 6.3 +66 val_66 100 6.3 +128 100 6.3 +213 val_213 100 6.3 +146 val_146 100 6.3 +406 val_406 100 6.3 + 100 6.3 + 100 6.3 + 100 6.3 +PREHOOK: query: explain extended select key, value, dt, ts from alter_coltype where dt is not null +PREHOOK: type: QUERY +POSTHOOK: query: explain extended select key, value, dt, ts from alter_coltype where dt is not null +POSTHOOK: type: QUERY +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_TABREF + TOK_TABNAME + alter_coltype + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_TABLE_OR_COL + key + TOK_SELEXPR + TOK_TABLE_OR_COL + value + TOK_SELEXPR + TOK_TABLE_OR_COL + dt + TOK_SELEXPR + TOK_TABLE_OR_COL + ts + TOK_WHERE + TOK_FUNCTION + TOK_ISNOTNULL + TOK_TABLE_OR_COL + dt --- load a new partition using new data type. --- insert overwrite table alter_coltype partition(dt='100x', ts=3.0) select * from src1; --- validate partition key column predicate can still work. --- select count(*) from alter_coltype where ts = '6:30pm'; --- explain extended select count(*) from alter_coltype where ts = '6:30pm'; +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 is a root stage --- validate partition key column predicate on two different partition column data type --- can still work. --- select count(*) from alter_coltype where ts = 3.0 and dt=10; --- explain extended select count(*) from alter_coltype where ts = 3.0 and dt=10; +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: alter_coltype + Statistics: Num rows: 50 Data size: 382 Basic stats: COMPLETE Column stats: NONE + GatherStats: false + Select Operator + expressions: key (type: string), value (type: string), dt (type: string), ts (type: double) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 50 Data size: 382 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + GlobalTableId: 0 +#### A masked pattern was here #### + NumFilesPerFileSink: 1 + Statistics: Num rows: 50 Data size: 382 Basic stats: COMPLETE Column stats: NONE +#### A masked pattern was here #### + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + columns _col0,_col1,_col2,_col3 + columns.types string:string:string:double + escape.delim \ + hive.serialization.extend.nesting.levels true + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + TotalFiles: 1 + GatherStats: false + MultiFileSpray: false + Path -> Alias: +#### A masked pattern was here #### + Path -> Partition: +#### A masked pattern was here #### + Partition + base file name: ts=3.0 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + dt 100 + ts 3.0 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.alter_coltype + numFiles 1 + numRows 25 + partition_columns dt/ts + partition_columns.types string:double + rawDataSize 191 + serialization.ddl struct alter_coltype { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.alter_coltype + partition_columns dt/ts + partition_columns.types string:double + serialization.ddl struct alter_coltype { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.alter_coltype + name: default.alter_coltype +#### A masked pattern was here #### + Partition + base file name: ts=6.30 + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + partition values: + dt 100 + ts 6.30 + properties: + COLUMN_STATS_ACCURATE true + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.alter_coltype + numFiles 1 + numRows 25 + partition_columns dt/ts + partition_columns.types string:double + rawDataSize 191 + serialization.ddl struct alter_coltype { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + totalSize 216 +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + properties: + bucket_count -1 + columns key,value + columns.comments + columns.types string:string +#### A masked pattern was here #### + name default.alter_coltype + partition_columns dt/ts + partition_columns.types string:double + serialization.ddl struct alter_coltype { string key, string value} + serialization.format 1 + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +#### A masked pattern was here #### + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + name: default.alter_coltype + name: default.alter_coltype + Truncated Path -> Alias: + /alter_coltype/dt=100/ts=3.0 [alter_coltype] + /alter_coltype/dt=100/ts=6.30 [alter_coltype] --- query where multiple partition values (of different datatypes) are being selected --- select key, value, dt, ts from alter_coltype where dt is not null; --- explain extended select key, value, dt, ts from alter_coltype where dt is not null; + Stage: Stage-0 + Fetch Operator + limit: -1 --- select count(*) from alter_coltype where ts = 3.0; - --- make sure the partition predicate still works. --- select count(*) from alter_coltype where dt = '100x' or dt = '10'; --- explain extended select count(*) from alter_coltype where dt = '100x' or dt = '10'; - --- desc alter_coltype; --- desc alter_coltype partition (dt='100x', ts='6:30pm'); --- desc alter_coltype partition (dt='100x', ts=3.0); --- desc alter_coltype partition (dt=10, ts=3.0); - -drop table alter_coltype +PREHOOK: query: select count(*) from alter_coltype where ts = 3.0 +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_coltype +PREHOOK: Input: default@alter_coltype@dt=100/ts=3.0 +#### A masked pattern was here #### +POSTHOOK: query: select count(*) from alter_coltype where ts = 3.0 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_coltype +POSTHOOK: Input: default@alter_coltype@dt=100/ts=3.0 +#### A masked pattern was here #### +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +25 +PREHOOK: query: -- make sure the partition predicate still works. +select count(*) from alter_coltype where dt = '100' +PREHOOK: type: QUERY +PREHOOK: Input: default@alter_coltype +PREHOOK: Input: default@alter_coltype@dt=100/ts=3.0 +PREHOOK: Input: default@alter_coltype@dt=100/ts=6.30 +#### A masked pattern was here #### +POSTHOOK: query: -- make sure the partition predicate still works. +select count(*) from alter_coltype where dt = '100' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@alter_coltype +POSTHOOK: Input: default@alter_coltype@dt=100/ts=3.0 +POSTHOOK: Input: default@alter_coltype@dt=100/ts=6.30 +#### A masked pattern was here #### +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +50 +PREHOOK: query: desc alter_coltype +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc alter_coltype +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +key string +value string +dt string +ts double + +# Partition Information +# col_name data_type comment + +dt string +ts double +PREHOOK: query: desc alter_coltype partition (dt='100', ts='6.30') +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc alter_coltype partition (dt='100', ts='6.30') +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +key string +value string +dt string +ts double + +# Partition Information +# col_name data_type comment + +dt string +ts double +PREHOOK: query: desc alter_coltype partition (dt='100', ts=3.0) +PREHOOK: type: DESCTABLE +POSTHOOK: query: desc alter_coltype partition (dt='100', ts=3.0) +POSTHOOK: type: DESCTABLE +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +key string +value string +dt string +ts double + +# Partition Information +# col_name data_type comment + +dt string +ts double +PREHOOK: query: drop table alter_coltype +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@alter_coltype +PREHOOK: Output: default@alter_coltype +POSTHOOK: query: drop table alter_coltype POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@alter_coltype POSTHOOK: Output: default@alter_coltype -POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] -POSTHOOK: Lineage: alter_coltype PARTITION(dt=100x,ts=6:30pm).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=3.0).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).key SIMPLE [(src1)src1.FieldSchema(name:key, type:string, comment:default), ] +POSTHOOK: Lineage: alter_coltype PARTITION(dt=100,ts=6.30).value SIMPLE [(src1)src1.FieldSchema(name:value, type:string, comment:default), ] Index: ql/src/test/queries/clientnegative/alter_partition_coltype_invalidtype_conversion.q =================================================================== --- ql/src/test/queries/clientnegative/alter_partition_coltype_invalidtype_conversion.q (revision 0) +++ ql/src/test/queries/clientnegative/alter_partition_coltype_invalidtype_conversion.q (revision 0) @@ -0,0 +1,10 @@ +-- create testing table +create table alter_coltype(key string, value string) partitioned by (dt string, ts string); + +-- insert and create a partition +insert overwrite table alter_coltype partition(dt='100x', ts='6:30pm') select * from src1; + +desc alter_coltype; + +-- alter partition key column data type for dt column to a wrong type +alter table alter_coltype partition column (dt int); Index: ql/src/test/queries/clientpositive/alter_partition_coltype.q =================================================================== --- ql/src/test/queries/clientpositive/alter_partition_coltype.q (revision 1584206) +++ ql/src/test/queries/clientpositive/alter_partition_coltype.q (working copy) @@ -2,56 +2,52 @@ create table alter_coltype(key string, value string) partitioned by (dt string, ts string); -- insert and create a partition. -insert overwrite table alter_coltype partition(dt='100x', ts='6:30pm') select * from src1; +insert overwrite table alter_coltype partition(dt='100', ts='6.30') select * from src1; desc alter_coltype; -- select with paritition predicate. -select count(*) from alter_coltype where dt = '100x'; +select count(*) from alter_coltype where dt = '100'; -- alter partition key column data type for dt column. --- alter table alter_coltype partition column (dt int); +alter table alter_coltype partition column (dt int); -- load a new partition using new data type. --- insert overwrite table alter_coltype partition(dt=10, ts='3.0') select * from src1; +insert overwrite table alter_coltype partition(dt=100, ts='3.0') select * from src1; -- make sure the partition predicate still works. --- select count(*) from alter_coltype where dt = '100x'; --- explain extended select count(*) from alter_coltype where dt = '100x'; +select count(*) from alter_coltype where dt = '100'; +explain extended select count(*) from alter_coltype where dt = '100'; --- select count(*) from alter_coltype where dt = '100'; - -- alter partition key column data type for ts column. --- alter table alter_coltype partition column (ts double); +alter table alter_coltype partition column (ts double); --- alter table alter_coltype partition column (dt string); +alter table alter_coltype partition column (dt string); -- load a new partition using new data type. --- insert overwrite table alter_coltype partition(dt='100x', ts=3.0) select * from src1; +insert overwrite table alter_coltype partition(dt='100', ts=3.0) select * from src1; -- validate partition key column predicate can still work. --- select count(*) from alter_coltype where ts = '6:30pm'; --- explain extended select count(*) from alter_coltype where ts = '6:30pm'; +select count(*) from alter_coltype where ts = '6.30'; +explain extended select count(*) from alter_coltype where ts = '6.30'; -- validate partition key column predicate on two different partition column data type -- can still work. --- select count(*) from alter_coltype where ts = 3.0 and dt=10; --- explain extended select count(*) from alter_coltype where ts = 3.0 and dt=10; +select count(*) from alter_coltype where ts = 3.0 and dt=100; +explain extended select count(*) from alter_coltype where ts = 3.0 and dt=100; -- query where multiple partition values (of different datatypes) are being selected --- select key, value, dt, ts from alter_coltype where dt is not null; --- explain extended select key, value, dt, ts from alter_coltype where dt is not null; +select key, value, dt, ts from alter_coltype where dt is not null; +explain extended select key, value, dt, ts from alter_coltype where dt is not null; --- select count(*) from alter_coltype where ts = 3.0; +select count(*) from alter_coltype where ts = 3.0; -- make sure the partition predicate still works. --- select count(*) from alter_coltype where dt = '100x' or dt = '10'; --- explain extended select count(*) from alter_coltype where dt = '100x' or dt = '10'; +select count(*) from alter_coltype where dt = '100'; --- desc alter_coltype; --- desc alter_coltype partition (dt='100x', ts='6:30pm'); --- desc alter_coltype partition (dt='100x', ts=3.0); --- desc alter_coltype partition (dt=10, ts=3.0); +desc alter_coltype; +desc alter_coltype partition (dt='100', ts='6.30'); +desc alter_coltype partition (dt='100', ts=3.0); drop table alter_coltype; Index: ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (revision 1584206) +++ ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (working copy) @@ -38,11 +38,13 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Map.Entry; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; +import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; @@ -114,6 +116,7 @@ import org.apache.hadoop.hive.ql.metadata.formatting.MetaDataFormatter; import org.apache.hadoop.hive.ql.parse.AlterTablePartMergeFilesDesc; import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer; +import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.AddPartitionDesc; import org.apache.hadoop.hive.ql.plan.AlterDatabaseDesc; import org.apache.hadoop.hive.ql.plan.AlterIndexDesc; @@ -180,7 +183,14 @@ import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; import org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.IOUtils; @@ -1255,7 +1265,53 @@ assert(tbl.isPartitioned()); List newPartitionKeys = new ArrayList(); - + + //Check if the existing partition values can be type casted to the new column type + // with a non null value before trying to alter the partition column type. + try { + Set partitions = db.getAllPartitionsOf(tbl); + int colIndex = -1; + for(FieldSchema col : tbl.getTTable().getPartitionKeys()) { + colIndex++; + if (col.getName().compareTo(alterPartitionDesc.getPartKeySpec().getName()) == 0) { + break; + } + } + + if (colIndex == -1 || colIndex == tbl.getTTable().getPartitionKeys().size()) { + throw new HiveException("Cannot find partition column " + + alterPartitionDesc.getPartKeySpec().getName()); + } + + TypeInfo expectedType = + TypeInfoUtils.getTypeInfoFromTypeString(alterPartitionDesc.getPartKeySpec().getType()); + ObjectInspector outputOI = + TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(expectedType); + Converter converter = ObjectInspectorConverters.getConverter( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, outputOI); + + // For all the existing partitions, check if the value can be type casted to a non-null object + for(Partition part : partitions) { + try { + String value = part.getValues().get(colIndex); + Object convertedValue = + converter.convert(value); + if (convertedValue == null) { + throw new HiveException(" Converting from " + TypeInfoFactory.stringTypeInfo + " to " + + expectedType + " for value : " + value + " resulted in NULL object"); + } + } catch (Exception e) { + throw new HiveException("Exception while converting " + + TypeInfoFactory.stringTypeInfo + " to " + + expectedType + " for value : " + part.getValues().get(colIndex)); + } + } + } catch(Exception e) { + throw new HiveException( + "Exception while checking type conversion of existing partition values to " + + alterPartitionDesc.getPartKeySpec() + " : " + e.getMessage()); + } + for(FieldSchema col : tbl.getTTable().getPartitionKeys()) { if (col.getName().compareTo(alterPartitionDesc.getPartKeySpec().getName()) == 0) { newPartitionKeys.add(alterPartitionDesc.getPartKeySpec());