[SPARK-32685] Script transform hive serde default field.delimit is '\t'

• Type: Sub-task
• Status: Resolved
• Priority: Major
• Resolution: Fixed
• Affects Version/s: 3.0.0
• Fix Version/s: 3.2.0
• Component/s: SQL
• Assignee: angerszhu
• Reporter: angerszhu
• Created: 21/Aug/20 15:56
• Updated: 06/Jan/21 03:59
• Resolved: 28/Dec/20 08:23

Description

 

select split(value, "\t") from (
SELECT TRANSFORM(a, b, c, null)
USING 'cat' 
FROM (select 1 as a, 2 as b, 3  as c) t
) temp;

result is :
_c0
["2","3","\\N"]

 

select split(value, "\t") from (
SELECT TRANSFORM(a, b, c, null)
  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
USING 'cat' 
  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
  WITH SERDEPROPERTIES (
   'serialization.last.column.takes.rest' = 'true'
  )
FROM (select 1 as a, 2 as b, 3  as c) t
) temp;


result is :
_c0
["2","3","\\N"]

 

 

 

select split(value, "\t") from (
SELECT TRANSFORM(a, b, c, null)
  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
USING 'cat' 
  ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
FROM (select 1 as a, 2 as b, 3  as c) t
) temp;

result is :
_c0 
["2"]

 

Issue Links

• links to: [Github] Pull Request #30942 (AngersZhuuuu)
• links to: [Github] Pull Request #31051 (AngersZhuuuu)
• links to: [Github] Pull Request #31051 (AngersZhuuuu)