[CARBONDATA-3954] Global sorting with array, if read from ORC format, write to carbon, error; If you use no_sort, success;

• Type: Bug
• Status: Resolved
• Priority: Major
• Resolution: Fixed
• Affects Version/s: 2.0.0
• Fix Version/s: 2.1.0
• Component/s: spark-integration
• Assignee: Unassigned
• Reporter: xiaohui
• Created: 18/Aug/20 09:38
• Updated: 23/Oct/20 09:15
• Resolved: 23/Oct/20 09:15

Description

orc table sql test:
create table array_orc(name string, col array<String>,fee int) STORED AS orc;
insert into array_orc values("xiao3",array('上呼吸道疾病 1','白内障1','胃溃疡1'),2);
insert into array_orc values("xiao3",array('上呼吸道疾病1 ','白内障1','胃溃疡1'),2);
insert into array_orc values("xiao3",array('上呼吸道疾病1','白内障 1','胃溃疡1'),2);
insert into array_orc values("xiao3",array('上呼吸道疾病1','白内障1','胃溃疡 1'),2);
insert into array_orc values("xiao3",array('上呼吸道疾病1','白内障1','胃溃疡1'),2);
insert into array_orc values("xiao5",array(null,'白内障1','胃溃疡1'),2);
insert into array_orc values("xiao5",null,2);
insert into array_orc values("xiao3",array('jjjjj'),2);
insert into array_orc values("xiao4",array('jjjjj','jjjjj'),2);
insert into array_orc values("xiao4",NULL,2);

0: jdbc:hive2://localhost:10000> use dict;
---------+

Result

---------+
---------+
No rows selected (0.391 seconds)
0: jdbc:hive2://localhost:10000> select * from array_orc;
-----------------------------------------+

name

col

fee

-----------------------------------------+

xiao3	["",null,"jkkkk"]	3
xiao2	["上呼吸道疾病1","白内障1","胃溃疡1"]	2
xiao3	["",null,"jkkkk"]	3
xiao1	["上呼吸道疾病","白内障","胃溃疡"]	1
xiao9	NULL	3
xiao9	NULL	3
xiao3	NULL	3
xiao6	NULL	3
xiao2	["上呼吸道疾病 1","白内障 1","胃溃疡 1"]	2
xiao1	["上呼吸道疾病 ","白内障 ","胃溃疡 "]	1
xiao3	NULL	3
xiao3	[null]	3
xiao3	[""]	3

-----------------------------------------+
13 rows selected (0.416 seconds)
0: jdbc:hive2://localhost:10000> create table array_carbon4(name string, col array<String>,fee int) STORED AS carbondata TBLPROPERTIES ('SORT_COLUMNS'='name',
0: jdbc:hive2://localhost:10000> 'TABLE_BLOCKSIZE'='128',
0: jdbc:hive2://localhost:10000> 'TABLE_BLOCKLET_SIZE'='128',
0: jdbc:hive2://localhost:10000> 'SORT_SCOPE'='no_SORT');
---------+

Result

---------+
---------+
No rows selected (1.04 seconds)
0: jdbc:hive2://localhost:10000> insert overwrite table array_carbon4 select name,col,fee from array_orc;
---------+

Result

---------+
---------+
No rows selected (5.065 seconds)
0: jdbc:hive2://localhost:10000> create table array_carbon5(name string, col array<String>,fee int) STORED AS carbondata TBLPROPERTIES ('SORT_COLUMNS'='name',
0: jdbc:hive2://localhost:10000> 'TABLE_BLOCKSIZE'='128',
0: jdbc:hive2://localhost:10000> 'TABLE_BLOCKLET_SIZE'='128',
0: jdbc:hive2://localhost:10000> 'SORT_SCOPE'='global_SORT');
---------+

Result

---------+
---------+
No rows selected (0.098 seconds)
0: jdbc:hive2://localhost:10000> insert overwrite table array_carbon5 select name,col,fee from array_orc;
Error: java.lang.Exception: DataLoad failure (state=,code=0)