[CARBONDATA-4228] Deleted records are reappearing in the select queries from the Alter added carbon segments after delete,update opertions.

• Type: Bug
• Status: Resolved
• Priority: Major
• Resolution: Fixed
• Affects Version/s: 1.6.1
• Fix Version/s: 2.3.0
• Assignee: Unassigned
• Reporter: Prasanna Ravichandran
• Created: 23/Jun/21 07:23
• Updated: 07/Oct/21 23:27
• Resolved: 07/Oct/21 11:13

Description

Deleted records are not deleting and displaying in the select queries from the Alter added carbon segments after delete, update operations. 

Test queries:

drop table uniqdata;
CREATE TABLE uniqdata (cust_id int,cust_name String,active_emui_version string, dob timestamp, doj timestamp, bigint_column1 bigint,bigint_column2 bigint,decimal_column1 decimal(30,10), decimal_column2 decimal(36,36),double_column1 double, double_column2 double,integer_column1 int) stored as carbondata;
load data inpath 'hdfs://hacluster/user/prasanna/2000_UniqData.csv' into table uniqdata options('fileheader'='cust_id,cust_name,active_emui_version,dob,doj,bigint_column1,bigint_column2,decimal_column1,decimal_column2,double_column1,double_column2,integer_column1','bad_records_action'='force');

--Create a copy of files from the first seg;
--hdfs dfs -rm -r -f /uniq1/*;
--hdfs dfs -mkdir -p /uniq1/
--hdfs dfs -cp /user/hive/warehouse/carbon.store/rps/uniqdata/Fact/Part0/Segment_0/* /uniq1/;
--hdfs dfs -ls /uniq1/;

use rps;
Alter table uniqdata add segment options ('path'='hdfs://hacluster/uniq1/','format'='carbon');

--update and delete works fine without throwing error but it wont work on the added carbon segments;
delete from uniqdata where cust_id=9001;
update uniqdata set (cust_name)=('Rahu') where cust_id=10000;

set carbon.input.segments.rps.uniqdata=1;
--First segment represent the added segment;
select cust_name from uniqdata where cust_id=10000;--CUST_NAME_01000 - incorrect value should be Rahu;
select count from uniqdata where cust_id=9001;--returns 1 - incorrect, should be 0 as 9001 cust_id records are deleted through Delete DDL;
reset;

 

Console:

> Alter table uniqdata add segment options ('path'='hdfs://hacluster/uniq1/','format'='carbon');
---------

Result

---------
---------
No rows selected (1.226 seconds)
>
> delete from uniqdata where cust_id=9001;
INFO : Execution ID: 139
--------------------

Deleted Row Count

--------------------

2

--------------------
1 row selected (5.321 seconds)
> update uniqdata set (cust_name)=('Rahu') where cust_id=10000;
INFO : Execution ID: 142
--------------------

Updated Row Count

--------------------

2

--------------------
1 row selected (7.938 seconds)
>
>
> set carbon.input.segments.rps.uniqdata=1;
-------------------------------------------+

key

value

-------------------------------------------+

carbon.input.segments.rps.uniqdata

1

-------------------------------------------+
1 row selected (0.05 seconds)
> --First segment represent the added segment;
> select cust_name from uniqdata where cust_id=10000;--CUST_NAME_01000 - incorrect value should be Rahu;
INFO : Execution ID: 147
------------------

cust_name

------------------

CUST_NAME_01000

------------------
1 row selected (0.93 seconds)
> select count from uniqdata where cust_id=9001;--returns 1 - incorrect, should be 0 as 9001 cust_id records are deleted through Delete DDL;
INFO : Execution ID: 148
-----------

count(1)

-----------

1

-----------
1 row selected (1.149 seconds)