[CASSANDRA-11887] Duplicate rows after a 2.2.5 to 3.0.4 migration

• Type: Bug
• Status: Resolved
• Priority: Urgent
• Resolution: Duplicate
• Severity: Critical
• Assignee: Unassigned
• Reporter: Julien Anguenot
• Created: 24/May/16 17:11
• Updated: 16/Apr/19 09:30
• Resolved: 21/Sep/16 23:40

Description

After migrating from 2.2.5 to 3.0.4, some tables seem to carry duplicate primary keys.

Below an example. Note, repair / scrub of such table do not seem to fix nor indicate any issues.

Table definition:

CREATE TABLE core.edge_ipsec_vpn_service (
    edge_uuid text PRIMARY KEY,
    enabled boolean,
    endpoints set<frozen<edge_ipsec_vpn_endpoint>>,
    tunnels set<frozen<edge_ipsec_vpn_tunnel>>
) WITH bloom_filter_fp_chance = 0.01
    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
    AND comment = ''
    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
    AND crc_check_chance = 1.0
    AND dclocal_read_repair_chance = 0.1
    AND default_time_to_live = 0
    AND gc_grace_seconds = 864000
    AND max_index_interval = 2048
    AND memtable_flush_period_in_ms = 0
    AND min_index_interval = 128
    AND read_repair_chance = 0.0
    AND speculative_retry = '99PERCENTILE';

UDTs:

CREATE TYPE core.edge_ipsec_vpn_endpoint (
    network text,
    public_ip text
);

CREATE TYPE core.edge_ipsec_vpn_tunnel (
    name text,
    description text,
    peer_ip_address text,
    peer_id text,
    local_ip_address text,
    local_id text,
    local_subnets frozen<set<frozen<edge_ipsec_vpn_subnet>>>,
    peer_subnets frozen<set<frozen<edge_ipsec_vpn_subnet>>>,
    shared_secret text,
    shared_secret_encrypted boolean,
    encryption_protocol text,
    mtu int,
    enabled boolean,
    operational boolean,
    error_details text,
    vpn_peer frozen<edge_ipsec_vpn_peer>
);

CREATE TYPE core.edge_ipsec_vpn_subnet (
    name text,
    gateway text,
    netmask text
);

CREATE TYPE core.edge_ipsec_vpn_peer (
    type text,
    id text,
    name text,
    vcd_url text,
    vcd_org text,
    vcd_username text
);

sstabledump extract (IP addressees hidden as well as secrets)

[...]
 {
    "partition" : {
      "key" : [ "84d567cc-0165-4e64-ab97-3a9d06370ba9" ],
      "position" : 131146
    },
    "rows" : [
      {
        "type" : "row",
        "position" : 131236,
        "liveness_info" : { "tstamp" : "2016-05-06T17:07:15.416003Z" },
        "cells" : [
          { "name" : "enabled", "value" : "true" },
          { "name" : "tunnels", "path" : [ “XXX::1.2.3.4:1.2.3.4:1.2.3.4:1.2.3.4:XXX:XXX:false:AES256:1500:true:false::third party\\:1.2.3.4\\:\\:\\:\\:” ], "value" : "" }
        ]
      },
      {
        "type" : "row",
        "position" : 131597,
        "cells" : [
          { "name" : "endpoints", "path" : [ “XXX” ], "value" : "", "tstamp" : "2016-03-29T08:05:38.297015Z" },
          { "name" : "tunnels", "path" : [ “XXX::1.2.3.4:1.2.3.4:1.2.3.4:1.2.3.4:XXX:XXX:false:AES256:1500:true:true::third party\\:1.2.3.4\\:\\:\\:\\:” ], "value" : "", "tstamp" : "2016-03-29T08:05:38.297015Z" },
          { "name" : "tunnels", "path" : [ “XXX::1.2.3.4:1.2.3.4:1.2.3.4:1.2.3.4:XXX:XXX:false:AES256:1500:true:false::third party\\:1.2.3.4\\:\\:\\:\\:" ], "value" : "", "tstamp" : "2016-03-14T18:05:07.262001Z" },
          { "name" : "tunnels", "path" : [ “XXX::1.2.3.4:1.2.3.4:1.2.3.4:1.2.3.4XXX:XXX:false:AES256:1500:true:true::third party\\:1.2.3.4\\:\\:\\:\\:" ], "value" : "", "tstamp" : "2016-03-29T08:05:38.297015Z" }
        ]
      },
      {
        "type" : "row",
        "position" : 133644,
        "cells" : [
          { "name" : "tunnels", "path" : [ “XXX::1.2.3.4:1.2.3.4:1.2.3.4:1.2.3.4:XXX:XXX:false:AES256:1500:true:true::third party\\:1.2.3.4\\:\\:\\:\\:" ], "value" : "", "tstamp" : "2016-03-29T07:05:27.213013Z" },
          { "name" : "tunnels", "path" : [ “XXX::1.2.3.4.7:1.2.3.4:1.2.3.4:1.2.3.4:XXX:XXX:false:AES256:1500:true:true::third party\\:1.2.3.4\\:\\:\\:\\:" ], "value" : "", "tstamp" : "2016-03-29T07:05:27.213013Z" }
        ]
      }
    ]
  },

[...]

[...]

Issue Links

• duplicates: CASSANDRA-12144 Undeletable / duplicate rows after upgrading from 2.2.4 to 3.0.7
• is duplicated by: CASSANDRA-11900 Cassandra will not start after upgrade to 3.3
• is related to: CASSANDRA-13125 Duplicate rows after upgrading from 2.1.16 to 3.0.10/3.9
• relates to: CASSANDRA-11299 AssertionError when querying by secondary index