Uploaded image for project: 'Cassandra'
  1. Cassandra
  2. CASSANDRA-16049

org.apache.cassandra.distributed.test.ReadRepairTest#movingTokenReadRepairTest passes because the message drop filters are not dropping the messages

    XMLWordPrintableJSON

    Details

      Description

      The test tries to block messages from node 4 to 3 but used verb ordinal rather than id; this causes the verbs to sent to node 3 and allow all replies to happen.

      If you fix the test to use id (and actually filter), then the test fails

      Patch to show the issue

      diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java
      index f0c82b85e1..55d2e8f303 100644
      --- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java
      +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java
      @@ -30,14 +30,14 @@ import org.apache.cassandra.dht.Token;
      import org.apache.cassandra.distributed.Cluster;
      import org.apache.cassandra.distributed.api.ConsistencyLevel;
      import org.apache.cassandra.distributed.api.ICluster;
      -import org.apache.cassandra.distributed.shared.NetworkTopology;
      import org.apache.cassandra.locator.InetAddressAndPort;
      import org.apache.cassandra.service.PendingRangeCalculatorService;
      import org.apache.cassandra.service.StorageService;
      
      +import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
      +import static org.apache.cassandra.distributed.shared.AssertUtils.row;
      import static org.apache.cassandra.net.Verb.READ_REPAIR_REQ;
      import static org.apache.cassandra.net.Verb.READ_REQ;
      -import static org.apache.cassandra.distributed.shared.AssertUtils.*;
      
      public class ReadRepairTest extends TestBaseImpl
      {
      @@ -115,8 +115,8 @@ public class ReadRepairTest extends TestBaseImpl
      
                  // prevent #4 from reading or writing to #3, so our QUORUM must contain #2 and #4
                  // since #1 is taking over the range, this means any read-repair must make it to #1 as well
      -            cluster.filters().verbs(READ_REQ.ordinal()).from(4).to(3).drop();
      -            cluster.filters().verbs(READ_REPAIR_REQ.ordinal()).from(4).to(3).drop();
      +            cluster.filters().verbs(READ_REQ.id).from(4).to(3).drop();
      +            cluster.filters().verbs(READ_REPAIR_REQ.id).from(4).to(3).drop();
                  assertRows(cluster.coordinator(4).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?",
                                                            ConsistencyLevel.ALL, i),
                             row(i, 1, 1));
      

      Exception

      org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out - received only 2 responses.
      
      	at org.apache.cassandra.service.reads.ReadCallback.awaitResults(ReadCallback.java:120)
      	at org.apache.cassandra.service.reads.AbstractReadExecutor.awaitResponses(AbstractReadExecutor.java:373)
      	at org.apache.cassandra.service.StorageProxy.fetchRows(StorageProxy.java:1823)
      	at org.apache.cassandra.service.StorageProxy.readRegular(StorageProxy.java:1713)
      	at org.apache.cassandra.service.StorageProxy.read(StorageProxy.java:1630)
      	at org.apache.cassandra.db.SinglePartitionReadCommand$Group.execute(SinglePartitionReadCommand.java:1123)
      	at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:294)
      	at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:246)
      	at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:88)
      	at org.apache.cassandra.distributed.impl.Coordinator.executeInternal(Coordinator.java:100)
      	at org.apache.cassandra.distributed.impl.Coordinator.lambda$executeWithResult$0(Coordinator.java:62)
      	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
      	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
      	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
      	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
      	at java.lang.Thread.run(Thread.java:748)
      

        Attachments

          Activity

            People

            • Assignee:
              samt Sam Tunnicliffe
              Reporter:
              dcapwell David Capwell
              Authors:
              Sam Tunnicliffe
              Reviewers:
              Ekaterina Dimitrova
            • Votes:
              0 Vote for this issue
              Watchers:
              3 Start watching this issue

              Dates

              • Created:
                Updated:
                Resolved: