Uploaded image for project: 'Cassandra'
  1. Cassandra
  2. CASSANDRA-11120

Leak Detected while bringing nodes up and down when under stress.

    XMLWordPrintableJSON

    Details

    • Type: Bug
    • Status: Resolved
    • Priority: Low
    • Resolution: Fixed
    • Fix Version/s: 3.0.4, 3.4
    • Component/s: Legacy/Core
    • Labels:
      None
    • Severity:
      Low

      Description

      So after CASSANDRA-10688 has been fixed I'm able to reproduce some leaks consistently with my stress test suite that is doing repairs/stress/bringing nodes up and down.
      One such example:

      ERROR [Strong-Reference-Leak-Detector:1] 2016-02-03 23:32:38,827  NoSpamLogger.java:97 - Strong self-ref loop detected [/var/lib/cassandra/data/keyspace1/standard1-7cddd4c1cacc11e5aa69f375b464842d/ma-14-big, private java.util.concurrent.ScheduledFuture org.apache.cassandra.io.sstable.format.SSTableReader$GlobalTidy.readMeterSyncFuture-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask, final java.util.concurrent.ScheduledThreadPoolExecutor java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.this$0-java.util.concurrent.ScheduledThreadPoolExecutor, private final java.util.HashSet java.util.concurrent.ThreadPoolExecutor.workers-java.util.HashSet, private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap, transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;, transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node, final java.lang.Object java.util.HashMap$Node.key-java.util.concurrent.ThreadPoolExecutor$Worker, final java.lang.Thread java.util.concurrent.ThreadPoolExecutor$Worker.thread-java.lang.Thread, private java.lang.ThreadGroup java.lang.Thread.group-java.lang.ThreadGroup, private final java.lang.ThreadGroup java.lang.ThreadGroup.parent-java.lang.ThreadGroup, java.lang.Thread[] java.lang.ThreadGroup.threads-[Ljava.lang.Thread;, java.lang.Thread[] java.lang.ThreadGroup.threads-java.lang.Thread, private java.lang.Runnable java.lang.Thread.target-java.util.concurrent.ThreadPoolExecutor$Worker, final java.util.concurrent.ThreadPoolExecutor java.util.concurrent.ThreadPoolExecutor$Worker.this$0-java.util.concurrent.ScheduledThreadPoolExecutor, private final java.util.concurrent.BlockingQueue java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue, private final java.util.concurrent.BlockingQueue java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask, private java.util.concurrent.Callable java.util.concurrent.FutureTask.callable-java.util.concurrent.Executors$RunnableAdapter, final java.lang.Runnable java.util.concurrent.Executors$RunnableAdapter.task-sun.rmi.transport.DGCImpl$1, final sun.rmi.transport.DGCImpl sun.rmi.transport.DGCImpl$1.this$0-sun.rmi.transport.DGCImpl, private java.util.Map sun.rmi.transport.DGCImpl.leaseTable-java.util.HashMap, transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;, transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node, java.lang.Object java.util.HashMap$Node.value-sun.rmi.transport.DGCImpl$LeaseInfo, java.util.Set sun.rmi.transport.DGCImpl$LeaseInfo.notifySet-java.util.HashSet, private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap, transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;, transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node, final java.lang.Object java.util.HashMap$Node.key-sun.rmi.transport.Target, private final sun.rmi.transport.WeakRef sun.rmi.transport.Target.weakImpl-sun.rmi.transport.WeakRef, private java.lang.Object sun.rmi.transport.WeakRef.strongRef-javax.management.remote.rmi.RMIJRMPServerImpl, private final java.util.List javax.management.remote.rmi.RMIServerImpl.clientList-java.util.ArrayList, transient java.lang.Object[] java.util.ArrayList.elementData-[Ljava.lang.Object;, transient java.lang.Object[] java.util.ArrayList.elementData-java.lang.ref.WeakReference, private java.lang.Object java.lang.ref.Reference.referent-javax.management.remote.rmi.RMIConnectionImpl, private final javax.management.MBeanServer javax.management.remote.rmi.RMIConnectionImpl.mbeanServer-com.sun.jmx.mbeanserver.JmxMBeanServer, private volatile javax.management.MBeanServer com.sun.jmx.mbeanserver.JmxMBeanServer.mbsInterceptor-com.sun.jmx.interceptor.DefaultMBeanServerInterceptor, private final transient com.sun.jmx.mbeanserver.Repository com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.repository-com.sun.jmx.mbeanserver.Repository, private final java.util.Map com.sun.jmx.mbeanserver.Repository.domainTb-java.util.HashMap, transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;, transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node, java.lang.Object java.util.HashMap$Node.value-java.util.HashMap, transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;, transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node, java.lang.Object java.util.HashMap$Node.value-com.sun.jmx.mbeanserver.NamedObject, private final javax.management.DynamicMBean com.sun.jmx.mbeanserver.NamedObject.object-com.sun.jmx.mbeanserver.StandardMBeanSupport, private final java.lang.Object com.sun.jmx.mbeanserver.MBeanSupport.resource-org.apache.cassandra.db.ColumnFamilyStore, public final org.apache.cassandra.db.Keyspace org.apache.cassandra.db.ColumnFamilyStore.keyspace-org.apache.cassandra.db.Keyspace, private final java.util.concurrent.ConcurrentMap org.apache.cassandra.db.Keyspace.columnFamilyStores-java.util.concurrent.ConcurrentHashMap, private final java.util.concurrent.ConcurrentMap org.apache.cassandra.db.Keyspace.columnFamilyStores-org.apache.cassandra.db.ColumnFamilyStore, private final java.util.concurrent.ScheduledFuture org.apache.cassandra.db.ColumnFamilyStore.latencyCalculator-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask, final java.util.concurrent.ScheduledThreadPoolExecutor java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.this$0-org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor, private final java.util.concurrent.BlockingQueue java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue, private final java.util.concurrent.BlockingQueue java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask, private java.util.concurrent.Callable java.util.concurrent.FutureTask.callable-java.util.concurrent.Executors$RunnableAdapter, final java.lang.Runnable java.util.concurrent.Executors$RunnableAdapter.task-org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor$UncomplainingRunnable, private final java.lang.Runnable org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor$UncomplainingRunnable.runnable-org.apache.cassandra.db.ColumnFamilyStore$3, final org.apache.cassandra.db.ColumnFamilyStore org.apache.cassandra.db.ColumnFamilyStore$3.this$0-org.apache.cassandra.db.ColumnFamilyStore, public final org.apache.cassandra.db.Keyspace org.apache.cassandra.db.ColumnFamilyStore.keyspace-org.apache.cassandra.db.Keyspace, private final java.util.concurrent.ConcurrentMap org.apache.cassandra.db.Keyspace.columnFamilyStores-java.util.concurrent.ConcurrentHashMap, private final java.util.concurrent.ConcurrentMap org.apache.cassandra.db.Keyspace.columnFamilyStores-org.apache.cassandra.db.ColumnFamilyStore, private final org.apache.cassandra.db.lifecycle.Tracker org.apache.cassandra.db.ColumnFamilyStore.data-org.apache.cassandra.db.lifecycle.Tracker, public final java.util.Collection org.apache.cassandra.db.lifecycle.Tracker.subscribers-java.util.concurrent.CopyOnWriteArrayList, public final java.util.Collection org.apache.cassandra.db.lifecycle.Tracker.subscribers-org.apache.cassandra.db.compaction.CompactionStrategyManager, private volatile org.apache.cassandra.db.compaction.AbstractCompactionStrategy org.apache.cassandra.db.compaction.CompactionStrategyManager.repaired-org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy, private final java.util.Set org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy.sstables-java.util.HashSet, private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap, transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;, transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node, final java.lang.Object java.util.HashMap$Node.key-org.apache.cassandra.io.sstable.format.big.BigTableReader, private final org.apache.cassandra.io.sstable.format.SSTableReader$InstanceTidier org.apache.cassandra.io.sstable.format.SSTableReader.tidy-org.apache.cassandra.io.sstable.format.SSTableReader$InstanceTidier, private org.apache.cassandra.utils.concurrent.Ref org.apache.cassandra.io.sstable.format.SSTableReader$InstanceTidier.globalRef-org.apache.cassandra.utils.concurrent.Ref]
      

      CC Benedict Elliott Smith

        Attachments

          Activity

            People

            • Assignee:
              aweisberg Ariel Weisberg
              Reporter:
              jjordan Jeremiah Jordan
              Authors:
              Ariel Weisberg
              Reviewers:
              Jeremiah Jordan
            • Votes:
              0 Vote for this issue
              Watchers:
              6 Start watching this issue

              Dates

              • Created:
                Updated:
                Resolved: