[IGNITE-12734] Scan query shutting down the node in some cases

• Type: Bug
• Status: Resolved
• Priority: Blocker
• Resolution: Fixed
• Fix Version/s: 2.9, 2.8.1
• Assignee: Aleksey Plekhanov
• Reporter: Aleksey Plekhanov
• Created: 02/Mar/20 13:15
• Updated: 18/Mar/20 10:18
• Resolved: 18/Mar/20 10:18

Description

Reproducer:

public class CachePartitionEvictionQueryTest extends GridCommonAbstractTest {
    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
        return super.getConfiguration(name).setDataStorageConfiguration(new DataStorageConfiguration()
            .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true)));
    }

    @Override protected FailureHandler getFailureHandler(String igniteInstanceName) {
        return new StopNodeFailureHandler();
    }

    @Test
    public void testQuery() throws Exception {
        startGrid(getConfiguration(getTestIgniteInstanceName(0)).setConsistentId("1"));

        grid(0).cluster().active(true);

        grid(0).cluster().baselineAutoAdjustEnabled(true);
        grid(0).cluster().baselineAutoAdjustTimeout(0);

        IgniteCache<Integer, Integer> cache = grid(0).getOrCreateCache(
            new CacheConfiguration<Integer, Integer>(DEFAULT_CACHE_NAME).setBackups(0)
                .setAffinity(new RendezvousAffinityFunction().setPartitions(2)));

        cache.put(0, 0);
        cache.put(1, 1);

        Iterator iter = cache.query(new ScanQuery<>().setPageSize(1)).iterator();

        iter.next();

        startGrid(getConfiguration(getTestIgniteInstanceName(1)).setConsistentId("0"));

        awaitPartitionMapExchange();

        forceCheckpoint(grid(0));

        iter.next();
    }
}

Node fails with the reason:

[2020-03-02 15:17:46,663][ERROR][test-runner-#1%cache.CachePartitionEvictionQueryTest%][IgniteTestResources] Critical system error detected. Will be handled accordingly to configured handler [hnd=StopNodeFailureHandler [super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], failureCtx=FailureContext [type=CRITICAL_ERROR, err=class o.a.i.i.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted [pages(groupId, pageId)=[], msg=Runtime failure on bounds: [lower=null, upper=null]]]]
class org.apache.ignite.internal.processors.cache.persistence.tree.CorruptedTreeException: B+Tree is corrupted [pages(groupId, pageId)=[], msg=Runtime failure on bounds: [lower=null, upper=null]]
    at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.corruptedTreeException(BPlusTree.java:5927)
    at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1054)
    at org.apache.ignite.internal.processors.cache.tree.CacheDataTree.find(CacheDataTree.java:164)
    at org.apache.ignite.internal.processors.cache.tree.CacheDataTree.find(CacheDataTree.java:63)
    at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1021)
    at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.cursor(IgniteCacheOffheapManagerImpl.java:2914)
    at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.cursor(IgniteCacheOffheapManagerImpl.java:2884)
    at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.cursor(IgniteCacheOffheapManagerImpl.java:2878)
    at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$CacheDataStoreImpl.cursor(IgniteCacheOffheapManagerImpl.java:2866)
    at org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$GridCacheDataStore.cursor(GridCacheOffheapManager.java:2560)
    at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$3.onHasNext(IgniteCacheOffheapManagerImpl.java:937)
    at org.apache.ignite.internal.util.GridCloseableIteratorAdapter.hasNextX(GridCloseableIteratorAdapter.java:53)
    at org.apache.ignite.internal.util.lang.GridIteratorAdapter.hasNext(GridIteratorAdapter.java:45)
    at org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$ScanQueryIterator.advance(GridCacheQueryManager.java:3031)
    at org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$ScanQueryIterator.onHasNext(GridCacheQueryManager.java:2997)
    at org.apache.ignite.internal.util.GridCloseableIteratorAdapter.hasNextX(GridCloseableIteratorAdapter.java:53)
    at org.apache.ignite.internal.util.GridCloseableIteratorAdapter.nextX(GridCloseableIteratorAdapter.java:38)
    at org.apache.ignite.internal.util.lang.GridIteratorAdapter.next(GridIteratorAdapter.java:35)
    at org.apache.ignite.internal.processors.cache.AutoClosableCursorIterator.next(AutoClosableCursorIterator.java:59)
    at org.apache.ignite.internal.processors.cache.CachePartitionEvictionQueryTest.testQuery2(CachePartitionEvictionQueryTest.java:78)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.apache.ignite.testframework.junits.GridAbstractTest$7.run(GridAbstractTest.java:2090)
    at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.AssertionError: FullPageId [pageId=0001000100000002, effectivePageId=0000000100000002, grpId=1544803905]
    at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:820)
    at org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl.acquirePage(PageMemoryImpl.java:709)
    at org.apache.ignite.internal.processors.cache.persistence.DataStructure.acquirePage(DataStructure.java:158)
    at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.acquirePage(BPlusTree.java:5858)
    at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.findLowerUnbounded(BPlusTree.java:974)
    at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.find(BPlusTree.java:1039)
    ... 28 more

Issue Links

• links to: GitHub Pull Request #7494