Segment tar corruption we-retail stage 03.09.2018 08:16:02.705 *ERROR* [Apache Sling Repository Startup Thread] org.apache.jackrabbit.oak.segment.SegmentNotFoundExceptionListener Segment not found: 2b94e3a5-6284-4e94-ae59-53487c70c1d9. SegmentId age=0ms org.apache.jackrabbit.oak.segment.SegmentNotFoundException: Segment 2b94e3a5-6284-4e94-ae59-53487c70c1d9 not found at org.apache.jackrabbit.oak.segment.file.AbstractFileStore.readSegmentUncached(AbstractFileStore.java:276) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.file.FileStore.lambda$readSegment$5(FileStore.java:477) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.SegmentCache.lambda$getSegment$0(SegmentCache.java:116) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at com.google.common.cache.LocalCache$LocalManualCache$1.load(LocalCache.java:4724) [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002] at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3522) [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002] at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2315) [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002] at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2278) [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002] at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2193) [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002] at com.google.common.cache.LocalCache.get(LocalCache.java:3932) [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002] at com.google.common.cache.LocalCache$LocalManualCache.get(LocalCache.java:4721) [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002] at org.apache.jackrabbit.oak.segment.SegmentCache.getSegment(SegmentCache.java:113) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.file.FileStore.readSegment(FileStore.java:477) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.SegmentId.getSegment(SegmentId.java:154) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.SegmentId.getGcGeneration(SegmentId.java:246) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter$SegmentWriteOperation.isOldGeneration(DefaultSegmentWriter.java:991) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter$SegmentWriteOperation.deduplicateNode(DefaultSegmentWriter.java:958) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter$SegmentWriteOperation.writeNode(DefaultSegmentWriter.java:796) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter$SegmentWriteOperation.access$800(DefaultSegmentWriter.java:258) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter$8.execute(DefaultSegmentWriter.java:247) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.SegmentBufferWriterPool.execute(SegmentBufferWriterPool.java:101) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter.writeNode(DefaultSegmentWriter.java:243) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.SegmentWriter.writeNode(SegmentWriter.java:141) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.SegmentNodeBuilder.getNodeState(SegmentNodeBuilder.java:132) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.SegmentNodeBuilder.getNodeState(SegmentNodeBuilder.java:42) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder.(IndexDefinitionBuilder.java:74) [org.apache.jackrabbit.oak-lucene:1.8.2] at org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder.(IndexDefinitionBuilder.java:68) [org.apache.jackrabbit.oak-lucene:1.8.2] at com.adobe.granite.repository.impl.GraniteContent.configureRepTokenIndex(GraniteContent.java:790) [com.adobe.granite.repository:1.4.88] at com.adobe.granite.repository.impl.GraniteContent.initialize(GraniteContent.java:264) [com.adobe.granite.repository:1.4.88] at org.apache.jackrabbit.oak.spi.lifecycle.CompositeInitializer.initialize(CompositeInitializer.java:48) [org.apache.jackrabbit.oak-store-spi:1.8.2] at org.apache.jackrabbit.oak.OakInitializer.initialize(OakInitializer.java:53) [org.apache.jackrabbit.oak-core:1.8.2] at org.apache.jackrabbit.oak.Oak.createNewContentRepository(Oak.java:687) [org.apache.jackrabbit.oak-core:1.8.2] at org.apache.jackrabbit.oak.Oak.createContentRepository(Oak.java:675) [org.apache.jackrabbit.oak-core:1.8.2] at org.apache.jackrabbit.oak.jcr.Jcr.createContentRepository(Jcr.java:376) [org.apache.jackrabbit.oak-jcr:1.8.2] at com.adobe.granite.repository.impl.SlingRepositoryManager.createRepository(SlingRepositoryManager.java:427) [com.adobe.granite.repository:1.4.88] at com.adobe.granite.repository.impl.SlingRepositoryManager.acquireRepository(SlingRepositoryManager.java:274) [com.adobe.granite.repository:1.4.88] at org.apache.sling.jcr.base.AbstractSlingRepositoryManager.initializeAndRegisterRepositoryService(AbstractSlingRepositoryManager.java:471) [org.apache.sling.jcr.base:3.0.4] at org.apache.sling.jcr.base.AbstractSlingRepositoryManager.access$300(AbstractSlingRepositoryManager.java:85) [org.apache.sling.jcr.base:3.0.4] at org.apache.sling.jcr.base.AbstractSlingRepositoryManager$4.run(AbstractSlingRepositoryManager.java:455) [org.apache.sling.jcr.base:3.0.4] 03.09.2018 08:16:02.706 *ERROR* [Apache Sling Repository Startup Thread] com.adobe.granite.repository.impl.SlingRepositoryManager start: Uncaught Throwable trying to access Repository, calling stopRepository() org.apache.jackrabbit.oak.segment.SegmentNotFoundException: Cannot copy record from a generation that has been gc'ed already at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter$SegmentWriteOperation.isOldGeneration(DefaultSegmentWriter.java:1008) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter$SegmentWriteOperation.deduplicateNode(DefaultSegmentWriter.java:958) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter$SegmentWriteOperation.writeNode(DefaultSegmentWriter.java:796) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter$SegmentWriteOperation.access$800(DefaultSegmentWriter.java:258) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter$8.execute(DefaultSegmentWriter.java:247) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.SegmentBufferWriterPool.execute(SegmentBufferWriterPool.java:101) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter.writeNode(DefaultSegmentWriter.java:243) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.SegmentWriter.writeNode(SegmentWriter.java:141) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.SegmentNodeBuilder.getNodeState(SegmentNodeBuilder.java:132) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.SegmentNodeBuilder.getNodeState(SegmentNodeBuilder.java:42) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder.(IndexDefinitionBuilder.java:74) [org.apache.jackrabbit.oak-lucene:1.8.2] at org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder.(IndexDefinitionBuilder.java:68) [org.apache.jackrabbit.oak-lucene:1.8.2] at com.adobe.granite.repository.impl.GraniteContent.configureRepTokenIndex(GraniteContent.java:790) [com.adobe.granite.repository:1.4.88] at com.adobe.granite.repository.impl.GraniteContent.initialize(GraniteContent.java:264) [com.adobe.granite.repository:1.4.88] at org.apache.jackrabbit.oak.spi.lifecycle.CompositeInitializer.initialize(CompositeInitializer.java:48) [org.apache.jackrabbit.oak-store-spi:1.8.2] at org.apache.jackrabbit.oak.OakInitializer.initialize(OakInitializer.java:53) [org.apache.jackrabbit.oak-core:1.8.2] at org.apache.jackrabbit.oak.Oak.createNewContentRepository(Oak.java:687) [org.apache.jackrabbit.oak-core:1.8.2] at org.apache.jackrabbit.oak.Oak.createContentRepository(Oak.java:675) [org.apache.jackrabbit.oak-core:1.8.2] at org.apache.jackrabbit.oak.jcr.Jcr.createContentRepository(Jcr.java:376) [org.apache.jackrabbit.oak-jcr:1.8.2] at com.adobe.granite.repository.impl.SlingRepositoryManager.createRepository(SlingRepositoryManager.java:427) [com.adobe.granite.repository:1.4.88] at com.adobe.granite.repository.impl.SlingRepositoryManager.acquireRepository(SlingRepositoryManager.java:274) [com.adobe.granite.repository:1.4.88] at org.apache.sling.jcr.base.AbstractSlingRepositoryManager.initializeAndRegisterRepositoryService(AbstractSlingRepositoryManager.java:471) [org.apache.sling.jcr.base:3.0.4] at org.apache.sling.jcr.base.AbstractSlingRepositoryManager.access$300(AbstractSlingRepositoryManager.java:85) [org.apache.sling.jcr.base:3.0.4] at org.apache.sling.jcr.base.AbstractSlingRepositoryManager$4.run(AbstractSlingRepositoryManager.java:455) [org.apache.sling.jcr.base:3.0.4] Caused by: org.apache.jackrabbit.oak.segment.SegmentNotFoundException: Segment 2b94e3a5-6284-4e94-ae59-53487c70c1d9 not found at org.apache.jackrabbit.oak.segment.file.AbstractFileStore.readSegmentUncached(AbstractFileStore.java:276) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.file.FileStore.lambda$readSegment$5(FileStore.java:477) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.SegmentCache.lambda$getSegment$0(SegmentCache.java:116) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at com.google.common.cache.LocalCache$LocalManualCache$1.load(LocalCache.java:4724) [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002] at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3522) [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002] at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2315) [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002] at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2278) [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002] at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2193) [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002] at com.google.common.cache.LocalCache.get(LocalCache.java:3932) [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002] at com.google.common.cache.LocalCache$LocalManualCache.get(LocalCache.java:4721) [com.adobe.granite.osgi.wrapper.guava:15.0.0.0002] at org.apache.jackrabbit.oak.segment.SegmentCache.getSegment(SegmentCache.java:113) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.file.FileStore.readSegment(FileStore.java:477) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.SegmentId.getSegment(SegmentId.java:154) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.SegmentId.getGcGeneration(SegmentId.java:246) [org.apache.jackrabbit.oak-segment-tar:1.8.2] at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter$SegmentWriteOperation.isOldGeneration(DefaultSegmentWriter.java:991) [org.apache.jackrabbit.oak-segment-tar:1.8.2] ... 23 common frames omitted java -Dtar.memoryMapped=true -Xms4g -Xms4g -jar bin/oak-run-1.8.2.jar check repository/segmentstore --head Apache Jackrabbit Oak 1.8.2 Checking revision 61dc7445-cb40-46f4-aa58-5870b9611773:318 Checking head Checking / Error while traversing /home/users: org.apache.jackrabbit.oak.segment.SegmentNotFoundException: Segment 2a73fb36-c616-4e6b-a519-464310abe98b not found Checked 69 nodes and 197 properties Checking revision 9d095d94-eeb4-4edd-a4d5-20a24e4741b3:45 Checking head com.google.common.util.concurrent.UncheckedExecutionException: org.apache.jackrabbit.oak.segment.SegmentNotFoundException: Segment de757025-4601-48a8-a432-425ac8b2d7eb not found at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2199) at com.google.common.cache.LocalCache.get(LocalCache.java:3932) at com.google.common.cache.LocalCache$LocalManualCache.get(LocalCache.java:4721) at org.apache.jackrabbit.oak.segment.SegmentCache.getSegment(SegmentCache.java:113) at org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore.readSegment(ReadOnlyFileStore.java:114) at org.apache.jackrabbit.oak.segment.SegmentId.getSegment(SegmentId.java:154) at org.apache.jackrabbit.oak.segment.Record.getSegment(Record.java:70) at org.apache.jackrabbit.oak.segment.SegmentNodeState.getTemplateId(SegmentNodeState.java:101) at org.apache.jackrabbit.oak.segment.SegmentNodeState.getTemplate(SegmentNodeState.java:110) at org.apache.jackrabbit.oak.segment.SegmentNodeState.hasChildNode(SegmentNodeState.java:409) at org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker$NodeWrapper.deriveTraversableNodeOnPath(ConsistencyChecker.java:497) at org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker.checkPathAtRoot(ConsistencyChecker.java:382) at org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker.checkPathsAtRoot(ConsistencyChecker.java:352) at org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker.checkConsistency(ConsistencyChecker.java:199) at org.apache.jackrabbit.oak.segment.tool.Check.run(Check.java:243) at org.apache.jackrabbit.oak.run.CheckCommand.execute(CheckCommand.java:95) at org.apache.jackrabbit.oak.run.Main.main(Main.java:49) Caused by: org.apache.jackrabbit.oak.segment.SegmentNotFoundException: Segment de757025-4601-48a8-a432-425ac8b2d7eb not found at org.apache.jackrabbit.oak.segment.file.AbstractFileStore.readSegmentUncached(AbstractFileStore.java:276) at org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore$1.call(ReadOnlyFileStore.java:117) at org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore$1.call(ReadOnlyFileStore.java:114) at org.apache.jackrabbit.oak.segment.SegmentCache.lambda$getSegment$0(SegmentCache.java:116) at com.google.common.cache.LocalCache$LocalManualCache$1.load(LocalCache.java:4724) at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3522) at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2315) at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2278) at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2193) ... 16 more java -Dtar.memoryMapped=true -Xms4g -Xms4g -jar bin/oak-run-1.8.2.jar check repository/segmentstore Apache Jackrabbit Oak 1.8.2 Checking revision 61dc7445-cb40-46f4-aa58-5870b9611773:318 Checking head Checking / Error while traversing /home/users: org.apache.jackrabbit.oak.segment.SegmentNotFoundException: Segment 2a73fb36-c616-4e6b-a519-464310abe98b not found Checked 69 nodes and 197 properties Checking checkpoints Checking checkpoint 1fe1a384-08ed-42f1-8956-a2fe27bb3f95 Checking checkpoint eb91a6df-ca57-4edf-99a2-59ea21c263f6 Checking revision 9d095d94-eeb4-4edd-a4d5-20a24e4741b3:45 Checking head com.google.common.util.concurrent.UncheckedExecutionException: org.apache.jackrabbit.oak.segment.SegmentNotFoundException: Segment de757025-4601-48a8-a432-425ac8b2d7eb not found at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2199) at com.google.common.cache.LocalCache.get(LocalCache.java:3932) at com.google.common.cache.LocalCache$LocalManualCache.get(LocalCache.java:4721) at org.apache.jackrabbit.oak.segment.SegmentCache.getSegment(SegmentCache.java:113) at org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore.readSegment(ReadOnlyFileStore.java:114) at org.apache.jackrabbit.oak.segment.SegmentId.getSegment(SegmentId.java:154) at org.apache.jackrabbit.oak.segment.Record.getSegment(Record.java:70) at org.apache.jackrabbit.oak.segment.SegmentNodeState.getTemplateId(SegmentNodeState.java:101) at org.apache.jackrabbit.oak.segment.SegmentNodeState.getTemplate(SegmentNodeState.java:110) at org.apache.jackrabbit.oak.segment.SegmentNodeState.hasChildNode(SegmentNodeState.java:409) at org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker$NodeWrapper.deriveTraversableNodeOnPath(ConsistencyChecker.java:497) at org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker.checkPathAtRoot(ConsistencyChecker.java:382) at org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker.checkPathsAtRoot(ConsistencyChecker.java:352) at org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker.checkConsistency(ConsistencyChecker.java:199) at org.apache.jackrabbit.oak.segment.tool.Check.run(Check.java:243) at org.apache.jackrabbit.oak.run.CheckCommand.execute(CheckCommand.java:95) at org.apache.jackrabbit.oak.run.Main.main(Main.java:49) Caused by: org.apache.jackrabbit.oak.segment.SegmentNotFoundException: Segment de757025-4601-48a8-a432-425ac8b2d7eb not found at org.apache.jackrabbit.oak.segment.file.AbstractFileStore.readSegmentUncached(AbstractFileStore.java:276) at org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore$1.call(ReadOnlyFileStore.java:117) at org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore$1.call(ReadOnlyFileStore.java:114) at org.apache.jackrabbit.oak.segment.SegmentCache.lambda$getSegment$0(SegmentCache.java:116) at com.google.common.cache.LocalCache$LocalManualCache$1.load(LocalCache.java:4724) at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3522) at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2315) at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2278) at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2193) ... 16 more