[HUDI-6842] Fix flaky testHoodieAsyncClusteringJobWithScheduleAndExecute

• Type: Test
• Status: Closed
• Priority: Major
• Resolution: Fixed
• Fix Version/s: 0.14.0
• Component/s: tests-ci
• Labels: pull-request-available
• Epic Link: CI Instability
• Assignee: Unassigned
• Reporter: sivabalan narayanan
• Created: 10/Sep/23 17:59
• Updated: 12/Sep/23 10:20
• Resolved: 12/Sep/23 10:20

Description

Ref link: 

https://dev.azure.com/apache-hudi-ci-org/785b6ef4-2f42-4a89-8f0e-5f0d7039a0cc/_apis/build/builds/19772/logs/24

 

2023-09-09T04:44:43.5803915Z [ERROR] Tests run: 122, Failures: 0, Errors: 1, Skipped: 2, Time elapsed: 1,504.981 s <<< FAILURE! - in org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer
2023-09-09T04:44:43.5804824Z [ERROR] testHoodieAsyncClusteringJobWithScheduleAndExecute{String, HoodieRecordType}[4]  Time elapsed: 6.302 s  <<< ERROR!
2023-09-09T04:44:43.5805451Z java.util.ConcurrentModificationException
2023-09-09T04:44:43.5805815Z 	at java.util.HashMap$ValueSpliterator.forEachRemaining(HashMap.java:1657)
2023-09-09T04:44:43.5806180Z 	at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
2023-09-09T04:44:43.5806597Z 	at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
2023-09-09T04:44:43.5807002Z 	at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
2023-09-09T04:44:43.5807407Z 	at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
2023-09-09T04:44:43.5807810Z 	at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
2023-09-09T04:44:43.5808217Z 	at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
2023-09-09T04:44:43.5808637Z 	at org.apache.hudi.client.heartbeat.HoodieHeartbeatClient.stopHeartbeatTimers(HoodieHeartbeatClient.java:202)
2023-09-09T04:44:43.5809066Z 	at org.apache.hudi.client.heartbeat.HoodieHeartbeatClient.close(HoodieHeartbeatClient.java:283)
2023-09-09T04:44:43.5809640Z 	at org.apache.hudi.client.BaseHoodieClient.close(BaseHoodieClient.java:110)
2023-09-09T04:44:43.5810197Z 	at org.apache.hudi.client.BaseHoodieWriteClient.close(BaseHoodieWriteClient.java:1361)
2023-09-09T04:44:43.5810599Z 	at org.apache.hudi.utilities.streamer.StreamSync.close(StreamSync.java:1136)
2023-09-09T04:44:43.5811593Z 	at org.apache.hudi.utilities.streamer.HoodieStreamer$StreamSyncService.close(HoodieStreamer.java:925)
2023-09-09T04:44:43.5812081Z 	at org.apache.hudi.utilities.streamer.HoodieStreamer.lambda$shutdownGracefully$0(HoodieStreamer.java:193)
2023-09-09T04:44:43.5812519Z 	at org.apache.hudi.common.util.Option.ifPresent(Option.java:97)
2023-09-09T04:44:43.5812913Z 	at org.apache.hudi.utilities.streamer.HoodieStreamer.shutdownGracefully(HoodieStreamer.java:189)
2023-09-09T04:44:43.5813396Z 	at org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.deltaStreamerTestRunner(TestHoodieDeltaStreamer.java:981)
2023-09-09T04:44:43.5813914Z 	at org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.deltaStreamerTestRunner(TestHoodieDeltaStreamer.java:965)
2023-09-09T04:44:43.5814584Z 	at org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.deltaStreamerTestRunner(TestHoodieDeltaStreamer.java:1010)
2023-09-09T04:44:43.5815699Z 	at org.apache.hudi.utilities.deltastreamer.TestHoodieDeltaStreamer.testHoodieAsyncClusteringJobWithScheduleAndExecute(TestHoodieDeltaStreamer.java:1494)
2023-09-09T04:44:43.5817553Z 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2023-09-09T04:44:43.5818015Z 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
2023-09-09T04:44:43.5818460Z 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2023-09-09T04:44:43.5818989Z 	at java.lang.reflect.Method.invoke(Method.java:498)
2023-09-09T04:44:43.5905758Z 	at org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
2023-09-09T04:44:43.5906308Z 	at org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
2023-09-09T04:44:43.5906773Z 	at org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
2023-09-09T04:44:43.5907667Z 	at org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
2023-09-09T04:44:43.5908288Z 	at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:140)
2023-09-09T04:44:43.5908765Z 	at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestTemplateMethod(TimeoutExtension.java:92)
2023-09-09T04:44:43.5909433Z 	at org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
2023-09-09T04:44:43.5909933Z 	at org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
2023-09-09T04:44:43.5910426Z 	at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
2023-09-09T04:44:43.5910921Z 	at org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
2023-09-09T04:44:43.5911397Z 	at org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
2023-09-09T04:44:43.5911883Z 	at org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
2023-09-09T04:44:43.5912331Z 	at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
2023-09-09T04:44:43.5912753Z 	at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
2023-09-09T04:44:43.5913344Z 	at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$6(TestMethodTestDescriptor.java:210)
2023-09-09T04:44:43.5913992Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-09-09T04:44:43.5915191Z 	at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:206)
2023-09-09T04:44:43.5915678Z 	at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:131)
2023-09-09T04:44:43.5916148Z 	at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:65)
2023-09-09T04:44:43.5916605Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
2023-09-09T04:44:43.5917058Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-09-09T04:44:43.5917506Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
2023-09-09T04:44:43.5917933Z 	at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
2023-09-09T04:44:43.5918357Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
2023-09-09T04:44:43.5918808Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-09-09T04:44:43.5919259Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
2023-09-09T04:44:43.5919690Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
2023-09-09T04:44:43.5920170Z 	at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
2023-09-09T04:44:43.5920687Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask$DefaultDynamicTestExecutor.execute(NodeTestTask.java:212)
2023-09-09T04:44:43.5921967Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask$DefaultDynamicTestExecutor.execute(NodeTestTask.java:192)
2023-09-09T04:44:43.5922523Z 	at org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.execute(TestTemplateTestDescriptor.java:139)
2023-09-09T04:44:43.5923012Z 	at org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.lambda$execute$2(TestTemplateTestDescriptor.java:107)
2023-09-09T04:44:43.5923686Z 	at java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
2023-09-09T04:44:43.5924107Z 	at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-09-09T04:44:43.5928548Z 	at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175)
2023-09-09T04:44:43.5928863Z 	at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-09-09T04:44:43.5929352Z 	at java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
2023-09-09T04:44:43.5929866Z 	at java.util.stream.ReferencePipeline$11$1.accept(ReferencePipeline.java:440)
2023-09-09T04:44:43.5930156Z 	at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-09-09T04:44:43.5930441Z 	at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-09-09T04:44:43.5930986Z 	at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-09-09T04:44:43.5931499Z 	at java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
2023-09-09T04:44:43.5931801Z 	at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-09-09T04:44:43.5932080Z 	at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-09-09T04:44:43.5932382Z 	at java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948)
2023-09-09T04:44:43.5932681Z 	at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
2023-09-09T04:44:43.5932967Z 	at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
2023-09-09T04:44:43.5933274Z 	at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
2023-09-09T04:44:43.5933587Z 	at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
2023-09-09T04:44:43.5933874Z 	at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
2023-09-09T04:44:43.5934164Z 	at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
2023-09-09T04:44:43.5934459Z 	at java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:272)
2023-09-09T04:44:43.5934736Z 	at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-09-09T04:44:43.5935033Z 	at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-09-09T04:44:43.5935326Z 	at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
2023-09-09T04:44:43.5935614Z 	at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384)
2023-09-09T04:44:43.5935909Z 	at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
2023-09-09T04:44:43.5936202Z 	at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
2023-09-09T04:44:43.5936489Z 	at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
2023-09-09T04:44:43.5936801Z 	at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
2023-09-09T04:44:43.5937110Z 	at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
2023-09-09T04:44:43.5937380Z 	at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
2023-09-09T04:44:43.5937672Z 	at java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:272)
2023-09-09T04:44:43.5937970Z 	at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1384)
2023-09-09T04:44:43.5938252Z 	at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
2023-09-09T04:44:43.5938547Z 	at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
2023-09-09T04:44:43.5938848Z 	at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
2023-09-09T04:44:43.5939142Z 	at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
2023-09-09T04:44:43.5939444Z 	at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
2023-09-09T04:44:43.5939729Z 	at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:485)
2023-09-09T04:44:43.5940201Z 	at org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.execute(TestTemplateTestDescriptor.java:107)
2023-09-09T04:44:43.5940580Z 	at org.junit.jupiter.engine.descriptor.TestTemplateTestDescriptor.execute(TestTemplateTestDescriptor.java:42)
2023-09-09T04:44:43.5940957Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
2023-09-09T04:44:43.5941405Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-09-09T04:44:43.5941780Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
2023-09-09T04:44:43.5942121Z 	at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
2023-09-09T04:44:43.5942441Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
2023-09-09T04:44:43.5942809Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-09-09T04:44:43.5943179Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
2023-09-09T04:44:43.5943510Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
2023-09-09T04:44:43.5943800Z 	at java.util.ArrayList.forEach(ArrayList.java:1259)
2023-09-09T04:44:43.5944208Z 	at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
2023-09-09T04:44:43.5944687Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
2023-09-09T04:44:43.5945042Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-09-09T04:44:43.5945410Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
2023-09-09T04:44:43.5945755Z 	at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
2023-09-09T04:44:43.5946083Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
2023-09-09T04:44:43.5946451Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-09-09T04:44:43.5946813Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
2023-09-09T04:44:43.5947142Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
2023-09-09T04:44:43.5947427Z 	at java.util.ArrayList.forEach(ArrayList.java:1259)
2023-09-09T04:44:43.5947785Z 	at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
2023-09-09T04:44:43.5948195Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
2023-09-09T04:44:43.5948565Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-09-09T04:44:43.5948934Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
2023-09-09T04:44:43.5949432Z 	at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
2023-09-09T04:44:43.5949761Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
2023-09-09T04:44:43.5950267Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
2023-09-09T04:44:43.5950585Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
2023-09-09T04:44:43.5951291Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
2023-09-09T04:44:43.5951853Z 	at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
2023-09-09T04:44:43.5952560Z 	at org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
2023-09-09T04:44:43.5952929Z 	at org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
2023-09-09T04:44:43.5953387Z 	at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:108)
2023-09-09T04:44:43.5953760Z 	at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:88)
2023-09-09T04:44:43.5954125Z 	at org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:54)
2023-09-09T04:44:43.5954525Z 	at org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:67)
2023-09-09T04:44:43.5955227Z 	at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:52)
2023-09-09T04:44:43.5955916Z 	at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:96)
2023-09-09T04:44:43.5956211Z 	at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:75)
2023-09-09T04:44:43.5956724Z 	at org.apache.maven.surefire.junitplatform.JUnitPlatformProvider.invokeAllTests(JUnitPlatformProvider.java:150)
2023-09-09T04:44:43.5957291Z 	at org.apache.maven.surefire.junitplatform.JUnitPlatformProvider.invoke(JUnitPlatformProvider.java:124)
2023-09-09T04:44:43.5957651Z 	at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384)
2023-09-09T04:44:43.5957991Z 	at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345)
2023-09-09T04:44:43.5958306Z 	at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126)
2023-09-09T04:44:43.5958590Z 	at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418)
2023-09-09T04:44:43.5958726Z  

Issue Links

• links to: GitHub Pull Request #9671