[HUDI-6754] Fix NullPointerException w/ AbstractRealTimRecordReader

• Type: Bug
• Status: Patch Available
• Priority: Major
• Resolution: Unresolved
• Component/s: reader-core
• Labels: pull-request-available
• Assignee: Unassigned
• Reporter: sivabalan narayanan
• Created: 25/Aug/23 06:12
• Updated: 25/Aug/23 20:23

Description

recently, we are seeing NPE w/ AbstractRealTimeRecordReader in our CI test runs. we need to chase them and put in a fix. 

Ref: https://dev.azure.com/apache-hudi-ci-org/785b6ef4-2f42-4a89-8f0e-5f0d7039a0cc/_apis/build/builds/19420/logs/35

 

2023-08-23T08:16:08.7070674Z [INFO] Running org.apache.hudi.hadoop.realtime.TestHoodieMergeOnReadSnapshotReader
2023-08-23T08:16:09.4083756Z 10259 [main] WARN  org.apache.hudi.hadoop.realtime.AbstractRealtimeRecordReader [] - fall to init HiveAvroSerializer to support payload merge
2023-08-23T08:16:09.4084752Z java.lang.NullPointerException: null
2023-08-23T08:16:09.4085512Z 	at org.apache.hudi.hadoop.realtime.AbstractRealtimeRecordReader.prepareHiveAvroSerializer(AbstractRealtimeRecordReader.java:116) ~[classes/:?]
2023-08-23T08:16:09.4086265Z 	at org.apache.hudi.hadoop.realtime.AbstractRealtimeRecordReader.<init>(AbstractRealtimeRecordReader.java:103) ~[classes/:?]
2023-08-23T08:16:09.4087168Z 	at org.apache.hudi.hadoop.realtime.HoodieMergeOnReadSnapshotReader.<init>(HoodieMergeOnReadSnapshotReader.java:106) ~[classes/:?]
2023-08-23T08:16:09.4087997Z 	at org.apache.hudi.hadoop.realtime.TestHoodieMergeOnReadSnapshotReader.lambda$testReaderInternal$0(TestHoodieMergeOnReadSnapshotReader.java:162) ~[test-classes/:?]
2023-08-23T08:16:09.4088379Z 	at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_382]
2023-08-23T08:16:09.4089029Z 	at org.apache.hudi.hadoop.realtime.TestHoodieMergeOnReadSnapshotReader.testReaderInternal(TestHoodieMergeOnReadSnapshotReader.java:131) ~[test-classes/:?]
2023-08-23T08:16:09.4089707Z 	at org.apache.hudi.hadoop.realtime.TestHoodieMergeOnReadSnapshotReader.testSnapshotReaderPartitioned(TestHoodieMergeOnReadSnapshotReader.java:102) ~[test-classes/:?]
2023-08-23T08:16:09.4090092Z 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_382]
2023-08-23T08:16:09.4090857Z 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_382]
2023-08-23T08:16:09.4091226Z 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_382]
2023-08-23T08:16:09.4091546Z 	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_382]
2023-08-23T08:16:09.4092052Z 	at org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688) ~[junit-platform-commons-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4092633Z 	at org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4093293Z 	at org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4093917Z 	at org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4094722Z 	at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:140) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4095348Z 	at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestMethod(TimeoutExtension.java:84) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4096018Z 	at org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4096648Z 	at org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4097318Z 	at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4097975Z 	at org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4098632Z 	at org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4099252Z 	at org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4099846Z 	at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4100425Z 	at org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4101049Z 	at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$6(TestMethodTestDescriptor.java:210) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4101707Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4102360Z 	at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:206) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4103264Z 	at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:131) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4103884Z 	at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:65) ~[junit-jupiter-engine-5.7.2.jar:5.7.2]
2023-08-23T08:16:09.4104528Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4106052Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4106990Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4107583Z 	at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4108187Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4108806Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4109396Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4109981Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4110468Z 	at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_382]
2023-08-23T08:16:09.4111088Z 	at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4111775Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4112394Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4113018Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4113562Z 	at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4114167Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4114784Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4115372Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4115951Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4116299Z 	at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_382]
2023-08-23T08:16:09.4116910Z 	at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4117580Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4118201Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4118821Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4119362Z 	at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4119955Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4121040Z 	at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4122714Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4193183Z 	at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4194507Z 	at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4195370Z 	at org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4196135Z 	at org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51) ~[junit-platform-engine-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4196868Z 	at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:108) ~[junit-platform-launcher-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4200268Z 	at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:88) ~[junit-platform-launcher-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4201139Z 	at org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:54) ~[junit-platform-launcher-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4201915Z 	at org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:67) ~[junit-platform-launcher-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4202796Z 	at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:52) ~[junit-platform-launcher-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4203474Z 	at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:96) ~[junit-platform-launcher-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4204162Z 	at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:75) ~[junit-platform-launcher-1.7.2.jar:1.7.2]
2023-08-23T08:16:09.4204909Z 	at org.apache.maven.surefire.junitplatform.JUnitPlatformProvider.invokeAllTests(JUnitPlatformProvider.java:150) ~[surefire-junit-platform-2.22.2.jar:2.22.2]
2023-08-23T08:16:09.4205770Z 	at org.apache.maven.surefire.junitplatform.JUnitPlatformProvider.invoke(JUnitPlatformProvider.java:124) ~[surefire-junit-platform-2.22.2.jar:2.22.2]
2023-08-23T08:16:09.4206445Z 	at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384) ~[surefire-booter-2.22.2.jar:2.22.2]
2023-08-23T08:16:09.4207110Z 	at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345) ~[surefire-booter-2.22.2.jar:2.22.2]
2023-08-23T08:16:09.4207744Z 	at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126) ~[surefire-booter-2.22.2.jar:2.22.2]
2023-08-23T08:16:09.4208359Z 	at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418) ~[surefire-booter-2.22.2.jar:2.22.2]
2023-08-23T08:16:11.4067874Z # WARNING: Unable to attach Serviceability Agent. You can try again with escalated privileges. Two options: a) use -Djol.tryWithSudo=true to try with sudo; b) echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
2023-08-23T08:16:11.7324751Z 12593 [main] WARN  org.apache.hudi.hadoop.realtime.AbstractRealtimeRecordReader [] - fall to init HiveAvroSerializer to support payload merge 

 

Issue Links

• links to: GitHub Pull Request #9535