[BEAM-7756] Stack trace not useful in SQL dsl for Java

• Type: Improvement
• Status: Open
• Priority: P3
• Resolution: Unresolved
• Component/s: dsl-sql
• Assignee: Unassigned
• Reporter: Jules Testard
• Created: 16/Jul/19 18:02
• Updated: 04/Jun/22 12:36

Description

I am testing Apache Beam to be used in our company's environment. I could be missing something, but it seems the stack tracing is very limited, in particular when using the SQL dsl available for the Beam Java SDK. This makes troubleshooting more difficult. 

 

One minimal example shows it very well: https://github.com/jtestard/beam-sql-stack-trace (the pom.xml in this repo should detail all dependencies used, the JDK used is Java 8). This GitHub repo gives attempts perform two back-to-back SQL transforms, the first of which contains a mismatch between input and output schema. The stack trace, however, gives little indication of the problem and does not trace to which transform the problem originated in:

 

org.apache.beam.sdk.Pipeline$PipelineExecutionException: java.lang.RuntimeException: CalcFn failed to evaluate: {
{{ final org.apache.beam.sdk.values.Row current = (org.apache.beam.sdk.values.Row) c.element();}}
{{ c.output(org.apache.beam.sdk.values.Row.withSchema(outputSchema).addValue(current.getValue(0)).addValue(current.getValue(1)).build());}}
{{ }}}at org.apache.beam.runners.direct.DirectRunner$DirectPipelineResult.waitUntilFinish(DirectRunner.java:348)
{{ at org.apache.beam.runners.direct.DirectRunner$DirectPipelineResult.waitUntilFinish(DirectRunner.java:318)}}
{{ at org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:213)}}
{{ at org.apache.beam.runners.direct.DirectRunner.run(DirectRunner.java:67)}}
{{ at org.apache.beam.sdk.Pipeline.run(Pipeline.java:313)}}
{{ at org.apache.beam.sdk.Pipeline.run(Pipeline.java:299)}}
{{ at dummy.MainTest.testMain(MainTest.java:18)}}
{{ at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)}}
{{ at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)}}
{{ at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)}}
{{ at java.lang.reflect.Method.invoke(Method.java:498)}}
{{ at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)}}
{{ at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)}}
{{ at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)}}
{{ at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)}}
{{ at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)}}
{{ at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)}}
{{ at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)}}
{{ at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)}}
{{ at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)}}
{{ at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)}}
{{ at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)}}
{{ at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)}}
{{ at org.junit.runners.ParentRunner.run(ParentRunner.java:363)}}
{{ at org.junit.runner.JUnitCore.run(JUnitCore.java:137)}}
{{ at com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)}}
{{ at com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)}}
{{ at com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)}}
{{ at com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)}}
{{ Caused by: java.lang.RuntimeException: CalcFn failed to evaluate: {}}
{{ final org.apache.beam.sdk.values.Row current = (org.apache.beam.sdk.values.Row) c.element();}}
{{ c.output(org.apache.beam.sdk.values.Row.withSchema(outputSchema).addValue(current.getValue(0)).addValue(current.getValue(1)).build());}}
{{ }}}at org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel$CalcFn.processElement(BeamCalcRel.java:253)
{{ Caused by: java.lang.RuntimeException: org.apache.beam.sdk.coders.CoderException: java.io.EOFException}}
{{ at org.apache.beam.runners.direct.ImmutabilityCheckingBundleFactory$ImmutabilityEnforcingBundle.add(ImmutabilityCheckingBundleFactory.java:114)}}
{{ at org.apache.beam.runners.direct.ParDoEvaluator$BundleOutputManager.output(ParDoEvaluator.java:264)}}
{{ at org.apache.beam.repackaged.beam_runners_direct_java.runners.core.SimpleDoFnRunner.outputWindowedValue(SimpleDoFnRunner.java:248)}}
{{ at org.apache.beam.repackaged.beam_runners_direct_java.runners.core.SimpleDoFnRunner.access$700(SimpleDoFnRunner.java:74)}}
{{ at org.apache.beam.repackaged.beam_runners_direct_java.runners.core.SimpleDoFnRunner$DoFnProcessContext.output(SimpleDoFnRunner.java:560)}}
{{ at org.apache.beam.repackaged.beam_runners_direct_java.runners.core.SimpleDoFnRunner$DoFnProcessContext.output(SimpleDoFnRunner.java:548)}}
{{ at SC.eval0(Unknown Source)}}
{{ at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)}}
{{ at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)}}
{{ at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)}}
{{ at java.lang.reflect.Method.invoke(Method.java:498)}}
{{ at org.apache.beam.sdks.java.extensions.sql.repackaged.org.codehaus.janino.ScriptEvaluator.evaluate(ScriptEvaluator.java:798)}}
{{ at org.apache.beam.sdks.java.extensions.sql.repackaged.org.codehaus.janino.ScriptEvaluator.evaluate(ScriptEvaluator.java:790)}}
{{ at org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel$CalcFn.processElement(BeamCalcRel.java:250)}}
{{ at org.apache.beam.sdk.extensions.sql.impl.rel.BeamCalcRel$CalcFn$DoFnInvoker.invokeProcessElement(Unknown Source)}}
{{ at org.apache.beam.repackaged.beam_runners_direct_java.runners.core.SimpleDoFnRunner.invokeProcessElement(SimpleDoFnRunner.java:214)}}
{{ at org.apache.beam.repackaged.beam_runners_direct_java.runners.core.SimpleDoFnRunner.processElement(SimpleDoFnRunner.java:179)}}
{{ at org.apache.beam.repackaged.beam_runners_direct_java.runners.core.SimplePushbackSideInputDoFnRunner.processElementInReadyWindows(SimplePushbackSideInputDoFnRunner.java:78)}}
{{ at org.apache.beam.runners.direct.ParDoEvaluator.processElement(ParDoEvaluator.java:211)}}
{{ at org.apache.beam.runners.direct.DoFnLifecycleManagerRemovingTransformEvaluator.processElement(DoFnLifecycleManagerRemovingTransformEvaluator.java:54)}}
{{ at org.apache.beam.runners.direct.DirectTransformExecutor.processElements(DirectTransformExecutor.java:160)}}
{{ at org.apache.beam.runners.direct.DirectTransformExecutor.run(DirectTransformExecutor.java:124)}}
{{ at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)}}
{{ at java.util.concurrent.FutureTask.run(FutureTask.java:266)}}
{{ at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)}}
{{ at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)}}
{{ at java.lang.Thread.run(Thread.java:748)}}
{{ Caused by: org.apache.beam.sdk.coders.CoderException: java.io.EOFException}}
{{ at org.apache.beam.sdk.coders.StringUtf8Coder.decode(StringUtf8Coder.java:104)}}
{{ at org.apache.beam.sdk.coders.StringUtf8Coder.decode(StringUtf8Coder.java:90)}}
{{ at org.apache.beam.sdk.coders.StringUtf8Coder.decode(StringUtf8Coder.java:37)}}
{{ at org.apache.beam.sdk.coders.RowCoderGenerator$DecodeInstruction.decodeDelegate(RowCoderGenerator.java:311)}}
{{ at org.apache.beam.sdk.coders.Coder$ByteBuddy$Ib8rqlTB.decode(Unknown Source)}}
{{ at org.apache.beam.sdk.coders.Coder$ByteBuddy$Ib8rqlTB.decode(Unknown Source)}}
{{ at org.apache.beam.sdk.coders.RowCoder.decode(RowCoder.java:150)}}
{{ at org.apache.beam.sdk.schemas.SchemaCoder.decode(SchemaCoder.java:86)}}
{{ at org.apache.beam.sdk.coders.Coder.decode(Coder.java:159)}}
{{ at org.apache.beam.sdk.util.CoderUtils.decodeFromSafeStream(CoderUtils.java:115)}}
{{ at org.apache.beam.sdk.util.CoderUtils.decodeFromByteArray(CoderUtils.java:98)}}
{{ at org.apache.beam.sdk.util.CoderUtils.decodeFromByteArray(CoderUtils.java:92)}}
{{ at org.apache.beam.sdk.util.CoderUtils.clone(CoderUtils.java:141)}}
{{ at org.apache.beam.sdk.util.MutationDetectors$CodedValueMutationDetector.<init>(MutationDetectors.java:113)}}
{{ at org.apache.beam.sdk.util.MutationDetectors.forValueWithCoder(MutationDetectors.java:44)}}
{{ at org.apache.beam.runners.direct.ImmutabilityCheckingBundleFactory$ImmutabilityEnforcingBundle.add(ImmutabilityCheckingBundleFactory.java:112)}}
{{ ... 26 more}}
{{ Caused by: java.io.EOFException}}
{{ at org.apache.beam.sdk.util.VarInt.decodeLong(VarInt.java:73)}}
{{ at org.apache.beam.sdk.util.VarInt.decodeInt(VarInt.java:56)}}
{{ at org.apache.beam.sdk.coders.StringUtf8Coder.readString(StringUtf8Coder.java:55)}}
{{ at org.apache.beam.sdk.coders.StringUtf8Coder.decode(StringUtf8Coder.java:100)}}
{{ ... 41 more}}