Uploaded image for project: 'Flink'
  1. Flink
  2. FLINK-16768

HadoopS3RecoverableWriterITCase.testRecoverWithStateWithMultiPart hangs

    XMLWordPrintableJSON

    Details

      Description

      Logs: https://dev.azure.com/rmetzger/Flink/_build/results?buildId=6584&view=logs&j=d44f43ce-542c-597d-bf94-b0718c71e5e8&t=d26b3528-38b0-53d2-05f7-37557c2405e4

      2020-03-24T15:52:18.9196862Z "main" #1 prio=5 os_prio=0 tid=0x00007fd36c00b800 nid=0xc21 runnable [0x00007fd3743ce000]
      2020-03-24T15:52:18.9197235Z    java.lang.Thread.State: RUNNABLE
      2020-03-24T15:52:18.9197536Z 	at java.net.SocketInputStream.socketRead0(Native Method)
      2020-03-24T15:52:18.9197931Z 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
      2020-03-24T15:52:18.9198340Z 	at java.net.SocketInputStream.read(SocketInputStream.java:171)
      2020-03-24T15:52:18.9198749Z 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
      2020-03-24T15:52:18.9199171Z 	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
      2020-03-24T15:52:18.9199840Z 	at sun.security.ssl.InputRecord.readV3Record(InputRecord.java:593)
      2020-03-24T15:52:18.9200265Z 	at sun.security.ssl.InputRecord.read(InputRecord.java:532)
      2020-03-24T15:52:18.9200663Z 	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:975)
      2020-03-24T15:52:18.9201213Z 	- locked <0x00000000927583d8> (a java.lang.Object)
      2020-03-24T15:52:18.9201589Z 	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:933)
      2020-03-24T15:52:18.9202026Z 	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
      2020-03-24T15:52:18.9202583Z 	- locked <0x0000000092758c00> (a sun.security.ssl.AppInputStream)
      2020-03-24T15:52:18.9203029Z 	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
      2020-03-24T15:52:18.9203558Z 	at org.apache.http.impl.io.SessionInputBufferImpl.read(SessionInputBufferImpl.java:198)
      2020-03-24T15:52:18.9204121Z 	at org.apache.http.impl.io.ContentLengthInputStream.read(ContentLengthInputStream.java:176)
      2020-03-24T15:52:18.9204626Z 	at org.apache.http.conn.EofSensorInputStream.read(EofSensorInputStream.java:135)
      2020-03-24T15:52:18.9205121Z 	at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:82)
      2020-03-24T15:52:18.9205679Z 	at com.amazonaws.event.ProgressInputStream.read(ProgressInputStream.java:180)
      2020-03-24T15:52:18.9206164Z 	at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:82)
      2020-03-24T15:52:18.9206786Z 	at com.amazonaws.services.s3.internal.S3AbortableInputStream.read(S3AbortableInputStream.java:125)
      2020-03-24T15:52:18.9207361Z 	at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:82)
      2020-03-24T15:52:18.9207839Z 	at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:82)
      2020-03-24T15:52:18.9208327Z 	at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:82)
      2020-03-24T15:52:18.9208809Z 	at com.amazonaws.event.ProgressInputStream.read(ProgressInputStream.java:180)
      2020-03-24T15:52:18.9209273Z 	at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:82)
      2020-03-24T15:52:18.9210003Z 	at com.amazonaws.util.LengthCheckInputStream.read(LengthCheckInputStream.java:107)
      2020-03-24T15:52:18.9210658Z 	at com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:82)
      2020-03-24T15:52:18.9211154Z 	at org.apache.hadoop.fs.s3a.S3AInputStream.lambda$read$3(S3AInputStream.java:445)
      2020-03-24T15:52:18.9211631Z 	at org.apache.hadoop.fs.s3a.S3AInputStream$$Lambda$42/1936375962.execute(Unknown Source)
      2020-03-24T15:52:18.9212044Z 	at org.apache.hadoop.fs.s3a.Invoker.once(Invoker.java:109)
      2020-03-24T15:52:18.9212553Z 	at org.apache.hadoop.fs.s3a.Invoker.lambda$retry$3(Invoker.java:260)
      2020-03-24T15:52:18.9212972Z 	at org.apache.hadoop.fs.s3a.Invoker$$Lambda$23/1457226878.execute(Unknown Source)
      2020-03-24T15:52:18.9213408Z 	at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:317)
      2020-03-24T15:52:18.9213866Z 	at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:256)
      2020-03-24T15:52:18.9214273Z 	at org.apache.hadoop.fs.s3a.Invoker.retry(Invoker.java:231)
      2020-03-24T15:52:18.9214701Z 	at org.apache.hadoop.fs.s3a.S3AInputStream.read(S3AInputStream.java:441)
      2020-03-24T15:52:18.9215443Z 	- locked <0x00000000926e88b0> (a org.apache.hadoop.fs.s3a.S3AInputStream)
      2020-03-24T15:52:18.9215852Z 	at java.io.DataInputStream.read(DataInputStream.java:149)
      2020-03-24T15:52:18.9216305Z 	at org.apache.flink.runtime.fs.hdfs.HadoopDataInputStream.read(HadoopDataInputStream.java:94)
      2020-03-24T15:52:18.9216781Z 	at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284)
      2020-03-24T15:52:18.9217187Z 	at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326)
      2020-03-24T15:52:18.9217571Z 	at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178)
      2020-03-24T15:52:18.9218108Z 	- locked <0x00000000926ea000> (a java.io.InputStreamReader)
      2020-03-24T15:52:18.9218475Z 	at java.io.InputStreamReader.read(InputStreamReader.java:184)
      2020-03-24T15:52:18.9218876Z 	at java.io.BufferedReader.fill(BufferedReader.java:161)
      2020-03-24T15:52:18.9219261Z 	at java.io.BufferedReader.readLine(BufferedReader.java:324)
      2020-03-24T15:52:18.9219890Z 	- locked <0x00000000926ea000> (a java.io.InputStreamReader)
      2020-03-24T15:52:18.9220256Z 	at java.io.BufferedReader.readLine(BufferedReader.java:389)
      2020-03-24T15:52:18.9220914Z 	at org.apache.flink.fs.s3hadoop.HadoopS3RecoverableWriterITCase.getContentsOfFile(HadoopS3RecoverableWriterITCase.java:423)
      2020-03-24T15:52:18.9221704Z 	at org.apache.flink.fs.s3hadoop.HadoopS3RecoverableWriterITCase.testResumeAfterMultiplePersist(HadoopS3RecoverableWriterITCase.java:411)
      2020-03-24T15:52:18.9222457Z 	at org.apache.flink.fs.s3hadoop.HadoopS3RecoverableWriterITCase.testResumeAfterMultiplePersistWithMultiPartUploads(HadoopS3RecoverableWriterITCase.java:364)
      2020-03-24T15:52:18.9223222Z 	at org.apache.flink.fs.s3hadoop.HadoopS3RecoverableWriterITCase.testRecoverWithStateWithMultiPart(HadoopS3RecoverableWriterITCase.java:330)
      2020-03-24T15:52:18.9223817Z 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      2020-03-24T15:52:18.9224232Z 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
      2020-03-24T15:52:18.9224729Z 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      2020-03-24T15:52:18.9225160Z 	at java.lang.reflect.Method.invoke(Method.java:498)
      2020-03-24T15:52:18.9225675Z 	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
      2020-03-24T15:52:18.9226171Z 	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
      2020-03-24T15:52:18.9226682Z 	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
      2020-03-24T15:52:18.9227187Z 	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
      2020-03-24T15:52:18.9227661Z 	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
      2020-03-24T15:52:18.9228145Z 	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
      2020-03-24T15:52:18.9228718Z 	at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55)
      2020-03-24T15:52:18.9229112Z 	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
      2020-03-24T15:52:18.9229582Z 	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
      2020-03-24T15:52:18.9230029Z 	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
      2020-03-24T15:52:18.9230525Z 	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
      2020-03-24T15:52:18.9230963Z 	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
      2020-03-24T15:52:18.9231546Z 	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
      2020-03-24T15:52:18.9231999Z 	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
      2020-03-24T15:52:18.9232432Z 	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
      2020-03-24T15:52:18.9232862Z 	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
      2020-03-24T15:52:18.9233307Z 	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
      2020-03-24T15:52:18.9233833Z 	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
      2020-03-24T15:52:18.9234284Z 	at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48)
      2020-03-24T15:52:18.9234700Z 	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
      2020-03-24T15:52:18.9235076Z 	at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
      2020-03-24T15:52:18.9235599Z 	at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:365)
      2020-03-24T15:52:18.9236124Z 	at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:273)
      2020-03-24T15:52:18.9236648Z 	at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:238)
      2020-03-24T15:52:18.9237167Z 	at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:159)
      2020-03-24T15:52:18.9237688Z 	at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384)
      2020-03-24T15:52:18.9238244Z 	at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345)
      2020-03-24T15:52:18.9238745Z 	at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126)
      2020-03-24T15:52:18.9239202Z 	at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418)
      2020-03-24T15:52:18.9239549Z 
      2020-03-24T15:52:18.9239794Z "VM Thread" os_prio=0 tid=0x00007fd36c260800 nid=0xc58 runnable 

       

        Attachments

          Issue Links

            Activity

              People

              • Assignee:
                rmetzger Robert Metzger
                Reporter:
                zjwang Zhijiang
              • Votes:
                0 Vote for this issue
                Watchers:
                5 Start watching this issue

                Dates

                • Created:
                  Updated:
                  Resolved: