[APEXCORE-807] In secure mode containers are failing after one day and the application is failing after seven days

• Type: Bug
• Status: Resolved
• Priority: Major
• Resolution: Fixed
• Fix Version/s: 4.0.0
• Assignee: Pramod Immaneni
• Reporter: Pramod Immaneni
• Created: 18/Apr/18 13:57
• Updated: 20/Jun/18 02:05
• Resolved: 20/Jun/18 02:05

Description

java.lang.RuntimeException: java.util.concurrent.ExecutionException: java.lang.RuntimeException: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): token (HDFS_DELEGATION_TOKEN token nnnnnn for xxxxxx) can't be found in cache
at com.google.common.base.Throwables.propagate(Throwables.java:156)
at com.datatorrent.stram.engine.Node.reportStats(Node.java:489)
at com.datatorrent.stram.engine.GenericNode.reportStats(GenericNode.java:825)
at com.datatorrent.stram.engine.GenericNode.processEndWindow(GenericNode.java:184)
at com.datatorrent.stram.engine.GenericNode.run(GenericNode.java:397)
at com.datatorrent.stram.engine.StreamingContainer$2.run(StreamingContainer.java:1465)
Caused by: java.util.concurrent.ExecutionException: java.lang.RuntimeException: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): token (HDFS_DELEGATION_TOKEN token nnnnnn for xxxxxx) can't be found in cache
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:192)
at com.datatorrent.stram.engine.Node.reportStats(Node.java:482)
... 4 more
Caused by: java.lang.RuntimeException: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): token (HDFS_DELEGATION_TOKEN token nnnnnn for xxxxxx) can't be found in cache
at com.google.common.base.Throwables.propagate(Throwables.java:156)
at com.datatorrent.common.util.AsyncFSStorageAgent.copyToHDFS(AsyncFSStorageAgent.java:131)
at com.datatorrent.common.util.AsyncFSStorageAgent.flush(AsyncFSStorageAgent.java:156)
at com.datatorrent.stram.engine.Node$CheckpointHandler.call(Node.java:706)
at com.datatorrent.stram.engine.Node$CheckpointHandler.call(Node.java:696)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.token.SecretManager$InvalidToken): token (HDFS_DELEGATION_TOKEN token nnnnnn for xxxxxx) can't be found in cache
at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1554)
at org.apache.hadoop.ipc.Client.call(Client.java:1498)
at org.apache.hadoop.ipc.Client.call(Client.java:1398)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233)
at com.sun.proxy.$Proxy10.create(Unknown Source)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:313)
at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:291)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:203)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:185)
at com.sun.proxy.$Proxy11.create(Unknown Source)
at org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:1822)
at org.apache.hadoop.hdfs.DFSClient.primitiveCreate(DFSClient.java:1762)
at org.apache.hadoop.fs.Hdfs.createInternal(Hdfs.java:104)
at org.apache.hadoop.fs.Hdfs.createInternal(Hdfs.java:60)
at org.apache.hadoop.fs.AbstractFileSystem.create(AbstractFileSystem.java:585)
at org.apache.hadoop.fs.FileContext$3.next(FileContext.java:688)
at org.apache.hadoop.fs.FileContext$3.next(FileContext.java:684)
at org.apache.hadoop.fs.FSLinkResolver.resolve(FSLinkResolver.java:90)
at org.apache.hadoop.fs.FileContext.create(FileContext.java:684)
at com.datatorrent.common.util.AsyncFSStorageAgent.copyToHDFS(AsyncFSStorageAgent.java:119)
... 9 more

Issue Links

• links to: GitHub Pull Request #594