[TEZ-2889] ATSHistoryACLPolicyManager makes DAG Submission wait on ATS requests

• Type: Bug
• Status: Open
• Priority: Major
• Resolution: Unresolved
• Affects Version/s: 0.7.0, 0.8.2
• Labels: ATS Performance
• Assignee: Unassigned
• Reporter: Gopal Vijayaraghavan
• Created: 14/Oct/15 00:45
• Updated: 08/Aug/16 23:31

Description

On a cluster with a broken ATS install, the ATS domain creation is in-thread & bottlenecks all Tez queries, reducing the cluster throughput by ~3x for very small queries.

java.net.SocketInputStream.socketRead0(FileDescriptor, byte[], int, int, int) SocketInputStream.java (native)
java.net.SocketInputStream.socketRead(FileDescriptor, byte[], int, int, int) SocketInputStream.java:116
java.net.SocketInputStream.read(byte[], int, int, int) SocketInputStream.java:170
java.net.SocketInputStream.read(byte[], int, int) SocketInputStream.java:141
java.io.BufferedInputStream.fill() BufferedInputStream.java:246
java.io.BufferedInputStream.read1(byte[], int, int) BufferedInputStream.java:286
java.io.BufferedInputStream.read(byte[], int, int) BufferedInputStream.java:345
sun.net.www.http.HttpClient.parseHTTPHeader(MessageHeader, ProgressSource, HttpURLConnection) HttpClient.java:704
sun.net.www.http.HttpClient.parseHTTP(MessageHeader, ProgressSource, HttpURLConnection) HttpClient.java:647
sun.net.www.protocol.http.HttpURLConnection.getInputStream0() HttpURLConnection.java:1535
sun.net.www.protocol.http.HttpURLConnection.getInputStream() HttpURLConnection.java:1440
java.net.HttpURLConnection.getResponseCode() HttpURLConnection.java:480
org.apache.hadoop.security.authentication.client.AuthenticatedURL.extractToken(HttpURLConnection, AuthenticatedURL$Token) AuthenticatedURL.java:252
org.apache.hadoop.security.authentication.client.PseudoAuthenticator.authenticate(URL, AuthenticatedURL$Token) PseudoAuthenticator.java:77
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.authenticate(URL, AuthenticatedURL$Token) DelegationTokenAuthenticator.java:128
org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(URL, AuthenticatedURL$Token) AuthenticatedURL.java:215
org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.openConnection(URL, DelegationTokenAuthenticatedURL$Token, String) DelegationTokenAuthenticatedURL.java:322
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineURLConnectionFactory.getHttpURLConnection(URL) TimelineClientImpl.java:493
com.sun.jersey.client.urlconnection.URLConnectionClientHandler._invoke(ClientRequest) URLConnectionClientHandler.java:159
com.sun.jersey.client.urlconnection.URLConnectionClientHandler.handle(ClientRequest) URLConnectionClientHandler.java:147
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineJerseyRetryFilter$1.run() TimelineClientImpl.java:234
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineClientConnectionRetry.retryOn(TimelineClientImpl$TimelineClientRetryOp) TimelineClientImpl.java:183
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineJerseyRetryFilter.handle(ClientRequest) TimelineClientImpl.java:245
com.sun.jersey.api.client.Client.handle(ClientRequest) Client.java:648
com.sun.jersey.api.client.WebResource.handle(Class, ClientRequest) WebResource.java:670
com.sun.jersey.api.client.WebResource.access$200(WebResource, Class, ClientRequest) WebResource.java:74
com.sun.jersey.api.client.WebResource$Builder.put(Class, Object) WebResource.java:533
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.doPostingObject(Object, String) TimelineClientImpl.java:478
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$1.run() TimelineClientImpl.java:323
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$1.run() TimelineClientImpl.java:320
java.security.AccessController.doPrivileged(PrivilegedExceptionAction, AccessControlContext) AccessController.java (native)
javax.security.auth.Subject.doAs(Subject, PrivilegedExceptionAction) Subject.java:422
org.apache.hadoop.security.UserGroupInformation.doAs(PrivilegedExceptionAction) UserGroupInformation.java:1657
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.doPosting(Object, String) TimelineClientImpl.java:320
org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.putDomain(TimelineDomain) TimelineClientImpl.java:313
org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createTimelineDomain(String, Configuration, DAGAccessControls) ATSHistoryACLPolicyManager.java:129
org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.createDAGDomain(Configuration, ApplicationId, String, DAGAccessControls) ATSHistoryACLPolicyManager.java:207
org.apache.tez.dag.history.ats.acls.ATSHistoryACLPolicyManager.setupSessionDAGACLs(Configuration, ApplicationId, String, DAGAccessControls) ATSHistoryACLPolicyManager.java:240
org.apache.tez.client.TezClient.submitDAGSession(DAG) TezClient.java:491
org.apache.tez.client.TezClient.submitDAG(DAG) TezClient.java:452
org.apache.hadoop.hive.ql.exec.tez.TezTask.submit(JobConf, DAG, Path, LocalResource, TezSessionState, List, String[], Map) TezTask.java:427
org.apache.hadoop.hive.ql.exec.tez.TezTask.execute(DriverContext) TezTask.java:170
org.apache.hadoop.hive.ql.exec.Task.executeTask() Task.java:160
org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential() TaskRunner.java:89
org.apache.hadoop.hive.ql.Driver.launchTask(Task, String, boolean, String, int, DriverContext) Driver.java:1758
org.apache.hadoop.hive.ql.Driver.execute() Driver.java:1517
org.apache.hadoop.hive.ql.Driver.runInternal(String, boolean) Driver.java:1275
org.apache.hadoop.hive.ql.Driver.run(String, boolean) Driver.java:1091
org.apache.hadoop.hive.ql.Driver.run(String) Driver.java:1081
org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(String, CommandProcessor, CliSessionState) CliDriver.java:225
org.apache.hadoop.hive.cli.CliDriver.processCmd(String) CliDriver.java:177
org.apache.hadoop.hive.cli.CliDriver.processLine(String, boolean) CliDriver.java:388
org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliSessionState, HiveConf, OptionsProcessor) CliDriver.java:759
org.apache.hadoop.hive.cli.CliDriver.run(String[]) CliDriver.java:704
org.apache.hadoop.hive.cli.CliDriver.main(String[]) CliDriver.java:633
sun.reflect.NativeMethodAccessorImpl.invoke0(Method, Object, Object[]) NativeMethodAccessorImpl.java (native)
sun.reflect.NativeMethodAccessorImpl.invoke(Object, Object[]) NativeMethodAccessorImpl.java:62
sun.reflect.DelegatingMethodAccessorImpl.invoke(Object, Object[]) DelegatingMethodAccessorImpl.java:43
java.lang.reflect.Method.invoke(Object, Object[]) Method.java:497
org.apache.hadoop.util.RunJar.run(String[]) RunJar.java:221
org.apache.hadoop.util.RunJar.main(String[]) RunJar.java:136

Issue Links

• is duplicated by: TEZ-3404 Move blocking call for YARN Timeline domain creation from client side to AM