Major Description
ENV: Keberos/SPNEGO enabled
set hive.exec.post.hook;
org.apache.hadoop.hive.ql.hooks.ATSHook,org.apache.atlas.hive.hook.HiveHook
ATLAS Hook use InMemoryJAASConfiguration
This is a sequence of the event while issue reproduces:
WebHcat -> hcat -> Hive Driver -> post hook execution create ATSHook -> hook start the spnego auth and stuck while finding InMemoryJAASConfiguration used by the AtlasHook (this happens in separate thread ATS Logger)
Hcat jstack
Found one Java-level deadlock: ============================= "ATS Logger 0": waiting to lock monitor 0x00007efdc8003a38 (object 0x00000000f3fcfe28, a org.apache.atlas.plugin.classloader.AtlasPluginClassLoader), which is held by "main" "main": waiting to lock monitor 0x00007efdc8003da8 (object 0x00000000c0050d40, a org.apache.hadoop.hive.ql.exec.UDFClassLoader), which is held by "ATS Logger 0" Java stack information for the threads listed above: =================================================== "ATS Logger 0": at org.apache.atlas.security.InMemoryJAASConfiguration.getAppConfigurationEntry(InMemoryJAASConfiguration.java:238) at sun.security.jgss.LoginConfigImpl.getAppConfigurationEntry(LoginConfigImpl.java:145) at javax.security.auth.login.LoginContext.init(LoginContext.java:251) at javax.security.auth.login.LoginContext.<init>(LoginContext.java:512) at sun.security.jgss.GSSUtil.login(GSSUtil.java:256) at sun.security.jgss.krb5.Krb5Util.getTicket(Krb5Util.java:158) at sun.security.jgss.krb5.Krb5InitCredential$1.run(Krb5InitCredential.java:335) at sun.security.jgss.krb5.Krb5InitCredential$1.run(Krb5InitCredential.java:331) at java.security.AccessController.doPrivileged(Native Method) at sun.security.jgss.krb5.Krb5InitCredential.getTgt(Krb5InitCredential.java:330) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:145) at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:122) at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187) at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:224) at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) at sun.security.jgss.spnego.SpNegoContext.GSS_initSecContext(SpNegoContext.java:882) at sun.security.jgss.spnego.SpNegoContext.initSecContext(SpNegoContext.java:317) at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:248) at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) at sun.net.www.protocol.http.spnego.NegotiatorImpl.init(NegotiatorImpl.java:108) at sun.net.www.protocol.http.spnego.NegotiatorImpl.<init>(NegotiatorImpl.java:117) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at sun.net.www.protocol.http.Negotiator.getNegotiator(Negotiator.java:63) at sun.net.www.protocol.http.NegotiateAuthentication.isSupportedImpl(NegotiateAuthentication.java:130) - locked <0x00000000f48c4d90> (a java.lang.Class for sun.net.www.protocol.http.NegotiateAuthentication) at sun.net.www.protocol.http.NegotiateAuthentication.isSupported(NegotiateAuthentication.java:102) - locked <0x00000000c0050d40> (a org.apache.hadoop.hive.ql.exec.UDFClassLoader) at sun.net.www.protocol.http.AuthenticationHeader.parse(AuthenticationHeader.java:180) at sun.net.www.protocol.http.AuthenticationHeader.<init>(AuthenticationHeader.java:126) at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1660) - locked <0x00000000f47b7298> (a sun.net.www.protocol.https.DelegateHttpsURLConnection) at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1441) - locked <0x00000000f47b7298> (a sun.net.www.protocol.https.DelegateHttpsURLConnection) at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:480) at sun.net.www.protocol.https.HttpsURLConnectionImpl.getResponseCode(HttpsURLConnectionImpl.java:338) at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.authenticate(KerberosAuthenticator.java:191) at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.authenticate(DelegationTokenAuthenticator.java:133) at org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(AuthenticatedURL.java:216) at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticatedURL.openConnection(DelegationTokenAuthenticatedURL.java:322) at org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineURLConnectionFactory.getHttpURLConnection(TimelineClientImpl.java:476) at com.sun.jersey.client.urlconnection.URLConnectionClientHandler._invoke(URLConnectionClientHandler.java:159) at com.sun.jersey.client.urlconnection.URLConnectionClientHandler.handle(URLConnectionClientHandler.java:147) at org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl$TimelineJerseyRetryFilter$1.run(TimelineClientImpl.java:237) "main": at java.lang.ClassLoader.loadClass(ClassLoader.java:406) - waiting to lock <0x00000000c0050d40> (a org.apache.hadoop.hive.ql.exec.UDFClassLoader) at java.lang.ClassLoader.loadClass(ClassLoader.java:411) - locked <0x00000000f430b818> (a org.apache.atlas.plugin.classloader.AtlasPluginClassLoader$MyClassLoader) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) at org.apache.atlas.plugin.classloader.AtlasPluginClassLoader.loadClass(AtlasPluginClassLoader.java:136) at org.apache.atlas.hook.AtlasHook.<clinit>(AtlasHook.java:76) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:348) at org.apache.atlas.hive.hook.HiveHook.initialize(HiveHook.java:72) at org.apache.atlas.hive.hook.HiveHook.<init>(HiveHook.java:41) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at java.lang.Class.newInstance(Class.java:442) at org.apache.hadoop.hive.ql.hooks.HookUtils.getHooks(HookUtils.java:61) at org.apache.hadoop.hive.ql.Driver.getHooks(Driver.java:1398) at org.apache.hadoop.hive.ql.Driver.getHooks(Driver.java:1382) at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1615) at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1303) at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1170) at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1160) at org.apache.hive.hcatalog.cli.HCatDriver.run(HCatDriver.java:43) at org.apache.hive.hcatalog.cli.HCatCli.processCmd(HCatCli.java:291) at org.apache.hive.hcatalog.cli.HCatCli.processLine(HCatCli.java:245) at org.apache.hive.hcatalog.cli.HCatCli.main(HCatCli.java:183)