Found one Java-level deadlock: ============================= "pool-8-thread-4": waiting to lock monitor 0x00007f57485793d8 (object 0x00000006268c26e0, a java.util.concurrent.ConcurrentHashMap), which is held by "http-bio-8080-exec-3" "http-bio-8080-exec-3": waiting to lock monitor 0x00007f5766331fa8 (object 0x00000006268c2610, a java.lang.Class), which is held by "Thread-23" "Thread-23": waiting to lock monitor 0x00007f57485793d8 (object 0x00000006268c26e0, a java.util.concurrent.ConcurrentHashMap), which is held by "http-bio-8080-exec-3" Java stack information for the threads listed above: =================================================== "pool-8-thread-4": at org.apache.kylin.metadata.cachesync.Broadcaster.getInstance(Broadcaster.java:71) - waiting to lock <0x00000006268c26e0> (a java.util.concurrent.ConcurrentHashMap) at org.apache.kylin.metadata.cachesync.AbstractCache.getBroadcaster(AbstractCache.java:36) at org.apache.kylin.metadata.cachesync.SingleValueCache.put(SingleValueCache.java:54) at org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache.put(CaseInsensitiveStringCache.java:35) at org.apache.kylin.cube.CubeManager.updateCubeWithRetry(CubeManager.java:427) at org.apache.kylin.cube.CubeManager.updateCube(CubeManager.java:338) at org.apache.kylin.cube.CubeManager.buildDictionary(CubeManager.java:231) at org.apache.kylin.cube.cli.DictionaryGeneratorCLI.processSegment(DictionaryGeneratorCLI.java:50) at org.apache.kylin.cube.cli.DictionaryGeneratorCLI.processSegment(DictionaryGeneratorCLI.java:41) at org.apache.kylin.engine.mr.steps.CreateDictionaryJob.run(CreateDictionaryJob.java:54) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:90) at org.apache.kylin.engine.mr.common.HadoopShellExecutable.doWork(HadoopShellExecutable.java:63) at org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:113) at org.apache.kylin.job.execution.DefaultChainedExecutable.doWork(DefaultChainedExecutable.java:57) at org.apache.kylin.job.execution.AbstractExecutable.execute(AbstractExecutable.java:113) at org.apache.kylin.job.impl.threadpool.DefaultScheduler$JobRunner.run(DefaultScheduler.java:136) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "http-bio-8080-exec-3": at org.apache.kylin.metadata.project.ProjectManager.getInstance(ProjectManager.java:60) - waiting to lock <0x00000006268c2610> (a java.lang.Class for org.apache.kylin.metadata.project.ProjectManager) at org.apache.kylin.cube.CubeManager$CubeSyncListener.onEntityChange(CubeManager.java:169) at org.apache.kylin.metadata.cachesync.Broadcaster.notifyListener(Broadcaster.java:213) - locked <0x00000006268c26e0> (a java.util.concurrent.ConcurrentHashMap) at org.apache.kylin.rest.service.CacheService.notifyMetadataChange(CacheService.java:115) at org.apache.kylin.rest.controller.CacheController.wipeCache(CacheController.java:64) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:221) at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:136) at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:104) at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandleMethod(RequestMappingHandlerAdapter.java:743) at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:672) at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:82) at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:933) at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:867) at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:951) at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:864) at javax.servlet.http.HttpServlet.service(HttpServlet.java:653) at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:827) at javax.servlet.http.HttpServlet.service(HttpServlet.java:731) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:330) at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:118) at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:84) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:113) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.session.SessionManagementFilter.doFilter(SessionManagementFilter.java:103) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:113) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:54) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(RequestCacheAwareFilter.java:45) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilter(BasicAuthenticationFilter.java:150) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.authentication.ui.DefaultLoginPageGeneratingFilter.doFilter(DefaultLoginPageGeneratingFilter.java:91) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:183) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.authentication.logout.LogoutFilter.doFilter(LogoutFilter.java:105) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:87) at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342) at org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:192) at org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:160) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:343) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:260) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) at com.thetransactioncompany.cors.CORSFilter.doFilter(CORSFilter.java:209) at com.thetransactioncompany.cors.CORSFilter.doFilter(CORSFilter.java:244) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122) at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103) at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:436) at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1078) at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:625) at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316) - locked <0x00000006268c3260> (a org.apache.tomcat.util.net.SocketWrapper) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) at java.lang.Thread.run(Thread.java:745) "Thread-23": at org.apache.kylin.metadata.cachesync.Broadcaster.getInstance(Broadcaster.java:71) - waiting to lock <0x00000006268c26e0> (a java.util.concurrent.ConcurrentHashMap) at org.apache.kylin.metadata.project.ProjectManager.(ProjectManager.java:96) at org.apache.kylin.metadata.project.ProjectManager.getInstance(ProjectManager.java:65) - locked <0x00000006268c2610> (a java.lang.Class for org.apache.kylin.metadata.project.ProjectManager) at org.apache.kylin.cube.CubeManager.updateCubeWithRetry(CubeManager.java:430) at org.apache.kylin.cube.CubeManager.updateCube(CubeManager.java:338) at org.apache.kylin.cube.CubeManager.mergeSegments(CubeManager.java:564) at org.apache.kylin.rest.service.CubeService.mergeCubeSegment(CubeService.java:633) - locked <0x00000006267998f8> (a java.lang.Class for org.apache.kylin.rest.service.CubeService) at org.apache.kylin.rest.service.CubeService.updateOnNewSegmentReady(CubeService.java:581) at org.apache.kylin.rest.service.CubeService$$FastClassBySpringCGLIB$$17a07c0e.invoke() at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204) at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:629) at org.apache.kylin.rest.service.CubeService$$EnhancerBySpringCGLIB$$4dbca366.updateOnNewSegmentReady() at org.apache.kylin.rest.service.CacheService$1$1.run(CacheService.java:89) Found 1 deadlock.