Critical Description
We had a situation where a policy was messed up in the database. The x_policy_resource had multiple values for the 2 of the (policy_id, res_def_id) values. (How this happened in the 1st place is a separate bug - RANGER-663) The plugin came with a stale version while asking for policy download. Server proceeded to prepare the policy set to return. However, while doing that server ran into a problem while reading the policies because of bad db state (see stack trace below). But in the end it ended up return an empty policy set with a status 200 back to the plugin. As a result the plugin blocked all access after this problem. Since we returned the new version to the plugin after returning 200 once all subsequent requests got 304 back.
2015-09-23 07:48:25,536 [http-bio-6080-exec-67] ERROR org.apache.ranger.biz.ServiceDBStore (ServiceDBStore.java:1610) - ServiceDBStore.getServicePolicies(unifsec_adl_stage_hive): failed to read policies
javax.persistence.NonUniqueResultException: More than one result was returned from Query.getSingleResult()
at org.eclipse.persistence.internal.jpa.QueryImpl.throwNonUniqueResultException(QueryImpl.java:976)
at org.eclipse.persistence.internal.jpa.QueryImpl.getSingleResult(QueryImpl.java:525)
at org.eclipse.persistence.internal.jpa.EJBQueryImpl.getSingleResult(EJBQueryImpl.java:400)
at org.apache.ranger.db.XXPolicyResourceDao.findByResDefIdAndPolicyId(XXPolicyResourceDao.java:39)
at org.apache.ranger.service.RangerPolicyServiceBase.getResourcesForXXPolicy(RangerPolicyServiceBase.java:214)
at org.apache.ranger.service.RangerPolicyService.populateViewBean(RangerPolicyService.java:102)
at org.apache.ranger.service.RangerPolicyService.populateViewBean(RangerPolicyService.java:46)
at org.apache.ranger.service.RangerPolicyServiceBase.searchRangerPolicies(RangerPolicyServiceBase.java:140)
at org.apache.ranger.biz.ServiceDBStore.getPolicies(ServiceDBStore.java:1522)
at org.apache.ranger.biz.ServiceDBStore.getServicePolicies(ServiceDBStore.java:1608)
at org.apache.ranger.biz.ServiceDBStore.getServicePoliciesIfUpdated(ServiceDBStore.java:1668)
at org.apache.ranger.rest.ServiceREST.getServicePoliciesIfUpdated(ServiceREST.java:1251)
at org.apache.ranger.rest.ServiceREST$$FastClassByCGLIB$$92dab672.invoke(<generated>)
at net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:191)
at org.springframework.aop.framework.Cglib2AopProxy$CglibMethodInvocation.invokeJoinpoint(Cglib2AopProxy.java:689)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:110)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.framework.Cglib2AopProxy$DynamicAdvisedInterceptor.intercept(Cglib2AopProxy.java:622)
at org.apache.ranger.rest.ServiceREST$$EnhancerByCGLIB$$97f1eb18.getServicePoliciesIfUpdated(<generated>)
at sun.reflect.GeneratedMethodAccessor108.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$TypeOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:168)
at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:70)
at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:279)
at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:136)
at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:86)
at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:136)
at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:74)
at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1357)
at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1289)
at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1239)
at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1229)
at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:420)
at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:497)
at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:684)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:727)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:186)
at org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:160)
at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:346)
at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:259)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:501)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:171)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:408)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1070)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:611)
at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:314)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
Admitted error thrown by persistence layer is a fatal error and unusual. We should probably return 500 server error back in that case to the caller.