[CLOUDSTACK-7449] "CloudRuntimeException: Can not see storage pool" after trying to add a new host

• Type: Bug
• Status: Open
• Priority: Major
• Resolution: Unresolved
• Affects Version/s: 4.4.1
• Component/s: Management Server, XenServer
• Security Level: Public (Anyone can view this level - this is the default.)
• Environment: XenServer 6.2, CentOS 6.4
• Assignee: Unassigned
• Reporter: Dusan Batas
• Created: 27/Aug/14 14:16
• Updated: 10/Feb/15 20:54

Description

• performed upgrade from 4.3 to 4.4
• upgraded XenServer hosts from 6.0.2 to 6.2
• adding a new host to cluster results with the error

INFO [c.c.n.s.SecurityGroupListener] (catalina-exec-4:ctx-7cb01020 ctx-027ba5b4) Received a host startup notification
INFO [c.c.n.s.SecurityGroupListener] (catalina-exec-4:ctx-7cb01020 ctx-027ba5b4) Scheduled network rules cleanup, interval=2524
INFO [c.c.n.s.SecurityGroupListener] (catalina-exec-4:ctx-7cb01020 ctx-027ba5b4) Received a host startup notification
INFO [c.c.v.VirtualMachinePowerStateSyncImpl] (catalina-exec-4:ctx-7cb01020 ctx-027ba5b4) Reset VM power state sync for host: 78
WARN [c.c.h.x.r.CitrixResourceBase] (DirectAgent-3:ctx-ede0dd1c) ModifyStoragePoolCommand add XenAPIException:Can not see storage pool: 614cda04-67ce-3f18-9721-99b8473b1ff0 from on host:e2cd9dda-8bfe-4cdf-831a-be7d5a26737f host:e2cd9dda-8bfe-4cdf-831a-be7d5a26737f pool: 10.168.12.20/vol1/cluster
com.cloud.utils.exception.CloudRuntimeException: Can not see storage pool: 614cda04-67ce-3f18-9721-99b8473b1ff0 from on host:e2cd9dda-8bfe-4cdf-831a-be7d5a26737f
at com.cloud.hypervisor.xen.resource.CitrixResourceBase.getStorageRepository(CitrixResourceBase.java:6848)
at com.cloud.hypervisor.xen.resource.CitrixResourceBase.execute(CitrixResourceBase.java:5166)
at com.cloud.hypervisor.xen.resource.CitrixResourceBase.executeRequest(CitrixResourceBase.java:476)
at com.cloud.hypervisor.xen.resource.XenServer56Resource.executeRequest(XenServer56Resource.java:61)
at com.cloud.hypervisor.xen.resource.XenServer610Resource.executeRequest(XenServer610Resource.java:102)
at com.cloud.hypervisor.xen.resource.XenServer620SP1Resource.executeRequest(XenServer620SP1Resource.java:65)
at com.cloud.agent.manager.DirectAgentAttache$Task.runInContext(DirectAgentAttache.java:216)
at org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
at org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
WARN [o.a.c.alerts] (catalina-exec-4:ctx-7cb01020 ctx-027ba5b4) alertType:: 7 // dataCenterId:: 1 // podId:: 1 // clusterId:: null // message:: Unable to attach storage pool214 to the host78
WARN [c.c.s.l.StoragePoolMonitor] (catalina-exec-4:ctx-7cb01020 ctx-027ba5b4) Unable to connect host 78 to pool Pool[214|NetworkFilesystem] due to com.cloud.utils.exception.CloudRuntimeException: Unable establish connection from storage head to storage pool 214 due to ModifyStoragePoolCommand add XenAPIException:Can not see storage pool: 614cda04-67ce-3f18-9721-99b8473b1ff0 from on host:e2cd9dda-8bfe-4cdf-831a-be7d5a26737f host:e2cd9dda-8bfe-4cdf-831a-be7d5a26737f pool: 10.168.12.20/vol1/cluster214
com.cloud.utils.exception.CloudRuntimeException: Unable establish connection from storage head to storage pool 214 due to ModifyStoragePoolCommand add XenAPIException:Can not see storage pool: 614cda04-67ce-3f18-9721-99b8473b1ff0 from on host:e2cd9dda-8bfe-4cdf-831a-be7d5a26737f host:e2cd9dda-8bfe-4cdf-831a-be7d5a26737f pool: 10.168.12.20/vol1/cluster214
at org.apache.cloudstack.storage.datastore.provider.DefaultHostListener.hostConnect(DefaultHostListener.java:67)
at com.cloud.storage.StorageManagerImpl.connectHostToSharedPool(StorageManagerImpl.java:866)
at com.cloud.storage.listener.StoragePoolMonitor.processConnect(StoragePoolMonitor.java:103)
at com.cloud.agent.manager.AgentManagerImpl.notifyMonitorsOfConnection(AgentManagerImpl.java:514)
at com.cloud.agent.manager.AgentManagerImpl.handleDirectConnectAgent(AgentManagerImpl.java:1427)
at com.cloud.resource.ResourceManagerImpl.createHostAndAgent(ResourceManagerImpl.java:1765)
at com.cloud.resource.ResourceManagerImpl.discoverHostsFull(ResourceManagerImpl.java:773)
at com.cloud.resource.ResourceManagerImpl.discoverHosts(ResourceManagerImpl.java:586)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
at com.sun.proxy.$Proxy147.discoverHosts(Unknown Source)
at org.apache.cloudstack.api.command.admin.host.AddHostCmd.execute(AddHostCmd.java:142)
at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:141)
at com.cloud.api.ApiServer.queueCommand(ApiServer.java:682)
at com.cloud.api.ApiServer.handleRequest(ApiServer.java:511)
at com.cloud.api.ApiServlet.processRequestInContext(ApiServlet.java:330)
at com.cloud.api.ApiServlet.access$000(ApiServlet.java:54)
at com.cloud.api.ApiServlet$1.run(ApiServlet.java:118)
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
at org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
at com.cloud.api.ApiServlet.processRequest(ApiServlet.java:115)
at com.cloud.api.ApiServlet.doPost(ApiServlet.java:82)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:102)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:555)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:298)
at org.apache.coyote.http11.Http11NioProcessor.process(Http11NioProcessor.java:889)
at org.apache.coyote.http11.Http11NioProtocol$Http11ConnectionHandler.process(Http11NioProtocol.java:721)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:2274)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)