diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java index 0dfdf20..2cff327 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java @@ -46,8 +46,22 @@ Resource getReservedResource(); + Resource getAllocatedResource(); + NodeId getReservedNode(); + + NodeId getAssignedNode(); Priority getReservedPriority(); + Priority getAssignedPriority(); + + long getStartTime(); + + long getFinishTime(); + + StringBuilder getDiagnostics(); + + String getLogURL(); + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index d44fd3f..5e794a5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; +import static org.apache.hadoop.yarn.util.StringHelper.join; + import java.util.EnumSet; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; @@ -25,6 +27,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -40,6 +43,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; +import org.apache.hadoop.yarn.util.ConverterUtils; @SuppressWarnings({"unchecked", "rawtypes"}) public class RMContainerImpl implements RMContainer { @@ -135,10 +139,17 @@ private final Container container; private final EventHandler eventHandler; private final ContainerAllocationExpirer containerAllocationExpirer; + private final StringBuilder diagnostics = new StringBuilder(); private Resource reservedResource; + private Resource allocatedResource; private NodeId reservedNode; + private NodeId assignedNode; private Priority reservedPriority; + private Priority assignedPriority; + private long startTime; + private long finishTime; + private String logURL; public RMContainerImpl(Container container, ApplicationAttemptId appAttemptId, NodeId nodeId, @@ -151,6 +162,7 @@ public RMContainerImpl(Container container, this.appAttemptId = appAttemptId; this.eventHandler = handler; this.containerAllocationExpirer = containerAllocationExpirer; + this.startTime = System.currentTimeMillis(); ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); @@ -185,24 +197,101 @@ public RMContainerState getState() { @Override public Resource getReservedResource() { - return reservedResource; + this.readLock.lock(); + + try { + return this.reservedResource; + } finally { + this.readLock.unlock(); + } + } + + @Override + public Resource getAllocatedResource() { + this.readLock.lock(); + + try { + return this.allocatedResource; + } finally { + this.readLock.unlock(); + } } @Override public NodeId getReservedNode() { - return reservedNode; + this.readLock.lock(); + + try { + return this.reservedNode; + } finally { + this.readLock.unlock(); + } + } + + @Override + public NodeId getAssignedNode() { + this.readLock.lock(); + + try { + return this.assignedNode; + } finally { + this.readLock.unlock(); + } } @Override public Priority getReservedPriority() { return reservedPriority; } - + + @Override + public Priority getAssignedPriority() { + return assignedPriority; + } + + @Override + public long getStartTime() { + return startTime; + } + + @Override + public long getFinishTime() { + this.readLock.lock(); + + try { + return this.finishTime; + } finally { + this.readLock.unlock(); + } + } + + @Override + public StringBuilder getDiagnostics() { + this.readLock.lock(); + + try { + return this.diagnostics; + } finally { + this.readLock.unlock(); + } + } + + @Override + public String getLogURL() { + this.readLock.lock(); + + try { + return this.logURL; + } finally { + this.readLock.unlock(); + } + } + @Override public String toString() { return containerId.toString(); } - + @Override public void handle(RMContainerEvent event) { LOG.debug("Processing " + event.getContainerId() + " of type " + event.getType()); @@ -256,6 +345,10 @@ public void transition(RMContainerImpl container, RMContainerEvent event) { public void transition(RMContainerImpl container, RMContainerEvent event) { container.eventHandler.handle(new RMAppAttemptContainerAllocatedEvent( container.appAttemptId, container.container)); + RMContainerStartedEvent e = (RMContainerStartedEvent) event; + container.allocatedResource = e.getAllocatedResource(); + container.assignedNode = e.getAssignedNode(); + container.assignedPriority = e.getAssignedPriority(); } } @@ -279,6 +372,10 @@ public void transition(RMContainerImpl container, RMContainerEvent event) { // Unregister from containerAllocationExpirer. container.containerAllocationExpirer.unregister(container .getContainerId()); + RMContainerLaunchedEvent e = (RMContainerLaunchedEvent) event; + container.logURL = join(HttpConfig.getSchemePrefix(), + container.container.getNodeHttpAddress(), "/node", "/containerlogs/", + ConverterUtils.toString(container.containerId), "/", e.getUser()); } } @@ -291,6 +388,9 @@ public void transition(RMContainerImpl container, RMContainerEvent event) { // Inform AppAttempt container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent( container.appAttemptId, finishedEvent.getRemoteContainerStatus())); + container.finishTime = System.currentTimeMillis(); + container.diagnostics.append( + finishedEvent.getRemoteContainerStatus().getDiagnostics()); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerLaunchedEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerLaunchedEvent.java new file mode 100644 index 0000000..33237f7 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerLaunchedEvent.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; + +import org.apache.hadoop.yarn.api.records.ContainerId; + + +public class RMContainerLaunchedEvent extends RMContainerEvent { + + private String user; + + public RMContainerLaunchedEvent(ContainerId containerId, String user) { + super(containerId, RMContainerEventType.LAUNCHED); + this.user = user; + } + + public String getUser() { + return user; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerStartedEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerStartedEvent.java new file mode 100644 index 0000000..4c7e80d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerStartedEvent.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; + + +public class RMContainerStartedEvent extends RMContainerEvent { + + private Resource allocatedResource; + private NodeId assignedNode; + private Priority assignedPriority; + + public RMContainerStartedEvent(ContainerId containerId, + Resource allocatedResource, NodeId assignedNode, + Priority assignedPriority) { + super(containerId, RMContainerEventType.START); + this.allocatedResource = allocatedResource; + this.assignedNode = assignedNode; + this.assignedPriority = assignedPriority; + } + + public Resource getAllocatedResource() { + return allocatedResource; + } + + public NodeId getAssignedNode() { + return assignedNode; + } + + public Priority getAssignedPriority() { + return assignedPriority; + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index a261dbf..d60e1e8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -48,6 +48,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerLaunchedEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerStartedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent; @@ -221,8 +223,8 @@ public synchronized void containerLaunchedOnNode(ContainerId containerId, return; } - rmContainer.handle(new RMContainerEvent(containerId, - RMContainerEventType.LAUNCHED)); + rmContainer.handle(new RMContainerLaunchedEvent(containerId, + this.appSchedulingInfo.getUser())); } synchronized public boolean containerCompleted(RMContainer rmContainer, @@ -289,8 +291,8 @@ synchronized public RMContainer allocate(NodeType type, FiCaSchedulerNode node, Resources.addTo(currentConsumption, container.getResource()); // Inform the container - rmContainer.handle( - new RMContainerEvent(container.getId(), RMContainerEventType.START)); + rmContainer.handle(new RMContainerStartedEvent(container.getId(), + container.getResource(), node.getNodeID(), priority)); if (LOG.isDebugEnabled()) { LOG.debug("allocate: applicationAttemptId=" diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java index 670e961..17a1798 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java @@ -45,6 +45,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerLaunchedEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerStartedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent; @@ -205,8 +207,8 @@ public synchronized void containerLaunchedOnNode(ContainerId containerId, return; } - rmContainer.handle(new RMContainerEvent(containerId, - RMContainerEventType.LAUNCHED)); + rmContainer.handle(new RMContainerLaunchedEvent(containerId, + this.appSchedulingInfo.getUser())); } synchronized public void containerCompleted(RMContainer rmContainer, @@ -552,8 +554,8 @@ else if (allowed.equals(NodeType.RACK_LOCAL) && Resources.addTo(currentConsumption, container.getResource()); // Inform the container - rmContainer.handle( - new RMContainerEvent(container.getId(), RMContainerEventType.START)); + rmContainer.handle(new RMContainerStartedEvent(container.getId(), + container.getResource(), node.getNodeID(), priority)); if (LOG.isDebugEnabled()) { LOG.debug("allocate: applicationAttemptId=" diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index d1262d8..14a85df 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; @@ -46,6 +47,7 @@ @SuppressWarnings({ "unchecked", "rawtypes" }) public class TestRMContainerImpl { + @SuppressWarnings("resource") @Test public void testReleaseWhileRunning() { @@ -75,21 +77,27 @@ public void testReleaseWhileRunning() { nodeId, eventHandler, expirer); assertEquals(RMContainerState.NEW, rmContainer.getState()); + assertTrue(rmContainer.getStartTime() > 0); - rmContainer.handle(new RMContainerEvent(containerId, - RMContainerEventType.START)); + rmContainer.handle(new RMContainerStartedEvent( + containerId, resource, nodeId, priority)); drainDispatcher.await(); assertEquals(RMContainerState.ALLOCATED, rmContainer.getState()); + assertEquals(resource, rmContainer.getAllocatedResource()); + assertEquals(nodeId, rmContainer.getAssignedNode()); + assertEquals(priority, rmContainer.getAssignedPriority()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED)); drainDispatcher.await(); assertEquals(RMContainerState.ACQUIRED, rmContainer.getState()); - rmContainer.handle(new RMContainerEvent(containerId, - RMContainerEventType.LAUNCHED)); + rmContainer.handle(new RMContainerLaunchedEvent(containerId, "user")); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); + assertEquals( + "http://host:3465/node/containerlogs/container_1_0001_01_000001/user", + rmContainer.getLogURL()); // In RUNNING state. Verify RELEASED and associated actions. reset(appAttemptEventHandler); @@ -100,6 +108,9 @@ public void testReleaseWhileRunning() { containerStatus, RMContainerEventType.RELEASED)); drainDispatcher.await(); assertEquals(RMContainerState.RELEASED, rmContainer.getState()); + assertEquals(SchedulerUtils.RELEASED_CONTAINER, + rmContainer.getDiagnostics().toString()); + assertTrue(rmContainer.getFinishTime() > 0); ArgumentCaptor captor = ArgumentCaptor .forClass(RMAppAttemptContainerFinishedEvent.class); @@ -116,6 +127,7 @@ public void testReleaseWhileRunning() { assertEquals(RMContainerState.RELEASED, rmContainer.getState()); } + @SuppressWarnings("resource") @Test public void testExpireWhileRunning() { @@ -146,20 +158,25 @@ public void testExpireWhileRunning() { assertEquals(RMContainerState.NEW, rmContainer.getState()); - rmContainer.handle(new RMContainerEvent(containerId, - RMContainerEventType.START)); + rmContainer.handle(new RMContainerStartedEvent( + containerId, resource, nodeId, priority)); drainDispatcher.await(); assertEquals(RMContainerState.ALLOCATED, rmContainer.getState()); + assertEquals(resource, rmContainer.getAllocatedResource()); + assertEquals(nodeId, rmContainer.getAssignedNode()); + assertEquals(priority, rmContainer.getAssignedPriority()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED)); drainDispatcher.await(); assertEquals(RMContainerState.ACQUIRED, rmContainer.getState()); - rmContainer.handle(new RMContainerEvent(containerId, - RMContainerEventType.LAUNCHED)); + rmContainer.handle(new RMContainerLaunchedEvent(containerId, "user")); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); + assertEquals( + "http://host:3465/node/containerlogs/container_1_0001_01_000001/user", + rmContainer.getLogURL()); // In RUNNING state. Verify EXPIRE and associated actions. reset(appAttemptEventHandler);