diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 38eb636f746..a00949748a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -480,6 +480,7 @@ protected void recoverActiveContainer(Application app, Container container = new ContainerImpl(getConfig(), dispatcher, launchContext, credentials, metrics, token, context, rcs); context.getContainers().put(token.getContainerID(), container); + containerScheduler.recoverActiveContainer(container, rcs.getStatus()); app.handle(new ApplicationContainerInitEvent(container)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index df837891d82..3e383284d27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -1060,17 +1060,16 @@ public void transition( UpdateContainerTokenEvent updateEvent = (UpdateContainerTokenEvent)event; // Update the container token container.setContainerTokenIdentifier(updateEvent.getUpdatedToken()); - if (updateEvent.isResourceChange()) { - try { - // Persist change in the state store. - container.context.getNMStateStore().storeContainerResourceChanged( - container.containerId, - container.getContainerTokenIdentifier().getVersion(), - container.getResource()); - } catch (IOException e) { - LOG.warn("Could not store container [" + container.containerId - + "] resource change..", e); - } + + boolean isResourceChange = updateEvent.isResourceChange(); + try { + // Persist change in the state store. + container.context.getNMStateStore() + .storeContainerUpdateToken(container.containerId, + container.getContainerTokenIdentifier(), isResourceChange); + } catch (IOException e) { + LOG.warn("Could not store container [" + container.containerId + + "] update..", e); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java index e43682214e6..8d7e6053964 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit; import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus; import org.apache.hadoop.yarn.server.nodemanager.Context; @@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -219,6 +221,33 @@ private void onUpdateContainer(UpdateContainerSchedulerEvent updateEvent) { } /** + * Populates auxiliary data structures used by the ContainerScheduler on + * recovery + * @param container container recovered + * @param rcs Recovered Container status + * @throws IOException + */ + public void recoverActiveContainer(Container container, + RecoveredContainerStatus rcs) throws IOException { + ExecutionType execType = + container.getContainerTokenIdentifier().getExecutionType(); + if (rcs == RecoveredContainerStatus.QUEUED + || rcs == RecoveredContainerStatus.PAUSED) { + if (execType == ExecutionType.GUARANTEED) { + queuedGuaranteedContainers.put(container.getContainerId(), container); + } else if (execType == ExecutionType.OPPORTUNISTIC) { + queuedOpportunisticContainers + .put(container.getContainerId(), container); + } else { + new YarnException("Unknown executiontype for container " + container + .getContainerId()); + } + } else if (rcs == RecoveredContainerStatus.LAUNCHED) { + runningContainers.put(container.getContainerId(), container); + } + } + + /** * Return number of queued containers. * @return Number of queued containers. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java index 2f9c0a75065..773d21433bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java @@ -20,6 +20,11 @@ import static org.fusesource.leveldbjni.JniDBFactory.asString; import static org.fusesource.leveldbjni.JniDBFactory.bytes; + +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.slf4j.LoggerFactory; import java.io.File; @@ -60,6 +65,7 @@ import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto; +import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.ContainerTokenIdentifierProto; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings; @@ -122,6 +128,8 @@ private static final String CONTAINER_PAUSED_KEY_SUFFIX = "/paused"; private static final String CONTAINER_RESOURCE_CHANGED_KEY_SUFFIX = "/resourceChanged"; + private static final String CONTAINER_UPDATE_TOKEN_SUFFIX = + "/updateToken"; private static final String CONTAINER_KILLED_KEY_SUFFIX = "/killed"; private static final String CONTAINER_EXIT_CODE_KEY_SUFFIX = "/exitcode"; private static final String CONTAINER_REMAIN_RETRIES_KEY_SUFFIX = @@ -291,8 +299,20 @@ private RecoveredContainerState loadContainerState(ContainerId containerId, rcs.status = RecoveredContainerStatus.COMPLETED; rcs.exitCode = Integer.parseInt(asString(entry.getValue())); } else if (suffix.equals(CONTAINER_RESOURCE_CHANGED_KEY_SUFFIX)) { - rcs.capability = new ResourcePBImpl( - ResourceProto.parseFrom(entry.getValue())); + // For backward compatibility, we have to add this key. For now, do + // nothing on recovery, as the resource change is already captured as a + // part of container_update_token_suffix + } else if (suffix.equals(CONTAINER_UPDATE_TOKEN_SUFFIX)) { + ContainerTokenIdentifierProto tokenIdentifierProto = + ContainerTokenIdentifierProto.parseFrom(entry.getValue()); + Token currentToken = rcs.getStartRequest().getContainerToken(); + Token updatedToken = Token + .newInstance(tokenIdentifierProto.toByteArray(), + ContainerTokenIdentifier.KIND.toString(), + currentToken.getPassword().array(), currentToken.getService()); + rcs.startRequest.setContainerToken(updatedToken); + rcs.capability = new ResourcePBImpl(tokenIdentifierProto.getResource()); + rcs.version = tokenIdentifierProto.getVersion(); } else if (suffix.equals(CONTAINER_REMAIN_RETRIES_KEY_SUFFIX)) { rcs.setRemainingRetryAttempts( Integer.parseInt(asString(entry.getValue()))); @@ -374,6 +394,21 @@ public void storeContainerQueued(ContainerId containerId) throws IOException { } } + private void removeContainerQueued(ContainerId containerId) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("removeContainerQueued: containerId=" + containerId); + } + + String key = CONTAINERS_KEY_PREFIX + containerId.toString() + + CONTAINER_QUEUED_KEY_SUFFIX; + try { + db.delete(bytes(key)); + } catch (DBException e) { + throw new IOException(e); + } + } + @Override public void storeContainerPaused(ContainerId containerId) throws IOException { if (LOG.isDebugEnabled()) { @@ -429,6 +464,8 @@ public void storeContainerLaunched(ContainerId containerId) LOG.debug("storeContainerLaunched: containerId=" + containerId); } + // Removing the container if queued for backward compatibility reasons + removeContainerQueued(containerId); String key = CONTAINERS_KEY_PREFIX + containerId.toString() + CONTAINER_LAUNCHED_KEY_SUFFIX; try { @@ -439,24 +476,34 @@ public void storeContainerLaunched(ContainerId containerId) } @Override - public void storeContainerResourceChanged(ContainerId containerId, - int containerVersion, Resource capability) throws IOException { + public void storeContainerUpdateToken(ContainerId containerId, + ContainerTokenIdentifier containerTokenIdentifier, + boolean isResourceChange) throws IOException { if (LOG.isDebugEnabled()) { - LOG.debug("storeContainerResourceChanged: containerId=" + containerId - + ", capability=" + capability); + LOG.debug("storeContainerUpdateToken: containerId=" + containerId); } - String keyResChng = CONTAINERS_KEY_PREFIX + containerId.toString() - + CONTAINER_RESOURCE_CHANGED_KEY_SUFFIX; + String keyUpdateToken = CONTAINERS_KEY_PREFIX + containerId.toString() + + CONTAINER_UPDATE_TOKEN_SUFFIX; String keyVersion = CONTAINERS_KEY_PREFIX + containerId.toString() + CONTAINER_VERSION_KEY_SUFFIX; + + String keyResChng = CONTAINERS_KEY_PREFIX + containerId.toString() + + CONTAINER_RESOURCE_CHANGED_KEY_SUFFIX; + try { WriteBatch batch = db.createWriteBatch(); try { // New value will overwrite old values for the same key - batch.put(bytes(keyResChng), - ProtoUtils.convertToProtoFormat(capability).toByteArray()); - batch.put(bytes(keyVersion), bytes(Integer.toString(containerVersion))); + batch.put(bytes(keyUpdateToken), + containerTokenIdentifier.getProto().toByteArray()); + if(isResourceChange) { + batch.put(bytes(keyResChng), ProtoUtils + .convertToProtoFormat(containerTokenIdentifier.getResource()) + .toByteArray()); + } + batch.put(bytes(keyVersion), + bytes(Integer.toString(containerTokenIdentifier.getVersion()))); db.write(batch); } finally { batch.close(); @@ -552,6 +599,7 @@ public void removeContainer(ContainerId containerId) batch.delete(bytes(keyPrefix + CONTAINER_PAUSED_KEY_SUFFIX)); batch.delete(bytes(keyPrefix + CONTAINER_KILLED_KEY_SUFFIX)); batch.delete(bytes(keyPrefix + CONTAINER_EXIT_CODE_KEY_SUFFIX)); + batch.delete(bytes(keyPrefix + CONTAINER_UPDATE_TOKEN_SUFFIX)); List unknownKeysForContainer = containerUnknownKeySuffixes .removeAll(containerId); for (String unknownKeySuffix : unknownKeysForContainer) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java index d1d0696935f..84473112304 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.records.MasterKey; // The state store to use when state isn't being stored @@ -99,8 +100,9 @@ public void storeContainerLaunched(ContainerId containerId) } @Override - public void storeContainerResourceChanged(ContainerId containerId, - int version, Resource capability) throws IOException { + public void storeContainerUpdateToken(ContainerId containerId, + ContainerTokenIdentifier containerTokenIdentifier, + boolean isResourceChange) throws IOException { } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java index 999d2d9dbaf..0c3bb0f93dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java @@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings; @@ -429,14 +430,14 @@ public abstract void storeContainerLaunched(ContainerId containerId) throws IOException; /** - * Record that a container resource has been changed + * Record that a container has been updated * @param containerId the container ID - * @param containerVersion the container version - * @param capability the container resource capability + * @param containerTokenIdentifier container token identifier * @throws IOException */ - public abstract void storeContainerResourceChanged(ContainerId containerId, - int containerVersion, Resource capability) throws IOException; + public abstract void storeContainerUpdateToken(ContainerId containerId, + ContainerTokenIdentifier containerTokenIdentifier, + boolean isResourceChange) throws IOException; /** * Record that a container has completed diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java index 59a225ad188..230516ecbea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java @@ -34,11 +34,14 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.ContainerManagerApplicationProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings; @@ -175,12 +178,18 @@ public synchronized void storeContainerLaunched(ContainerId containerId) } @Override - public synchronized void storeContainerResourceChanged( - ContainerId containerId, int version, Resource capability) - throws IOException { + public void storeContainerUpdateToken(ContainerId containerId, + ContainerTokenIdentifier containerTokenIdentifier, + boolean isResourceChange) throws IOException { RecoveredContainerState rcs = getRecoveredContainerState(containerId); - rcs.capability = capability; - rcs.version = version; + rcs.capability = containerTokenIdentifier.getResource(); + rcs.version = containerTokenIdentifier.getVersion(); + Token currentToken = rcs.getStartRequest().getContainerToken(); + Token updatedToken = Token + .newInstance(containerTokenIdentifier.getBytes(), + ContainerTokenIdentifier.KIND.toString(), + currentToken.getPassword().array(), currentToken.getService()); + rcs.startRequest.setContainerToken(updatedToken); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java index 8c13356cb22..fac84abdb5a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java @@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -307,13 +308,18 @@ public void testContainerStorage() throws IOException { assertEquals(1, recoveredContainers.size()); // increase the container size, and verify recovered - stateStore.storeContainerResourceChanged(containerId, 2, - Resource.newInstance(2468, 4)); + ContainerTokenIdentifier updateTokenIdentifier = + new ContainerTokenIdentifier(containerId, "host", "user", + Resource.newInstance(2468, 4), 9876543210L, 42, 2468, + Priority.newInstance(7), 13579); + + stateStore + .storeContainerUpdateToken(containerId, updateTokenIdentifier, true); restartStateStore(); recoveredContainers = stateStore.loadContainersState(); assertEquals(1, recoveredContainers.size()); rcs = recoveredContainers.get(0); - assertEquals(2, rcs.getVersion()); + assertEquals(0, rcs.getVersion()); assertEquals(RecoveredContainerStatus.LAUNCHED, rcs.getStatus()); assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode()); assertEquals(false, rcs.getKilled()); @@ -330,7 +336,9 @@ public void testContainerStorage() throws IOException { assertEquals(RecoveredContainerStatus.LAUNCHED, rcs.getStatus()); assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode()); assertTrue(rcs.getKilled()); - assertEquals(containerReq, rcs.getStartRequest()); + ContainerTokenIdentifier tokenReadFromRequest = BuilderUtils + .newContainerTokenIdentifier(rcs.getStartRequest().getContainerToken()); + assertEquals(updateTokenIdentifier, tokenReadFromRequest); assertEquals(diags.toString(), rcs.getDiagnostics()); // add yet more diags, mark container completed, and verify recovered @@ -344,7 +352,6 @@ public void testContainerStorage() throws IOException { assertEquals(RecoveredContainerStatus.COMPLETED, rcs.getStatus()); assertEquals(21, rcs.getExitCode()); assertTrue(rcs.getKilled()); - assertEquals(containerReq, rcs.getStartRequest()); assertEquals(diags.toString(), rcs.getDiagnostics()); // store remainingRetryAttempts, workDir and logDir