diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 34709104264..31859b45993 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -22,6 +22,7 @@ import com.google.protobuf.ByteString; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.UpdateContainerTokenEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent; +import org.apache.hadoop.yarn.server.utils.LeveldbIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -167,15 +168,8 @@ import java.net.InetSocketAddress; import java.net.URISyntaxException; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; @@ -347,25 +341,29 @@ protected ContainersMonitor createContainersMonitor(ContainerExecutor exec) { return new ContainersMonitorImpl(exec, dispatcher, this.context); } + @SuppressWarnings("unchecked") private void recover() throws IOException, URISyntaxException { NMStateStoreService stateStore = context.getNMStateStore(); if (stateStore.canRecover()) { rsrcLocalizationSrvc.recoverLocalizedResources( - stateStore.loadLocalizationState()); + stateStore.loadLocalizationState()); RecoveredApplicationsState appsState = stateStore.loadApplicationsState(); for (ContainerManagerApplicationProto proto : - appsState.getApplications()) { + appsState.getApplications()) { if (LOG.isDebugEnabled()) { LOG.debug("Recovering application with state: " + proto.toString()); } recoverApplication(proto); } - for (RecoveredContainerState rcs : stateStore.loadContainersState()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Recovering container with state: " + rcs); + Iterator it = stateStore.getIterator(); + + while(true) { + RecoveredContainerState rcs = stateStore.getNextRecoveredObject(it); + if (rcs == null) { + break; } recoverContainer(rcs); } @@ -379,8 +377,8 @@ private void recover() throws IOException, URISyntaxException { //so that all the paused, scheduled and queued containers will //be scheduled for execution on availability of resources. dispatcher.getEventHandler().handle( - new ContainerSchedulerEvent(null, - ContainerSchedulerEventType.RECOVERY_COMPLETED)); + new ContainerSchedulerEvent(null, + ContainerSchedulerEventType.RECOVERY_COMPLETED)); } else { LOG.info("Not a recoverable state store. Nothing to recover."); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java index 6f643b04d5f..d0b5e39cf87 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java @@ -65,16 +65,8 @@ import java.io.File; import java.io.IOException; import java.io.Serializable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import java.util.Set; -import java.util.Timer; -import java.util.TimerTask; import static org.fusesource.leveldbjni.JniDBFactory.asString; import static org.fusesource.leveldbjni.JniDBFactory.bytes; @@ -224,6 +216,46 @@ boolean isHealthy() { return isHealthy; } + @Override + public LeveldbIterator getIterator() { + LeveldbIterator iterator = new LeveldbIterator(db); + iterator.seek(bytes(CONTAINERS_KEY_PREFIX)); + return iterator; + } + + + @Override + public RecoveredContainerState getNextRecoveredObject(Iterator i) throws IOException{ + RecoveredContainerState rcs = null; + boolean breakout = false; + LeveldbIterator it = (LeveldbIterator)i; + while(it.hasNext()) { + Entry entry = it.peekNext(); + String key = asString(entry.getKey()); + try { + if (!key.startsWith(CONTAINERS_KEY_PREFIX)) { + return null; + } + int idEndPos = key.indexOf('/', CONTAINERS_KEY_PREFIX.length()); + if (idEndPos < 0) { + throw new IOException("Unable to determine container in key: " + key); + } + ContainerId containerId = ContainerId.fromString( + key.substring(CONTAINERS_KEY_PREFIX.length(), idEndPos)); + String keyPrefix = key.substring(0, idEndPos+1); + rcs = loadContainerState(containerId, + it, keyPrefix); + if(rcs.getStartRequest() != null) { + break; + } else { + removeContainer(containerId); + } + } catch (DBException e) { + + } + } + return rcs; + } @Override public List loadContainersState() throws IOException { @@ -238,6 +270,7 @@ boolean isHealthy() { while (iter.hasNext()) { Entry entry = iter.peekNext(); + String key = asString(entry.getKey()); if (!key.startsWith(CONTAINERS_KEY_PREFIX)) { break; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java index f217f2f8605..911ebb28f01 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.Serializable; +import java.util.Iterator; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -291,4 +292,15 @@ protected void startStorage() throws IOException { @Override protected void closeStorage() throws IOException { } + + @Override + public Iterator getIterator() { + return null; + } + + @Override + public RecoveredContainerState getNextRecoveredObject(Iterator it) throws IOException { + return null; + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java index 0ea0ef3b86c..2a3f81f523f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java @@ -20,10 +20,7 @@ import java.io.IOException; import java.io.Serializable; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -788,4 +785,8 @@ protected void updateContainerResourceMapping(Container container, container.getResourceMappings().addAssignedResources(resourceType, newAssigned); } + + public abstract Iterator getIterator(); + + public abstract RecoveredContainerState getNextRecoveredObject(Iterator it ) throws IOException; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java index b67d11fceb3..0ca069f65be 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java @@ -20,11 +20,7 @@ import java.io.IOException; import java.io.Serializable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -82,6 +78,16 @@ protected void startStorage() { protected void closeStorage() { } + @Override + public Iterator getIterator() { + return null; + } + + @Override + public RecoveredContainerState getNextRecoveredObject(Iterator it) throws IOException { + return null; + } + @Override public synchronized RecoveredApplicationsState loadApplicationsState() diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java index 265b3e68833..9ab0f9da11d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java @@ -89,6 +89,7 @@ import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager; import org.apache.hadoop.yarn.server.security.BaseNMTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.server.utils.LeveldbIterator; import org.iq80.leveldb.DB; import org.iq80.leveldb.DBException; import org.junit.After; @@ -96,6 +97,7 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; +import java.lang.instrument.Instrumentation; public class TestNMLeveldbStateStoreService { private static final File TMP_DIR = new File( @@ -226,12 +228,25 @@ public void testApplicationStorage() throws IOException { assertEquals(appProto1, state.getApplications().get(0)); } + + private RecoveredContainerState getOne(LeveldbIterator it) throws IOException { + RecoveredContainerState rcs = null; + rcs = stateStore.getNextRecoveredObject(it); + assertNull(stateStore.getNextRecoveredObject(it)); + assertNotNull(rcs); + return rcs; + } + + private boolean isStoreEmpty() throws IOException{ + return stateStore.getNextRecoveredObject(stateStore.getIterator()) == null; + } + @Test public void testContainerStorage() throws IOException { // test empty when no state - List recoveredContainers = - stateStore.loadContainersState(); - assertTrue(recoveredContainers.isEmpty()); +// List recoveredContainers = +// stateStore.loadContainersState(); +// assertTrue(recoveredContainers.isEmpty()); // create a container request ApplicationId appId = ApplicationId.newInstance(1234, 3); @@ -250,35 +265,36 @@ public void testContainerStorage() throws IOException { stateStore.getContainerVersionKey(containerId.toString())))); restartStateStore(); - recoveredContainers = stateStore.loadContainersState(); - assertEquals(1, recoveredContainers.size()); - RecoveredContainerState rcs = recoveredContainers.get(0); - assertEquals(0, rcs.getVersion()); - assertEquals(containerStartTime, rcs.getStartTime()); - assertEquals(RecoveredContainerStatus.REQUESTED, rcs.getStatus()); - assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode()); - assertEquals(false, rcs.getKilled()); - assertEquals(containerReq, rcs.getStartRequest()); - assertTrue(rcs.getDiagnostics().isEmpty()); + LeveldbIterator it = stateStore.getIterator(); + RecoveredContainerState recoveredContainer = getOne(it); + assertNotNull(recoveredContainer); + + //recoveredContainer = stateStore.loadContainersState(); + assertEquals(0, recoveredContainer.getVersion()); + assertEquals(containerStartTime, recoveredContainer.getStartTime()); + assertEquals(RecoveredContainerStatus.REQUESTED, recoveredContainer.getStatus()); + assertEquals(ContainerExitStatus.INVALID, recoveredContainer.getExitCode()); + assertEquals(false, recoveredContainer.getKilled()); + assertEquals(containerReq, recoveredContainer.getStartRequest()); + assertTrue(recoveredContainer.getDiagnostics().isEmpty()); // store a new container record without StartContainerRequest ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 6); stateStore.storeContainerLaunched(containerId1); - recoveredContainers = stateStore.loadContainersState(); - // check whether the new container record is discarded - assertEquals(1, recoveredContainers.size()); + it = stateStore.getIterator(); + recoveredContainer = getOne(it); + assertNotNull(recoveredContainer); + // queue the container, and verify recovered stateStore.storeContainerQueued(containerId); restartStateStore(); - recoveredContainers = stateStore.loadContainersState(); - assertEquals(1, recoveredContainers.size()); - rcs = recoveredContainers.get(0); - assertEquals(RecoveredContainerStatus.QUEUED, rcs.getStatus()); - assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode()); - assertEquals(false, rcs.getKilled()); - assertEquals(containerReq, rcs.getStartRequest()); - assertTrue(rcs.getDiagnostics().isEmpty()); + recoveredContainer = getOne(stateStore.getIterator()); + assertEquals(RecoveredContainerStatus.QUEUED, recoveredContainer.getStatus()); + assertEquals(ContainerExitStatus.INVALID, recoveredContainer.getExitCode()); + assertEquals(false, recoveredContainer.getKilled()); + assertEquals(containerReq, recoveredContainer.getStartRequest()); + assertTrue(recoveredContainer.getDiagnostics().isEmpty()); // launch the container, add some diagnostics, and verify recovered StringBuilder diags = new StringBuilder(); @@ -286,33 +302,31 @@ public void testContainerStorage() throws IOException { diags.append("some diags for container"); stateStore.storeContainerDiagnostics(containerId, diags); restartStateStore(); - recoveredContainers = stateStore.loadContainersState(); - assertEquals(1, recoveredContainers.size()); - rcs = recoveredContainers.get(0); - assertEquals(RecoveredContainerStatus.LAUNCHED, rcs.getStatus()); - assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode()); - assertEquals(false, rcs.getKilled()); - assertEquals(containerReq, rcs.getStartRequest()); - assertEquals(diags.toString(), rcs.getDiagnostics()); + recoveredContainer = getOne(stateStore.getIterator()); + assertEquals(RecoveredContainerStatus.LAUNCHED, recoveredContainer.getStatus()); + assertEquals(ContainerExitStatus.INVALID, recoveredContainer.getExitCode()); + assertEquals(false, recoveredContainer.getKilled()); + assertEquals(containerReq, recoveredContainer.getStartRequest()); + assertEquals(diags.toString(), recoveredContainer.getDiagnostics()); // pause the container, and verify recovered stateStore.storeContainerPaused(containerId); restartStateStore(); - recoveredContainers = stateStore.loadContainersState(); - assertEquals(1, recoveredContainers.size()); - rcs = recoveredContainers.get(0); + RecoveredContainerState rcs = getOne(stateStore.getIterator()); assertEquals(RecoveredContainerStatus.PAUSED, rcs.getStatus()); assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode()); assertEquals(false, rcs.getKilled()); assertEquals(containerReq, rcs.getStartRequest()); + + // Resume the container stateStore.removeContainerPaused(containerId); restartStateStore(); - recoveredContainers = stateStore.loadContainersState(); - assertEquals(1, recoveredContainers.size()); + recoveredContainer = getOne(stateStore.getIterator()); + - // increase the container size, and verify recovered + // increase the container size, and verify recovered ContainerTokenIdentifier updateTokenIdentifier = new ContainerTokenIdentifier(containerId, "host", "user", Resource.newInstance(2468, 4), 9876543210L, 42, 2468, @@ -321,9 +335,7 @@ public void testContainerStorage() throws IOException { stateStore .storeContainerUpdateToken(containerId, updateTokenIdentifier); restartStateStore(); - recoveredContainers = stateStore.loadContainersState(); - assertEquals(1, recoveredContainers.size()); - rcs = recoveredContainers.get(0); + rcs = getOne(stateStore.getIterator()); assertEquals(0, rcs.getVersion()); assertEquals(RecoveredContainerStatus.LAUNCHED, rcs.getStatus()); assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode()); @@ -335,9 +347,7 @@ public void testContainerStorage() throws IOException { stateStore.storeContainerDiagnostics(containerId, diags); stateStore.storeContainerKilled(containerId); restartStateStore(); - recoveredContainers = stateStore.loadContainersState(); - assertEquals(1, recoveredContainers.size()); - rcs = recoveredContainers.get(0); + rcs = getOne(stateStore.getIterator()); assertEquals(RecoveredContainerStatus.LAUNCHED, rcs.getStatus()); assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode()); assertTrue(rcs.getKilled()); @@ -351,22 +361,18 @@ public void testContainerStorage() throws IOException { stateStore.storeContainerDiagnostics(containerId, diags); stateStore.storeContainerCompleted(containerId, 21); restartStateStore(); - recoveredContainers = stateStore.loadContainersState(); - assertEquals(1, recoveredContainers.size()); - rcs = recoveredContainers.get(0); + rcs = getOne(stateStore.getIterator()); assertEquals(RecoveredContainerStatus.COMPLETED, rcs.getStatus()); assertEquals(21, rcs.getExitCode()); assertTrue(rcs.getKilled()); assertEquals(diags.toString(), rcs.getDiagnostics()); - +// // store remainingRetryAttempts, workDir and logDir stateStore.storeContainerRemainingRetryAttempts(containerId, 6); stateStore.storeContainerWorkDir(containerId, "/test/workdir"); stateStore.storeContainerLogDir(containerId, "/test/logdir"); restartStateStore(); - recoveredContainers = stateStore.loadContainersState(); - assertEquals(1, recoveredContainers.size()); - rcs = recoveredContainers.get(0); + rcs = getOne(stateStore.getIterator()); assertEquals(6, rcs.getRemainingRetryAttempts()); assertEquals("/test/workdir", rcs.getWorkDir()); assertEquals("/test/logdir", rcs.getLogDir()); @@ -375,8 +381,8 @@ public void testContainerStorage() throws IOException { // remove the container and verify not recovered stateStore.removeContainer(containerId); restartStateStore(); - recoveredContainers = stateStore.loadContainersState(); - assertTrue(recoveredContainers.isEmpty()); + //make sure nothing is in the store + assertNull(stateStore.getNextRecoveredObject(stateStore.getIterator())); } private void validateRetryAttempts(ContainerId containerId) @@ -387,7 +393,7 @@ private void validateRetryAttempts(ContainerId containerId) stateStore.storeContainerRestartTimes(containerId, finishTimeForRetryAttempts); restartStateStore(); - RecoveredContainerState rcs = stateStore.loadContainersState().get(0); + RecoveredContainerState rcs = getOne(stateStore.getIterator()); List recoveredRestartTimes = rcs.getRestartTimes(); assertEquals(1462700529039L, (long)recoveredRestartTimes.get(0)); assertEquals(1462700529050L, (long)recoveredRestartTimes.get(1)); @@ -1003,9 +1009,7 @@ protected DB openDatabase(Configuration conf) { @Test public void testUnexpectedKeyDoesntThrowException() throws IOException { // test empty when no state - List recoveredContainers = stateStore - .loadContainersState(); - assertTrue(recoveredContainers.isEmpty()); + assertTrue(isStoreEmpty()); ApplicationId appId = ApplicationId.newInstance(1234, 3); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, @@ -1019,9 +1023,7 @@ public void testUnexpectedKeyDoesntThrowException() throws IOException { + containerId.toString() + "/invalidKey1234").getBytes(); stateStore.getDB().put(invalidKey, new byte[1]); restartStateStore(); - recoveredContainers = stateStore.loadContainersState(); - assertEquals(1, recoveredContainers.size()); - RecoveredContainerState rcs = recoveredContainers.get(0); + RecoveredContainerState rcs = getOne(stateStore.getIterator()); assertEquals(RecoveredContainerStatus.REQUESTED, rcs.getStatus()); assertEquals(ContainerExitStatus.INVALID, rcs.getExitCode()); assertEquals(false, rcs.getKilled()); @@ -1136,9 +1138,7 @@ public void testAMRMProxyStorage() throws IOException { @Test public void testStateStoreForResourceMapping() throws IOException { // test empty when no state - List recoveredContainers = stateStore - .loadContainersState(); - assertTrue(recoveredContainers.isEmpty()); + assertTrue(isStoreEmpty()); ApplicationId appId = ApplicationId.newInstance(1234, 3); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, @@ -1164,9 +1164,7 @@ public void testStateStoreForResourceMapping() throws IOException { // add a invalid key restartStateStore(); - recoveredContainers = stateStore.loadContainersState(); - assertEquals(1, recoveredContainers.size()); - RecoveredContainerState rcs = recoveredContainers.get(0); + RecoveredContainerState rcs = getOne(stateStore.getIterator()); List res = rcs.getResourceMappings() .getAssignedResources("gpu"); Assert.assertTrue(res.equals(gpuRes1)); @@ -1227,7 +1225,7 @@ public void testEmptyRestartTimes() throws IOException { stateStore.storeContainerRestartTimes(containerId, restartTimes); restartStateStore(); - RecoveredContainerState rcs = stateStore.loadContainersState().get(0); + RecoveredContainerState rcs = getOne(stateStore.getIterator()); List recoveredRestartTimes = rcs.getRestartTimes(); assertTrue(recoveredRestartTimes.isEmpty()); }