diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryReader.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryReader.java index e8e90fc..eed4b16 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryReader.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryReader.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; +import java.io.IOException; import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; @@ -34,19 +35,22 @@ public interface ApplicationHistoryReader { /** - * This method returns Application {@link ApplicationHistoryData} for the specified - * {@link ApplicationId}. + * This method returns Application {@link ApplicationHistoryData} for the + * specified {@link ApplicationId}. * * @return {@link ApplicationHistoryData} for the ApplicationId. + * @throws {@link IOException} */ - ApplicationHistoryData getApplication(ApplicationId appId); + ApplicationHistoryData getApplication(ApplicationId appId) throws IOException; /** * This method returns all Application {@link ApplicationHistoryData}s * * @return map {@link ApplicationId, @link ApplicationHistoryData}s. + * @throws {@link IOException} */ - Map getAllApplications(); + Map getAllApplications() + throws IOException; /** * Application can have multiple application attempts @@ -54,9 +58,10 @@ * {@link ApplicationAttemptHistoryData}s for the Application. * * @return all {@link ApplicationAttemptHistoryData}s for the Application. + * @throws {@link IOException} */ Map getApplicationAttempts( - ApplicationId appId); + ApplicationId appId) throws IOException; /** * This method returns {@link ApplicationAttemptHistoryData} for specified @@ -64,17 +69,20 @@ * * @param {@link ApplicationAttemptId} * @return {@link ApplicationAttemptHistoryData} for ApplicationAttemptId + * @throws {@link IOException} */ ApplicationAttemptHistoryData getApplicationAttempt( - ApplicationAttemptId appAttemptId); + ApplicationAttemptId appAttemptId) throws IOException; /** - * This method returns {@link Container} for specified {@link ContainerId}. + * This method returns {@link ContainerHistoryData} for specified + * {@link ContainerId}. * * @param {@link ContainerId} - * @return {@link Container} for ContainerId + * @return {@link ContainerHistoryData} for ContainerId + * @throws {@link IOException} */ - ContainerHistoryData getAMContainer(ContainerId containerId); + ContainerHistoryData getContainer(ContainerId containerId) throws IOException; /** * This method returns {@link ContainerHistoryData} for specified @@ -82,6 +90,8 @@ ApplicationAttemptHistoryData getApplicationAttempt( * * @param {@link ApplicationAttemptId} * @return {@link ContainerHistoryData} for ApplicationAttemptId + * @throws {@link IOException} */ - ContainerHistoryData getContainer(ApplicationAttemptId appAttemptId); + ContainerHistoryData getAMContainer(ApplicationAttemptId appAttemptId) + throws IOException; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java new file mode 100644 index 0000000..3d0db8a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +@InterfaceAudience.Public +@InterfaceStability.Unstable +public interface ApplicationHistoryStore extends ApplicationHistoryReader, + ApplicationHistoryWriter { +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java new file mode 100644 index 0000000..25422c9 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; + +public class MemoryApplicationHistoryStore implements ApplicationHistoryStore { + + private static MemoryApplicationHistoryStore memStore = null; + + private ConcurrentHashMap applicationData = + new ConcurrentHashMap(); + private ConcurrentHashMap> applicationAttemptData = + new ConcurrentHashMap>(); + private ConcurrentHashMap containerData = + new ConcurrentHashMap(); + + private MemoryApplicationHistoryStore() { + } + + public static MemoryApplicationHistoryStore getMemoryStore() { + if (memStore == null) { + memStore = new MemoryApplicationHistoryStore(); + } + return memStore; + } + + @Override + public Map getAllApplications() { + Map listApps = + new HashMap(); + for (ApplicationId appId : applicationData.keySet()) { + listApps.put(appId, applicationData.get(appId)); + } + return listApps; + } + + @Override + public ApplicationHistoryData getApplication(ApplicationId appId) { + return applicationData.get(appId); + } + + @Override + public Map getApplicationAttempts( + ApplicationId appId) { + Map listAttempts = + null; + ConcurrentHashMap appAttempts = + applicationAttemptData.get(appId); + if (appAttempts != null) { + listAttempts = + new HashMap(); + for (ApplicationAttemptId attemptId : appAttempts.keySet()) { + listAttempts.put(attemptId, appAttempts.get(attemptId)); + } + } + return listAttempts; + } + + @Override + public ApplicationAttemptHistoryData getApplicationAttempt( + ApplicationAttemptId appAttemptId) { + ApplicationAttemptHistoryData appAttemptHistoryData = null; + ConcurrentHashMap appAttempts = + applicationAttemptData.get(appAttemptId.getApplicationId()); + if (appAttempts != null) { + appAttemptHistoryData = appAttempts.get(appAttemptId); + } + return appAttemptHistoryData; + } + + @Override + public ContainerHistoryData getAMContainer(ApplicationAttemptId appAttemptId) { + ContainerHistoryData Container = null; + ConcurrentHashMap appAttempts = + applicationAttemptData.get(appAttemptId.getApplicationId()); + if (appAttempts != null) { + containerData.get(appAttempts.get(appAttemptId).getMasterContainerId()); + } + return Container; + } + + @Override + public ContainerHistoryData getContainer(ContainerId containerId) { + return containerData.get(containerId); + } + + @Override + public void writeApplication(ApplicationHistoryData app) throws Throwable { + if (app != null) { + ApplicationHistoryData oldData = + applicationData.putIfAbsent(app.getApplicationId(), app); + if (oldData != null) { + throw new IOException("This application " + + app.getApplicationId().toString() + " is already present."); + } + } + } + + @Override + public void writeApplicationAttempt(ApplicationAttemptHistoryData appAttempt) + throws Throwable { + if (appAttempt != null) { + if (applicationAttemptData.containsKey(appAttempt + .getApplicationAttemptId().getApplicationId())) { + ConcurrentHashMap appAttemptmap = + applicationAttemptData.get(appAttempt.getApplicationAttemptId() + .getApplicationId()); + ApplicationAttemptHistoryData oldAppAttempt = + appAttemptmap.putIfAbsent(appAttempt.getApplicationAttemptId(), + appAttempt); + if (oldAppAttempt != null) { + throw new IOException("This application attempt " + + appAttempt.getApplicationAttemptId().toString() + + " already present."); + } + } else { + ConcurrentHashMap appAttemptmap = + new ConcurrentHashMap(); + appAttemptmap.put(appAttempt.getApplicationAttemptId(), appAttempt); + applicationAttemptData.putIfAbsent(appAttempt.getApplicationAttemptId() + .getApplicationId(), appAttemptmap); + } + } + } + + @Override + public void writeContainer(ContainerHistoryData container) throws Throwable { + if (container != null) { + ContainerHistoryData oldContainer = + containerData.putIfAbsent(container.getContainerId(), container); + if (oldContainer != null) { + throw new IOException("This container " + + container.getContainerId().toString() + " is already present."); + } + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java new file mode 100644 index 0000000..feb36d4 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.applicationhistoryservice; + +import java.util.HashMap; + +import junit.framework.Assert; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ApplicationAttemptHistoryDataPBImpl; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ApplicationHistoryDataPBImpl; +import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ContainerHistoryDataPBImpl; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestMemoryApplicationHistoryStore { + MemoryApplicationHistoryStore memstore = null; + + @Before + public void setup() throws Throwable { + memstore = MemoryApplicationHistoryStore.getMemoryStore(); + writeHistoryApplication(); + writeHistoryApplicationAttempt(); + writeContainer(); + } + + public void writeHistoryApplication() throws Throwable { + ApplicationHistoryData appData = new ApplicationHistoryDataPBImpl(); + appData.setApplicationId(ApplicationId.newInstance(1234, 1)); + memstore.writeApplication(appData); + } + + public void writeHistoryApplicationAttempt() throws Throwable { + ApplicationAttemptHistoryData appAttemptHistoryData = + new ApplicationAttemptHistoryDataPBImpl(); + appAttemptHistoryData.setApplicationAttemptId(ApplicationAttemptId + .newInstance(ApplicationId.newInstance(1234, 1), 1)); + memstore.writeApplicationAttempt(appAttemptHistoryData); + } + + public void writeContainer() throws Throwable { + ContainerHistoryData container = new ContainerHistoryDataPBImpl(); + container.setContainerId(ContainerId.newInstance(ApplicationAttemptId + .newInstance(ApplicationId.newInstance(1234, 1), 1), 1)); + memstore.writeContainer(container); + } + + @After + public void tearDown() { + } + + @Test + public void testReadApplication() { + HashMap map = + (HashMap) memstore + .getAllApplications(); + Assert.assertEquals(1, map.size()); + ApplicationHistoryData appData = null; + for (ApplicationId appId : map.keySet()) { + appData = map.get(appId); + Assert.assertEquals("application_1234_0001", appData.getApplicationId() + .toString()); + } + HashMap appAttempts = + (HashMap) memstore + .getApplicationAttempts(appData.getApplicationId()); + Assert.assertEquals(1, appAttempts.size()); + ApplicationAttemptHistoryData appAttempt = null; + for (ApplicationAttemptId appAttemptId : appAttempts.keySet()) { + appAttempt = appAttempts.get(appAttemptId); + Assert.assertEquals("appattempt_1234_0001_000001", appAttempt + .getApplicationAttemptId().toString()); + } + ContainerHistoryData amContainer = + memstore.getContainer(ContainerId.newInstance(ApplicationAttemptId + .newInstance(ApplicationId.newInstance(1234, 1), 1), 1)); + Assert.assertEquals("container_1234_0001_01_000001", amContainer + .getContainerId().toString()); + } +}