diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java index 2334fde..e7ecdf2 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.timeline.security.authorize.TimelinePolicyProvider; +import org.apache.hadoop.yarn.webapp.NotFoundException; public class ApplicationHistoryClientService extends AbstractService { private static final Log LOG = LogFactory @@ -149,9 +150,19 @@ public GetApplicationAttemptReportResponse getApplicationAttemptReport( GetApplicationAttemptReportResponse response = GetApplicationAttemptReportResponse.newInstance(history .getApplicationAttempt(request.getApplicationAttemptId())); + if (response == null || response.getApplicationAttemptReport() == null) { + // If there is no such attempt, throw + // ApplicationAttemptNotFoundException and let client to handle. + throw new ApplicationAttemptNotFoundException("Application attempt " + + "with id '" + request.getApplicationAttemptId() + + "' doesn't exist in timeline store."); + } return response; } catch (IOException e) { throw new ApplicationAttemptNotFoundException(e.getMessage()); + } catch (NotFoundException e) { + throw new ApplicationAttemptNotFoundException("Application attempt " + + "doesn't exist in timeline store."); } } @@ -163,6 +174,12 @@ public GetApplicationAttemptsResponse getApplicationAttempts( GetApplicationAttemptsResponse .newInstance(new ArrayList(history .getApplicationAttempts(request.getApplicationId()).values())); + if (response == null || response.getApplicationAttemptList().isEmpty()) { + // If there is no such application, throw + // ApplicationNotFoundException and let client to handle. + throw new ApplicationAttemptNotFoundException("Application with id '" + + request.getApplicationId() + "' doesn't have any attempts in timeline store."); + } return response; } @@ -174,9 +191,18 @@ public GetApplicationReportResponse getApplicationReport( GetApplicationReportResponse response = GetApplicationReportResponse.newInstance(history .getApplication(applicationId)); + if (response == null || response.getApplicationReport() == null) { + // If there is no such application, throw + // ApplicationNotFoundException and let client to handle. + throw new ApplicationNotFoundException("Application with id '" + + applicationId + "' doesn't exist in timeline store."); + } return response; } catch (IOException e) { throw new ApplicationNotFoundException(e.getMessage()); + } catch (NotFoundException e) { + throw new ApplicationNotFoundException("Application doesn't " + + "exist in timeline store."); } } @@ -186,6 +212,12 @@ public GetApplicationsResponse getApplications( GetApplicationsResponse response = GetApplicationsResponse.newInstance(new ArrayList( history.getAllApplications().values())); + if (response == null || response.getApplicationList().isEmpty()) { + // If there is no such container, throw + // ContainerNotFoundException and let client to handle. + throw new ApplicationNotFoundException("Application list is empty " + + "in timeline store."); + } return response; } @@ -196,9 +228,18 @@ public GetContainerReportResponse getContainerReport( GetContainerReportResponse response = GetContainerReportResponse.newInstance(history.getContainer(request .getContainerId())); + if (response == null || response.getContainerReport() == null) { + // If there is no such container, throw + // ContainerNotFoundException and let client to handle. + throw new ContainerNotFoundException("Container with id '" + + request.getContainerId() + "' doesn't exist in timeline store."); + } return response; } catch (IOException e) { throw new ContainerNotFoundException(e.getMessage()); + } catch (NotFoundException e) { + throw new ContainerNotFoundException("Contaner doesn't " + + "exist in timeline store."); } } @@ -208,6 +249,12 @@ public GetContainersResponse getContainers(GetContainersRequest request) GetContainersResponse response = GetContainersResponse.newInstance(new ArrayList( history.getContainers(request.getApplicationAttemptId()).values())); + if (response == null || response.getContainerList().isEmpty()) { + // If there is no such application, throw + // ApplicationNotFoundException and let client to handle. + throw new ContainerNotFoundException("No containers found for attempt '" + + request.getApplicationAttemptId() + "' in timeline store."); + } return response; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java index cd429d0..386aafd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java @@ -24,6 +24,8 @@ import java.util.List; import java.util.Map; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; @@ -46,9 +48,6 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; -import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; -import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; -import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; @@ -58,6 +57,7 @@ import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.common.annotations.VisibleForTesting; @@ -69,6 +69,9 @@ @VisibleForTesting static final String UNAVAILABLE = "N/A"; + private static final Log LOG = LogFactory. + getLog(ApplicationHistoryManagerOnTimelineStore.class); + private TimelineDataManager timelineDataManager; private ApplicationACLsManager aclsManager; private String serverHttpAddress; @@ -91,7 +94,16 @@ protected void serviceInit(Configuration conf) throws Exception { @Override public ApplicationReport getApplication(ApplicationId appId) throws YarnException, IOException { - return getApplication(appId, ApplicationReportField.ALL).appReport; + ApplicationReportExt report = + getApplication(appId, ApplicationReportField.ALL); + if (report != null) { + ApplicationReport appReport = report.appReport; + if (appReport != null) { + return appReport; + } + } + throw new NotFoundException("Application with id " + appId + + " not found in timeline store."); } @Override @@ -119,6 +131,10 @@ public ApplicationReport getApplication(ApplicationId appId) throws YarnException, IOException { ApplicationReportExt app = getApplication( appId, ApplicationReportField.USER_AND_ACLS); + if (app == null) { + throw new NotFoundException("Application with id " + + appId + " not found in timeline store."); + } checkAccess(app); TimelineEntities entities = timelineDataManager.getEntities( AppAttemptMetricsConstants.ENTITY_TYPE, @@ -142,15 +158,19 @@ public ApplicationAttemptReport getApplicationAttempt( ApplicationAttemptId appAttemptId) throws YarnException, IOException { ApplicationReportExt app = getApplication( appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS); + if (app == null) { + throw new NotFoundException("Application with id " + + appAttemptId.getApplicationId() + " not found in timeline store."); + } checkAccess(app); TimelineEntity entity = timelineDataManager.getEntity( AppAttemptMetricsConstants.ENTITY_TYPE, appAttemptId.toString(), EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser()); if (entity == null) { - throw new ApplicationAttemptNotFoundException( - "The entity for application attempt " + appAttemptId + + LOG.debug("The entity for application attempt " + appAttemptId + " doesn't exist in the timeline store"); + return null; } else { return convertToApplicationAttemptReport(entity); } @@ -162,15 +182,20 @@ public ContainerReport getContainer(ContainerId containerId) ApplicationReportExt app = getApplication( containerId.getApplicationAttemptId().getApplicationId(), ApplicationReportField.USER_AND_ACLS); + if (app == null) { + throw new NotFoundException("Application with id " + + containerId.getApplicationAttemptId().getApplicationId() + + " not found in timeline store."); + } checkAccess(app); TimelineEntity entity = timelineDataManager.getEntity( ContainerMetricsConstants.ENTITY_TYPE, containerId.toString(), EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser()); if (entity == null) { - throw new ContainerNotFoundException( - "The entity for container " + containerId + + LOG.debug("The entity for container " + containerId + " doesn't exist in the timeline store"); + return null; } else { return convertToContainerReport( entity, serverHttpAddress, app.appReport.getUser()); @@ -181,6 +206,9 @@ public ContainerReport getContainer(ContainerId containerId) public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) throws YarnException, IOException { ApplicationAttemptReport appAttempt = getApplicationAttempt(appAttemptId); + if (appAttempt == null) { + return null; + } return getContainer(appAttempt.getAMContainerId()); } @@ -189,6 +217,10 @@ public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId) ApplicationAttemptId appAttemptId) throws YarnException, IOException { ApplicationReportExt app = getApplication( appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS); + if (app == null) { + throw new NotFoundException("Application with id " + + appAttemptId.getApplicationId() + " not found in timeline store."); + } checkAccess(app); TimelineEntities entities = timelineDataManager.getEntities( ContainerMetricsConstants.ENTITY_TYPE, @@ -539,8 +571,9 @@ private ApplicationReportExt getApplication(ApplicationId appId, appId.toString(), EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser()); if (entity == null) { - throw new ApplicationNotFoundException("The entity for application " + + LOG.debug("The entity for application " + appId + " doesn't exist in the timeline store"); + return null; } else { return generateApplicationReport(entity, field); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java index 7c2593d..a9a62f5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java @@ -41,6 +41,9 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; +import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; +import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; @@ -53,12 +56,13 @@ public class TestApplicationHistoryClientService { private static ApplicationHistoryClientService clientService; + private final static int MAX_APPS = 2; @BeforeClass public static void setup() throws Exception { Configuration conf = new YarnConfiguration(); TimelineStore store = - TestApplicationHistoryManagerOnTimelineStore.createStore(2); + TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS); TimelineACLsManager aclsManager = new TimelineACLsManager(conf); TimelineDataManager dataManager = new TimelineDataManager(store, aclsManager); @@ -71,6 +75,26 @@ public static void setup() throws Exception { } @Test + public void testApplicationNotFound() throws IOException, YarnException { + ApplicationId appId = null; + appId = ApplicationId.newInstance(0, MAX_APPS + 1); + GetApplicationReportRequest request = + GetApplicationReportRequest.newInstance(appId); + try { + @SuppressWarnings("unused") + GetApplicationReportResponse response = + clientService.getClientHandler().getApplicationReport(request); + Assert.fail("Exception should have been thrown before we reach here."); + } catch (ApplicationNotFoundException e) { + //This exception is expected. + Assert.assertTrue(e.getMessage(). + contains("Application doesn't exist in timeline store.")); + } catch (Exception e) { + Assert.fail("Undesired exception caught"); + } + } + + @Test public void testApplicationReport() throws IOException, YarnException { ApplicationId appId = null; appId = ApplicationId.newInstance(0, 1); @@ -102,6 +126,28 @@ public void testApplications() throws IOException, YarnException { } @Test + public void testApplicationAttemptNotFound() throws IOException, YarnException { + ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, MAX_APPS + 1); + GetApplicationAttemptReportRequest request = + GetApplicationAttemptReportRequest.newInstance(appAttemptId); + try { + @SuppressWarnings("unused") + GetApplicationAttemptReportResponse response = + clientService.getClientHandler().getApplicationAttemptReport(request); + Assert.fail("Exception should have been thrown before we reach here."); + } catch (ApplicationAttemptNotFoundException e) { + //This Exception is expected + System.out.println(e.getMessage()); + Assert.assertTrue( e.getMessage().contains("Application attempt") && + e.getMessage().contains("doesn't exist in timeline store.")); + } catch (Exception e) { + Assert.fail("Undesired exception caught"); + } + } + + @Test public void testApplicationAttemptReport() throws IOException, YarnException { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = @@ -138,6 +184,28 @@ public void testApplicationAttempts() throws IOException, YarnException { } @Test + public void testContainerNotFound() throws IOException, YarnException { + ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + ContainerId containerId = ContainerId.newContainerId(appAttemptId, + MAX_APPS + 1); + GetContainerReportRequest request = + GetContainerReportRequest.newInstance(containerId); + try { + @SuppressWarnings("unused") + GetContainerReportResponse response = + clientService.getClientHandler().getContainerReport(request); + } catch (ContainerNotFoundException e) { + //This exception is expected + Assert.assertTrue(e.getMessage().contains("Container with id") && + e.getMessage().contains("doesn't exist in timeline store.")); + } catch (Exception e) { + Assert.fail("Undesired exception caught"); + } + } + + @Test public void testContainerReport() throws IOException, YarnException { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java index 76bf8c3..2b7ec57 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java @@ -81,12 +81,13 @@ private static ApplicationHistoryManagerOnTimelineStore historyManager; private static final String[] USERS = new String[] { "foo" , "bar" }; + private static final int MAX_APPS = 5; @BeforeClass public static void setupClass() throws Exception { Configuration conf = new YarnConfiguration(); TimelineStore store = - TestApplicationHistoryManagerOnTimelineStore.createStore(5); + TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS); TimelineACLsManager aclsManager = new TimelineACLsManager(conf); TimelineDataManager dataManager = new TimelineDataManager(store, aclsManager); @@ -254,6 +255,20 @@ public void testSingleApp() throws Exception { } @Test + public void testInvalidApp() { + ApplicationId appId = ApplicationId.newInstance(0, MAX_APPS + 1); + WebResource r = resource(); + ClientResponse response = + r.path("ws").path("v1").path("applicationhistory").path("apps") + .path(appId.toString()) + .queryParam("user.name", USERS[round]) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals("404 not found expected", Status.NOT_FOUND, + response.getClientResponseStatus()); + } + + @Test public void testMultipleAttempts() throws Exception { ApplicationId appId = ApplicationId.newInstance(0, 1); WebResource r = resource(); @@ -308,6 +323,27 @@ public void testSingleAttempt() throws Exception { } @Test + public void testInvalidAttempt() { + ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, MAX_APPS + 1); + WebResource r = resource(); + ClientResponse response = + r.path("ws").path("v1").path("applicationhistory").path("apps") + .path(appId.toString()).path("appattempts") + .path(appAttemptId.toString()) + .queryParam("user.name", USERS[round]) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + if (round == 1) { + assertEquals(Status.FORBIDDEN, response.getClientResponseStatus()); + return; + } + assertEquals("404 not found expected", Status.NOT_FOUND, + response.getClientResponseStatus()); + } + + @Test public void testMultipleContainers() throws Exception { ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationAttemptId appAttemptId = @@ -373,4 +409,29 @@ public void testSingleContainer() throws Exception { container.getString("containerState")); } + @Test + public void testInvalidContainer() throws Exception { + ApplicationId appId = ApplicationId.newInstance(0, 1); + ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + ContainerId containerId = ContainerId.newContainerId(appAttemptId, + MAX_APPS + 1); + WebResource r = resource(); + ClientResponse response = + r.path("ws").path("v1").path("applicationhistory").path("apps") + .path(appId.toString()).path("appattempts") + .path(appAttemptId.toString()).path("containers") + .path(containerId.toString()) + .queryParam("user.name", USERS[round]) + .accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + if (round == 1) { + assertEquals( + Status.FORBIDDEN, response.getClientResponseStatus()); + return; + } + assertEquals("404 not found expected", Status.NOT_FOUND, + response.getClientResponseStatus()); + } + } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java index 385d10a..353442a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java @@ -223,6 +223,8 @@ public ApplicationReport run() throws Exception { } }); } + } catch (NotFoundException e) { + app = null; } catch (Exception e) { rewrapAndThrowException(e); } @@ -280,6 +282,8 @@ public ApplicationAttemptReport run() throws Exception { } }); } + } catch (NotFoundException e) { + appAttempt = null; } catch (Exception e) { rewrapAndThrowException(e); } @@ -341,6 +345,8 @@ public ContainerReport run() throws Exception { } }); } + } catch (NotFoundException e) { + container = null; } catch (Exception e) { rewrapAndThrowException(e); }