commit 9e3b5bcc84fc684ef6ffac42516418418250df87 Author: Vinod Kumar Vavilapalli Date: Mon Jan 13 12:12:59 2014 -0800 Fixing test issues. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 4a4a750..09e25ca 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -263,11 +263,6 @@ RM_PREFIX + "nodemanagers.heartbeat-interval-ms"; public static final long DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS = 1000; - /** The setting that controls whether RM writes history data. */ - public static final String RM_HISTORY_WRITER_ENABLED = RM_PREFIX - + "history-writer.enabled"; - public static final boolean DEFAULT_RM_HISTORY_WRITER_ENABLED = false; - /** Number of worker threads that write the history data. */ public static final String RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE = RM_PREFIX + "history-writer.multi-threaded-dispatcher.pool-size"; @@ -950,6 +945,11 @@ public static final String AHS_PREFIX = YARN_PREFIX + "ahs."; + /** The setting that controls whether history-service is enabled or not.. */ + public static final String YARN_HISTORY_SERVICE_ENABLED = AHS_PREFIX + + ".enabled"; + public static final boolean DEFAULT_YARN_HISTORY_SERVICE_ENABLED = false; + /** URI for FileSystemApplicationHistoryStore */ public static final String FS_HISTORY_STORE_URI = AHS_PREFIX + "fs-history-store.uri"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 23a62de..7eda37b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -87,6 +87,7 @@ protected long submitPollIntervalMillis; private long asyncApiPollIntervalMillis; protected AHSClient historyClient; + private boolean historyServiceEnabled; private static final String ROOT = "root"; @@ -107,8 +108,14 @@ protected void serviceInit(Configuration conf) throws Exception { YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS, YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS); } - historyClient = AHSClientImpl.createAHSClient(); - historyClient.init(getConfig()); + + if (conf.getBoolean(YarnConfiguration.YARN_HISTORY_SERVICE_ENABLED, + YarnConfiguration.DEFAULT_YARN_HISTORY_SERVICE_ENABLED)) { + historyServiceEnabled = true; + historyClient = AHSClientImpl.createAHSClient(); + historyClient.init(getConfig()); + } + super.serviceInit(conf); } @@ -117,7 +124,9 @@ protected void serviceStart() throws Exception { try { rmClient = ClientRMProxy.createRMProxy(getConfig(), ApplicationClientProtocol.class); - historyClient.start(); + if (historyServiceEnabled) { + historyClient.start(); + } } catch (IOException e) { throw new YarnRuntimeException(e); } @@ -129,7 +138,9 @@ protected void serviceStop() throws Exception { if (this.rmClient != null) { RPC.stopProxy(this.rmClient); } - historyClient.stop(); + if (historyServiceEnabled) { + historyClient.stop(); + } super.serviceStop(); } @@ -225,11 +236,19 @@ public ApplicationReport getApplicationReport(ApplicationId appId) request.setApplicationId(appId); response = rmClient.getApplicationReport(request); } catch (YarnException e) { + + if (!historyServiceEnabled) { + // Just throw it as usual if historyService is not enabled. + throw e; + } + + // Even if history-service is enabled, treat all exceptions still the same + // except the following if (!(e.getClass() == ApplicationNotFoundException.class)) { throw e; } } - if (response == null || response.getApplicationReport() == null) { + if ((response == null || response.getApplicationReport() == null)) { return historyClient.getApplicationReport(appId); } return response.getApplicationReport(); @@ -397,25 +416,37 @@ public void setRMClient(ApplicationClientProtocol rmClient) { @Override public ApplicationAttemptReport getApplicationAttemptReport( ApplicationAttemptId appAttemptId) throws YarnException, IOException { - return historyClient.getApplicationAttemptReport(appAttemptId); + if (historyServiceEnabled) { + return historyClient.getApplicationAttemptReport(appAttemptId); + } + throw new YarnException("History service is not enabled."); } @Override public List getApplicationAttempts( ApplicationId appId) throws YarnException, IOException { - return historyClient.getApplicationAttempts(appId); + if (historyServiceEnabled) { + return historyClient.getApplicationAttempts(appId); + } + throw new YarnException("History service is not enabled."); } @Override public ContainerReport getContainerReport(ContainerId containerId) throws YarnException, IOException { - return historyClient.getContainerReport(containerId); + if (historyServiceEnabled) { + return historyClient.getContainerReport(containerId); + } + throw new YarnException("History service is not enabled."); } @Override public List getContainers( ApplicationAttemptId applicationAttemptId) throws YarnException, IOException { - return historyClient.getContainers(applicationAttemptId); + if (historyServiceEnabled) { + return historyClient.getContainers(applicationAttemptId); + } + throw new YarnException("History service is not enabled."); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java index 51c01ff..15da7e4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java @@ -201,7 +201,7 @@ public void testMassiveWriteContainerHistory() throws IOException { writeContainerFinishData(containerId); } long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb; - Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 100); + Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 200); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java index fa17e20..bd34bbd 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java @@ -68,6 +68,7 @@ private Dispatcher dispatcher; private ApplicationHistoryWriter writer; + private boolean historyServiceEnabled; public RMApplicationHistoryWriter() { super(RMApplicationHistoryWriter.class.getName()); @@ -76,6 +77,11 @@ public RMApplicationHistoryWriter() { @Override protected synchronized void serviceInit( Configuration conf) throws Exception { + + historyServiceEnabled = conf.getBoolean( + YarnConfiguration.YARN_HISTORY_SERVICE_ENABLED, + YarnConfiguration.DEFAULT_YARN_HISTORY_SERVICE_ENABLED); + writer = createApplicationHistoryStore(conf); addIfService(writer); @@ -96,12 +102,9 @@ protected Dispatcher createDispatcher(Configuration conf) { protected ApplicationHistoryStore createApplicationHistoryStore( Configuration conf) { - boolean ahsEnabled = conf.getBoolean( - YarnConfiguration.RM_HISTORY_WRITER_ENABLED, - YarnConfiguration.DEFAULT_RM_HISTORY_WRITER_ENABLED); // If the history writer is not enabled, a dummy store will be used to // write nothing - if (ahsEnabled) { + if (historyServiceEnabled) { try { Class storeClass = conf.getClass(YarnConfiguration.RM_HISTORY_WRITER_CLASS, @@ -225,42 +228,49 @@ public void applicationFinished(RMApp app) { @SuppressWarnings("unchecked") public void applicationAttemptStarted(RMAppAttempt appAttempt) { - dispatcher.getEventHandler().handle( + if (historyServiceEnabled) { + dispatcher.getEventHandler().handle( new WritingApplicationAttemptStartEvent(appAttempt.getAppAttemptId(), - ApplicationAttemptStartData.newInstance( - appAttempt.getAppAttemptId(), appAttempt.getHost(), - appAttempt.getRpcPort(), appAttempt.getMasterContainer() - .getId()))); + ApplicationAttemptStartData.newInstance(appAttempt.getAppAttemptId(), + appAttempt.getHost(), appAttempt.getRpcPort(), appAttempt + .getMasterContainer().getId()))); + } } @SuppressWarnings("unchecked") public void applicationAttemptFinished(RMAppAttempt appAttempt) { - dispatcher.getEventHandler().handle( + if (historyServiceEnabled) { + dispatcher.getEventHandler().handle( new WritingApplicationAttemptFinishEvent(appAttempt.getAppAttemptId(), - ApplicationAttemptFinishData.newInstance(appAttempt - .getAppAttemptId(), appAttempt.getDiagnostics().toString(), - appAttempt.getTrackingUrl(), appAttempt - .getFinalApplicationStatus(), appAttempt - .createApplicationAttemptState()))); + ApplicationAttemptFinishData.newInstance( + appAttempt.getAppAttemptId(), appAttempt.getDiagnostics() + .toString(), appAttempt.getTrackingUrl(), appAttempt + .getFinalApplicationStatus(), appAttempt + .createApplicationAttemptState()))); + } } @SuppressWarnings("unchecked") public void containerStarted(RMContainer container) { - dispatcher.getEventHandler().handle( + if (historyServiceEnabled) { + dispatcher.getEventHandler().handle( new WritingContainerStartEvent(container.getContainerId(), - ContainerStartData.newInstance(container.getContainerId(), - container.getAllocatedResource(), container.getAllocatedNode(), - container.getAllocatedPriority(), container.getStartTime()))); + ContainerStartData.newInstance(container.getContainerId(), + container.getAllocatedResource(), container.getAllocatedNode(), + container.getAllocatedPriority(), container.getStartTime()))); + } } @SuppressWarnings("unchecked") public void containerFinished(RMContainer container) { - dispatcher.getEventHandler().handle( + if (historyServiceEnabled) { + dispatcher.getEventHandler().handle( new WritingContainerFinishEvent(container.getContainerId(), - ContainerFinishData.newInstance(container.getContainerId(), - container.getFinishTime(), container.getDiagnosticsInfo(), - container.getLogURL(), container.getContainerExitStatus(), - container.getContainerState()))); + ContainerFinishData.newInstance(container.getContainerId(), + container.getFinishTime(), container.getDiagnosticsInfo(), + container.getLogURL(), container.getContainerExitStatus(), + container.getContainerState()))); + } } /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java index 4050493..80fe913 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java @@ -105,7 +105,7 @@ public void testReleaseWhileRunning() { drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); assertEquals( - "http://host:3465/node/containerlogs/container_1_0001_01_000001/user", + "http://host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify RELEASED and associated actions. @@ -192,7 +192,7 @@ public void testExpireWhileRunning() { drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); assertEquals( - "http://host:3465/node/containerlogs/container_1_0001_01_000001/user", + "http://host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify EXPIRE and associated actions.