diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index c29707c82ee..6d6c1d0b59e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2226,13 +2226,34 @@ public static boolean isAclEnabled(Configuration conf) {
public static final String NM_AUX_SERVICES =
NM_PREFIX + "aux-services";
+ /**
+ * Boolean indicating whether loading aux services from a manifest is
+ * enabled. If enabled, aux services may be dynamically modified through
+ * reloading the manifest via filesystem changes or a REST API. When
+ * enabled, aux services configuration properties unrelated to the manifest
+ * will be ignored.
+ */
+ public static final String NM_AUX_SERVICES_MANIFEST_ENABLED =
+ NM_AUX_SERVICES + ".manifest.enabled";
+
+ public static final boolean DEFAULT_NM_AUX_SERVICES_MANIFEST_ENABLED =
+ false;
+
+ /**
+ * File containing auxiliary service specifications.
+ */
public static final String NM_AUX_SERVICES_MANIFEST =
NM_AUX_SERVICES + ".manifest";
+ /**
+ * Interval at which manifest file will be reloaded when modifications are
+ * found (<= 0 means that the file will not be checked for modifications
+ * and reloaded).
+ */
public static final String NM_AUX_SERVICES_MANIFEST_RELOAD_MS =
NM_AUX_SERVICES + ".manifest.reload-ms";
- public static final long DEFAULT_NM_AUX_SERVICES_MANIFEST_RELOAD_MS = 120000;
+ public static final long DEFAULT_NM_AUX_SERVICES_MANIFEST_RELOAD_MS = 0;
public static final String NM_AUX_SERVICE_FMT =
NM_PREFIX + "aux-services.%s.class";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index a0e0eda74f4..23b65b9df9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1922,9 +1922,17 @@
- A file containing auxiliary service specifications. If
- manifest file is specified, yarn.nodemanager.aux-services and other
- aux services configuration properties will be ignored.
+ Boolean indicating whether loading aux services from a manifest
+ is enabled. If enabled, aux services may be dynamically modified through
+ reloading the manifest via filesystem changes or a REST API. When
+ enabled, aux services configuration properties unrelated to the manifest
+ will be ignored.
+ yarn.nodemanager.aux-services.manifest.enabled
+ false
+
+
+
+ A file containing auxiliary service specifications.
yarn.nodemanager.aux-services.manifest
@@ -1933,7 +1941,7 @@
Length of time in ms to wait between reloading aux services
manifest. If 0 or less, manifest will not be reloaded.
yarn.nodemanager.aux-services.manifest.reload-ms
-
+ 0
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
index 01d901a4cde..38cd1d70b9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
@@ -110,6 +110,7 @@
private Path stateStoreRoot = null;
private FileSystem stateStoreFs = null;
+ private boolean manifestEnabled = false;
private Path manifest;
private FileSystem manifestFS;
private Timer manifestReloadTimer;
@@ -140,6 +141,13 @@
}
/**
+ * Returns whether aux services manifest / dynamic loading is enabled.
+ */
+ public boolean isManifestEnabled() {
+ return manifestEnabled;
+ }
+
+ /**
* Adds a service to the service map.
*
* @param name aux service name
@@ -480,7 +488,8 @@ private AuxiliaryService initAuxService(AuxServiceRecord service,
*
* @throws IOException if manifest can't be loaded
*/
- private void reloadManifest() throws IOException {
+ @VisibleForTesting
+ protected void reloadManifest() throws IOException {
loadManifest(getConfig(), true);
}
@@ -488,9 +497,14 @@ private void reloadManifest() throws IOException {
* Reloads auxiliary services. Must be called after service init.
*
* @param services a list of auxiliary services
- * @throws IOException if aux services have not been started yet
+ * @throws IOException if aux services have not been started yet or dynamic
+ * reloading is not enabled
*/
public void reload(AuxServiceRecords services) throws IOException {
+ if (!manifestEnabled) {
+ throw new IOException("Dynamic reloading is not enabled via " +
+ YarnConfiguration.NM_AUX_SERVICES_MANIFEST_ENABLED);
+ }
if (getServiceState() != Service.STATE.STARTED) {
throw new IOException("Auxiliary services have not been started yet, " +
"please retry later");
@@ -578,6 +592,10 @@ private synchronized AuxServiceRecords maybeReadManifestFile() throws
@VisibleForTesting
protected synchronized void loadManifest(Configuration conf, boolean
startServices) throws IOException {
+ if (!manifestEnabled) {
+ throw new IOException("Dynamic reloading is not enabled via " +
+ YarnConfiguration.NM_AUX_SERVICES_MANIFEST_ENABLED);
+ }
if (manifest == null) {
return;
}
@@ -730,8 +748,10 @@ public synchronized void serviceInit(Configuration conf) throws Exception {
STATE_STORE_ROOT_NAME);
stateStoreFs = FileSystem.getLocal(conf);
}
- String manifestStr = conf.get(YarnConfiguration.NM_AUX_SERVICES_MANIFEST);
- if (manifestStr == null) {
+ manifestEnabled = conf.getBoolean(
+ YarnConfiguration.NM_AUX_SERVICES_MANIFEST_ENABLED,
+ YarnConfiguration.DEFAULT_NM_AUX_SERVICES_MANIFEST_ENABLED);
+ if (!manifestEnabled) {
Collection auxNames = conf.getStringCollection(
YarnConfiguration.NM_AUX_SERVICES);
for (final String sName : auxNames) {
@@ -742,14 +762,20 @@ public synchronized void serviceInit(Configuration conf) throws Exception {
addService(sName, s, service);
}
} else {
- manifest = new Path(manifestStr);
- manifestFS = FileSystem.get(new URI(manifestStr), conf);
- loadManifest(conf, false);
+ String manifestStr = conf.get(YarnConfiguration.NM_AUX_SERVICES_MANIFEST);
+ if (manifestStr != null) {
+ manifest = new Path(manifestStr);
+ manifestFS = FileSystem.get(new URI(manifestStr), conf);
+ loadManifest(conf, false);
+ manifestReloadInterval = conf.getLong(
+ YarnConfiguration.NM_AUX_SERVICES_MANIFEST_RELOAD_MS,
+ YarnConfiguration.DEFAULT_NM_AUX_SERVICES_MANIFEST_RELOAD_MS);
+ manifestReloadTask = new ManifestReloadTask();
+ } else {
+ LOG.info("Auxiliary services manifest is enabled, but no manifest " +
+ "file is specified in the configuration.");
+ }
}
- manifestReloadInterval = conf.getLong(
- YarnConfiguration.NM_AUX_SERVICES_MANIFEST_RELOAD_MS,
- YarnConfiguration.DEFAULT_NM_AUX_SERVICES_MANIFEST_RELOAD_MS);
- manifestReloadTask = new ManifestReloadTask();
super.serviceInit(conf);
}
@@ -781,8 +807,10 @@ public synchronized void serviceStart() throws Exception {
String name = entry.getKey();
startAuxService(name, service, serviceRecordMap.get(name));
}
- if (manifest != null && manifestReloadInterval > 0) {
- manifestReloadTimer = new Timer("AuxServicesManifestRelaod-Timer",
+ if (manifestEnabled && manifest != null && manifestReloadInterval > 0) {
+ LOG.info("Scheduling reloading auxiliary services manifest file at " +
+ "interval " + manifestReloadInterval + " ms");
+ manifestReloadTimer = new Timer("AuxServicesManifestReload-Timer",
true);
manifestReloadTimer.schedule(manifestReloadTask,
manifestReloadInterval, manifestReloadInterval);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index 2267e072a62..a3c0b8b5afe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -567,6 +567,9 @@ public Object getNMResourceInfo(
public AuxiliaryServicesInfo getAuxiliaryServices(@javax.ws.rs.core.Context
HttpServletRequest hsr) {
init();
+ if (!this.nmContext.getAuxServices().isManifestEnabled()) {
+ throw new NotFoundException("Auxiliary services manifest is not enabled");
+ }
AuxiliaryServicesInfo auxiliaryServices = new AuxiliaryServicesInfo();
Collection loadedServices = nmContext.getAuxServices()
.getServiceRecords();
@@ -582,6 +585,10 @@ public AuxiliaryServicesInfo getAuxiliaryServices(@javax.ws.rs.core.Context
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public Response putAuxiliaryServices(@javax.ws.rs.core.Context
HttpServletRequest req, AuxServiceRecords services) {
+ init();
+ if (!this.nmContext.getAuxServices().isManifestEnabled()) {
+ throw new NotFoundException("Auxiliary services manifest is not enabled");
+ }
if (!hasAdminAccess(req)) {
return Response.status(Status.FORBIDDEN).build();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
index 6cf2b7e6a46..aef40911f2d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
@@ -252,6 +252,7 @@ public ByteBuffer getMetaData() {
private void writeManifestFile(AuxServiceRecords services, Configuration
conf) throws IOException {
+ conf.setBoolean(YarnConfiguration.NM_AUX_SERVICES_MANIFEST_ENABLED, true);
conf.set(YarnConfiguration.NM_AUX_SERVICES_MANIFEST, manifest
.getAbsolutePath());
mapper.writeValue(manifest, services);
@@ -901,6 +902,7 @@ public void testRemoveManifest() throws IOException {
@Test
public void testManualReload() throws IOException {
+ Assume.assumeTrue(useManifest);
Configuration conf = getABConf();
final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER,
MOCK_CONTEXT, MOCK_DEL_SERVICE);
@@ -921,4 +923,28 @@ public void testManualReload() throws IOException {
assertEquals(0, aux.getServices().size());
aux.stop();
}
+
+ @Test
+ public void testReloadWhenDisabled() throws IOException {
+ Configuration conf = new Configuration();
+ final AuxServices aux = new AuxServices(MOCK_AUX_PATH_HANDLER,
+ MOCK_CONTEXT, MOCK_DEL_SERVICE);
+ aux.init(conf);
+ try {
+ aux.reload(null);
+ Assert.fail("Should receive the exception.");
+ } catch (IOException e) {
+ assertTrue("Wrong message: " + e.getMessage(),
+ e.getMessage().equals("Dynamic reloading is not enabled via " +
+ YarnConfiguration.NM_AUX_SERVICES_MANIFEST_ENABLED));
+ }
+ try {
+ aux.reloadManifest();
+ Assert.fail("Should receive the exception.");
+ } catch (IOException e) {
+ assertTrue("Wrong message: " + e.getMessage(),
+ e.getMessage().equals("Dynamic reloading is not enabled via " +
+ YarnConfiguration.NM_AUX_SERVICES_MANIFEST_ENABLED));
+ }
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java
index b6cc0dba4ff..4cb030dd71e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesAuxServices.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -33,6 +35,7 @@
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
+import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.filter.LoggingFilter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
@@ -54,6 +57,7 @@
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.AfterClass;
import org.junit.Before;
@@ -195,6 +199,7 @@ public void testNodeAuxServicesNone() throws Exception {
private void addAuxServices(AuxServiceRecord... records) {
AuxServices auxServices = mock(AuxServices.class);
when(auxServices.getServiceRecords()).thenReturn(Arrays.asList(records));
+ when(auxServices.isManifestEnabled()).thenReturn(true);
nmContext.setAuxServices(auxServices);
}
@@ -238,7 +243,7 @@ public void testNodeHelper(String path, String media) throws Exception {
}
@Test
- public void testNodeContainerXML() throws Exception {
+ public void testNodeAuxServicesXML() throws Exception {
AuxServiceRecord r1 = new AuxServiceRecord().name("name1").launchTime(new
Date(123L)).version("1");
AuxServiceRecord r2 = new AuxServiceRecord().name("name2").launchTime(new
@@ -259,10 +264,42 @@ public void testNodeContainerXML() throws Exception {
Document dom = db.parse(is);
NodeList nodes = dom.getElementsByTagName("service");
assertEquals("incorrect number of elements", 2, nodes.getLength());
- verifyContainersInfoXML(nodes, r1, r2);
+ verifyAuxServicesInfoXML(nodes, r1, r2);
}
- public void verifyContainersInfoXML(NodeList nodes, AuxServiceRecord...
+ @Test
+ public void testAuxServicesDisabled() throws JSONException, Exception {
+ AuxServices auxServices = mock(AuxServices.class);
+ when(auxServices.isManifestEnabled()).thenReturn(false);
+ nmContext.setAuxServices(auxServices);
+ WebResource r = resource();
+ try {
+ r.path("ws").path("v1").path("node").path(AUX_SERVICES_PATH)
+ .accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
+ fail("should have thrown exception on invalid user query");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertResponseStatusCode(ClientResponse.Status.NOT_FOUND, response.getStatusInfo());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ WebServicesTestUtils.checkStringMatch(
+ "exception message",
+ "java.lang.Exception: Auxiliary services manifest is not enabled",
+ message);
+ WebServicesTestUtils.checkStringMatch("exception type",
+ "NotFoundException", type);
+ WebServicesTestUtils.checkStringMatch("exception classname",
+ "org.apache.hadoop.yarn.webapp.NotFoundException", classname);
+ }
+ }
+
+ public void verifyAuxServicesInfoXML(NodeList nodes, AuxServiceRecord...
records) {
for (int i = 0; i < nodes.getLength(); i++) {
Element element = (Element) nodes.item(i);